1 import os.path, re, urllib, time
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow import rend
16 from allmydata import interfaces, uri, webish, dirnode
17 from allmydata.storage.shares import get_share_file
18 from allmydata.storage_client import StorageFarmBroker
19 from allmydata.immutable import upload
20 from allmydata.immutable.downloader.status import DownloadStatus
21 from allmydata.dirnode import DirectoryNode
22 from allmydata.nodemaker import NodeMaker
23 from allmydata.unknown import UnknownNode
24 from allmydata.web import status, common
25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
26 from allmydata.util import fileutil, base32, hashutil
27 from allmydata.util.consumer import download_to_data
28 from allmydata.util.netstring import split_netstring
29 from allmydata.util.encodingutil import to_str
30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
31 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
32 make_mutable_file_uri, create_mutable_filenode
33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
34 from allmydata.mutable import servermap, publish, retrieve
35 import allmydata.test.common_util as testutil
36 from allmydata.test.no_network import GridTestMixin
37 from allmydata.test.common_web import HTTPClientGETFactory, \
39 from allmydata.client import Client, SecretHolder
40 from allmydata.introducer import IntroducerNode
42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
43 # create a webserver that works against them
45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
54 class FakeStatsProvider:
56 stats = {'stats': {}, 'counters': {}}
59 class FakeNodeMaker(NodeMaker):
64 'max_segment_size':128*1024 # 1024=KiB
66 def _create_lit(self, cap):
67 return FakeCHKFileNode(cap)
68 def _create_immutable(self, cap):
69 return FakeCHKFileNode(cap)
70 def _create_mutable(self, cap):
71 return FakeMutableFileNode(None,
73 self.encoding_params, None).init_from_cap(cap)
74 def create_mutable_file(self, contents="", keysize=None,
75 version=SDMF_VERSION):
76 n = FakeMutableFileNode(None, None, self.encoding_params, None)
77 return n.create(contents, version=version)
79 class FakeUploader(service.Service):
81 def upload(self, uploadable):
82 d = uploadable.get_size()
83 d.addCallback(lambda size: uploadable.read(size))
86 n = create_chk_filenode(data)
87 ur = upload.UploadResults(file_size=len(data),
94 uri_extension_data={},
95 uri_extension_hash="fake",
96 verifycapstr="fakevcap")
97 ur.set_uri(n.get_uri())
99 d.addCallback(_got_data)
101 def get_helper_info(self):
105 def __init__(self, binaryserverid):
106 self.binaryserverid = binaryserverid
107 def get_name(self): return "short"
108 def get_longname(self): return "long"
109 def get_serverid(self): return self.binaryserverid
112 ds = DownloadStatus("storage_index", 1234)
115 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
116 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
117 storage_index = hashutil.storage_index_hash("SI")
118 e0 = ds.add_segment_request(0, now)
120 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
121 e1 = ds.add_segment_request(1, now+2)
123 # two outstanding requests
124 e2 = ds.add_segment_request(2, now+4)
125 e3 = ds.add_segment_request(3, now+5)
126 del e2,e3 # hush pyflakes
128 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
129 e = ds.add_segment_request(4, now)
131 e.deliver(now, 0, 140, 0.5)
133 e = ds.add_dyhb_request(serverA, now)
134 e.finished([1,2], now+1)
135 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
137 e = ds.add_read_event(0, 120, now)
138 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
140 e = ds.add_read_event(120, 30, now+2) # left unfinished
142 e = ds.add_block_request(serverA, 1, 100, 20, now)
143 e.finished(20, now+1)
144 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
146 # make sure that add_read_event() can come first too
147 ds1 = DownloadStatus(storage_index, 1234)
148 e = ds1.add_read_event(0, 120, now)
149 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
155 _all_upload_status = [upload.UploadStatus()]
156 _all_download_status = [build_one_ds()]
157 _all_mapupdate_statuses = [servermap.UpdateStatus()]
158 _all_publish_statuses = [publish.PublishStatus()]
159 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
161 def list_all_upload_statuses(self):
162 return self._all_upload_status
163 def list_all_download_statuses(self):
164 return self._all_download_status
165 def list_all_mapupdate_statuses(self):
166 return self._all_mapupdate_statuses
167 def list_all_publish_statuses(self):
168 return self._all_publish_statuses
169 def list_all_retrieve_statuses(self):
170 return self._all_retrieve_statuses
171 def list_all_helper_statuses(self):
174 class FakeClient(Client):
176 # don't upcall to Client.__init__, since we only want to initialize a
178 service.MultiService.__init__(self)
179 self.nodeid = "fake_nodeid"
180 self.nickname = "fake_nickname"
181 self.introducer_furl = "None"
182 self.stats_provider = FakeStatsProvider()
183 self._secret_holder = SecretHolder("lease secret", "convergence secret")
185 self.convergence = "some random string"
186 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
187 self.introducer_client = None
188 self.history = FakeHistory()
189 self.uploader = FakeUploader()
190 self.uploader.setServiceParent(self)
191 self.blacklist = None
192 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
195 self.mutable_file_default = SDMF_VERSION
197 def startService(self):
198 return service.MultiService.startService(self)
199 def stopService(self):
200 return service.MultiService.stopService(self)
202 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
204 class WebMixin(object):
206 self.s = FakeClient()
207 self.s.startService()
208 self.staticdir = self.mktemp()
210 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
212 self.ws.setServiceParent(self.s)
213 self.webish_port = self.ws.getPortnum()
214 self.webish_url = self.ws.getURL()
215 assert self.webish_url.endswith("/")
216 self.webish_url = self.webish_url[:-1] # these tests add their own /
218 l = [ self.s.create_dirnode() for x in range(6) ]
219 d = defer.DeferredList(l)
221 self.public_root = res[0][1]
222 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
223 self.public_url = "/uri/" + self.public_root.get_uri()
224 self.private_root = res[1][1]
228 self._foo_uri = foo.get_uri()
229 self._foo_readonly_uri = foo.get_readonly_uri()
230 self._foo_verifycap = foo.get_verify_cap().to_string()
231 # NOTE: we ignore the deferred on all set_uri() calls, because we
232 # know the fake nodes do these synchronously
233 self.public_root.set_uri(u"foo", foo.get_uri(),
234 foo.get_readonly_uri())
236 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
237 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
238 self._bar_txt_verifycap = n.get_verify_cap().to_string()
241 # XXX: Do we ever use this?
242 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
244 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
247 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
248 assert self._quux_txt_uri.startswith("URI:MDMF")
249 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
251 foo.set_uri(u"empty", res[3][1].get_uri(),
252 res[3][1].get_readonly_uri())
253 sub_uri = res[4][1].get_uri()
254 self._sub_uri = sub_uri
255 foo.set_uri(u"sub", sub_uri, sub_uri)
256 sub = self.s.create_node_from_uri(sub_uri)
259 _ign, n, blocking_uri = self.makefile(1)
260 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
262 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
263 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
264 # still think of it as an umlaut
265 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
267 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
268 self._baz_file_uri = baz_file
269 sub.set_uri(u"baz.txt", baz_file, baz_file)
271 _ign, n, self._bad_file_uri = self.makefile(3)
272 # this uri should not be downloadable
273 del FakeCHKFileNode.all_contents[self._bad_file_uri]
276 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
277 rodir.get_readonly_uri())
278 rodir.set_uri(u"nor", baz_file, baz_file)
284 # public/foo/quux.txt
285 # public/foo/blockingfile
288 # public/foo/sub/baz.txt
290 # public/reedownlee/nor
291 self.NEWFILE_CONTENTS = "newfile contents\n"
293 return foo.get_metadata_for(u"bar.txt")
295 def _got_metadata(metadata):
296 self._bar_txt_metadata = metadata
297 d.addCallback(_got_metadata)
300 def makefile(self, number):
301 contents = "contents of file %s\n" % number
302 n = create_chk_filenode(contents)
303 return contents, n, n.get_uri()
305 def makefile_mutable(self, number, mdmf=False):
306 contents = "contents of mutable file %s\n" % number
307 n = create_mutable_filenode(contents, mdmf)
308 return contents, n, n.get_uri(), n.get_readonly_uri()
311 return self.s.stopService()
313 def failUnlessIsBarDotTxt(self, res):
314 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
316 def failUnlessIsQuuxDotTxt(self, res):
317 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
319 def failUnlessIsBazDotTxt(self, res):
320 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
322 def failUnlessIsSubBazDotTxt(self, res):
323 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
325 def failUnlessIsBarJSON(self, res):
326 data = simplejson.loads(res)
327 self.failUnless(isinstance(data, list))
328 self.failUnlessEqual(data[0], "filenode")
329 self.failUnless(isinstance(data[1], dict))
330 self.failIf(data[1]["mutable"])
331 self.failIfIn("rw_uri", data[1]) # immutable
332 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
333 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
334 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
336 def failUnlessIsQuuxJSON(self, res, readonly=False):
337 data = simplejson.loads(res)
338 self.failUnless(isinstance(data, list))
339 self.failUnlessEqual(data[0], "filenode")
340 self.failUnless(isinstance(data[1], dict))
342 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
344 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
345 self.failUnless(metadata['mutable'])
347 self.failIfIn("rw_uri", metadata)
349 self.failUnlessIn("rw_uri", metadata)
350 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
351 self.failUnlessIn("ro_uri", metadata)
352 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
353 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
355 def failUnlessIsFooJSON(self, res):
356 data = simplejson.loads(res)
357 self.failUnless(isinstance(data, list))
358 self.failUnlessEqual(data[0], "dirnode", res)
359 self.failUnless(isinstance(data[1], dict))
360 self.failUnless(data[1]["mutable"])
361 self.failUnlessIn("rw_uri", data[1]) # mutable
362 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
363 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
364 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
366 kidnames = sorted([unicode(n) for n in data[1]["children"]])
367 self.failUnlessEqual(kidnames,
368 [u"bar.txt", u"baz.txt", u"blockingfile",
369 u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
370 kids = dict( [(unicode(name),value)
372 in data[1]["children"].iteritems()] )
373 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
374 self.failUnlessIn("metadata", kids[u"sub"][1])
375 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
376 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
377 self.failUnlessIn("linkcrtime", tahoe_md)
378 self.failUnlessIn("linkmotime", tahoe_md)
379 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
380 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
381 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
382 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
383 self._bar_txt_verifycap)
384 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
385 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
386 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
387 self._bar_txt_metadata["tahoe"]["linkcrtime"])
388 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
390 self.failUnlessIn("quux.txt", kids)
391 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
393 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
394 self._quux_txt_readonly_uri)
396 def GET(self, urlpath, followRedirect=False, return_response=False,
398 # if return_response=True, this fires with (data, statuscode,
399 # respheaders) instead of just data.
400 assert not isinstance(urlpath, unicode)
401 url = self.webish_url + urlpath
402 factory = HTTPClientGETFactory(url, method="GET",
403 followRedirect=followRedirect, **kwargs)
404 reactor.connectTCP("localhost", self.webish_port, factory)
407 return (data, factory.status, factory.response_headers)
409 d.addCallback(_got_data)
410 return factory.deferred
412 def HEAD(self, urlpath, return_response=False, **kwargs):
413 # this requires some surgery, because twisted.web.client doesn't want
414 # to give us back the response headers.
415 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
416 reactor.connectTCP("localhost", self.webish_port, factory)
419 return (data, factory.status, factory.response_headers)
421 d.addCallback(_got_data)
422 return factory.deferred
424 def PUT(self, urlpath, data, **kwargs):
425 url = self.webish_url + urlpath
426 return client.getPage(url, method="PUT", postdata=data, **kwargs)
428 def DELETE(self, urlpath):
429 url = self.webish_url + urlpath
430 return client.getPage(url, method="DELETE")
432 def POST(self, urlpath, followRedirect=False, **fields):
433 sepbase = "boogabooga"
437 form.append('Content-Disposition: form-data; name="_charset"')
441 for name, value in fields.iteritems():
442 if isinstance(value, tuple):
443 filename, value = value
444 form.append('Content-Disposition: form-data; name="%s"; '
445 'filename="%s"' % (name, filename.encode("utf-8")))
447 form.append('Content-Disposition: form-data; name="%s"' % name)
449 if isinstance(value, unicode):
450 value = value.encode("utf-8")
453 assert isinstance(value, str)
460 body = "\r\n".join(form) + "\r\n"
461 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
462 return self.POST2(urlpath, body, headers, followRedirect)
464 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
465 url = self.webish_url + urlpath
466 return client.getPage(url, method="POST", postdata=body,
467 headers=headers, followRedirect=followRedirect)
469 def shouldFail(self, res, expected_failure, which,
470 substring=None, response_substring=None):
471 if isinstance(res, failure.Failure):
472 res.trap(expected_failure)
474 self.failUnlessIn(substring, str(res), which)
475 if response_substring:
476 self.failUnlessIn(response_substring, res.value.response, which)
478 self.fail("%s was supposed to raise %s, not get '%s'" %
479 (which, expected_failure, res))
481 def shouldFail2(self, expected_failure, which, substring,
483 callable, *args, **kwargs):
484 assert substring is None or isinstance(substring, str)
485 assert response_substring is None or isinstance(response_substring, str)
486 d = defer.maybeDeferred(callable, *args, **kwargs)
488 if isinstance(res, failure.Failure):
489 res.trap(expected_failure)
491 self.failUnlessIn(substring, str(res),
492 "'%s' not in '%s' for test '%s'" % \
493 (substring, str(res), which))
494 if response_substring:
495 self.failUnlessIn(response_substring, res.value.response,
496 "'%s' not in '%s' for test '%s'" % \
497 (response_substring, res.value.response,
500 self.fail("%s was supposed to raise %s, not get '%s'" %
501 (which, expected_failure, res))
505 def should404(self, res, which):
506 if isinstance(res, failure.Failure):
507 res.trap(error.Error)
508 self.failUnlessReallyEqual(res.value.status, "404")
510 self.fail("%s was supposed to Error(404), not get '%s'" %
513 def should302(self, res, which):
514 if isinstance(res, failure.Failure):
515 res.trap(error.Error)
516 self.failUnlessReallyEqual(res.value.status, "302")
518 self.fail("%s was supposed to Error(302), not get '%s'" %
522 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
523 def test_create(self):
526 def test_welcome(self):
529 self.failUnlessIn('Welcome to Tahoe-LAFS', res)
530 self.failUnlessIn(FAVICON_MARKUP, res)
531 self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
533 self.s.basedir = 'web/test_welcome'
534 fileutil.make_dirs("web/test_welcome")
535 fileutil.make_dirs("web/test_welcome/private")
537 d.addCallback(_check)
540 def test_status(self):
541 h = self.s.get_history()
542 dl_num = h.list_all_download_statuses()[0].get_counter()
543 ul_num = h.list_all_upload_statuses()[0].get_counter()
544 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
545 pub_num = h.list_all_publish_statuses()[0].get_counter()
546 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
547 d = self.GET("/status", followRedirect=True)
549 self.failUnlessIn('Upload and Download Status', res)
550 self.failUnlessIn('"down-%d"' % dl_num, res)
551 self.failUnlessIn('"up-%d"' % ul_num, res)
552 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
553 self.failUnlessIn('"publish-%d"' % pub_num, res)
554 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
555 d.addCallback(_check)
556 d.addCallback(lambda res: self.GET("/status/?t=json"))
557 def _check_json(res):
558 data = simplejson.loads(res)
559 self.failUnless(isinstance(data, dict))
560 #active = data["active"]
561 # TODO: test more. We need a way to fake an active operation
563 d.addCallback(_check_json)
565 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
567 self.failUnlessIn("File Download Status", res)
568 d.addCallback(_check_dl)
569 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
570 def _check_dl_json(res):
571 data = simplejson.loads(res)
572 self.failUnless(isinstance(data, dict))
573 self.failUnlessIn("read", data)
574 self.failUnlessEqual(data["read"][0]["length"], 120)
575 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
576 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
577 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
578 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
579 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
580 # serverids[] keys are strings, since that's what JSON does, but
581 # we'd really like them to be ints
582 self.failUnlessEqual(data["serverids"]["0"], "phwr")
583 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
584 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
585 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
586 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
587 self.failUnlessIn("dyhb", data)
588 self.failUnlessIn("misc", data)
589 d.addCallback(_check_dl_json)
590 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
592 self.failUnlessIn("File Upload Status", res)
593 d.addCallback(_check_ul)
594 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
595 def _check_mapupdate(res):
596 self.failUnlessIn("Mutable File Servermap Update Status", res)
597 d.addCallback(_check_mapupdate)
598 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
599 def _check_publish(res):
600 self.failUnlessIn("Mutable File Publish Status", res)
601 d.addCallback(_check_publish)
602 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
603 def _check_retrieve(res):
604 self.failUnlessIn("Mutable File Retrieve Status", res)
605 d.addCallback(_check_retrieve)
609 def test_status_numbers(self):
610 drrm = status.DownloadResultsRendererMixin()
611 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
612 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
613 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
614 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
615 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
616 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
617 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
618 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
619 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
621 urrm = status.UploadResultsRendererMixin()
622 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
623 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
624 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
625 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
626 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
627 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
628 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
629 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
630 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
632 def test_GET_FILEURL(self):
633 d = self.GET(self.public_url + "/foo/bar.txt")
634 d.addCallback(self.failUnlessIsBarDotTxt)
637 def test_GET_FILEURL_range(self):
638 headers = {"range": "bytes=1-10"}
639 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
640 return_response=True)
641 def _got((res, status, headers)):
642 self.failUnlessReallyEqual(int(status), 206)
643 self.failUnless(headers.has_key("content-range"))
644 self.failUnlessReallyEqual(headers["content-range"][0],
645 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
646 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
650 def test_GET_FILEURL_partial_range(self):
651 headers = {"range": "bytes=5-"}
652 length = len(self.BAR_CONTENTS)
653 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
654 return_response=True)
655 def _got((res, status, headers)):
656 self.failUnlessReallyEqual(int(status), 206)
657 self.failUnless(headers.has_key("content-range"))
658 self.failUnlessReallyEqual(headers["content-range"][0],
659 "bytes 5-%d/%d" % (length-1, length))
660 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
664 def test_GET_FILEURL_partial_end_range(self):
665 headers = {"range": "bytes=-5"}
666 length = len(self.BAR_CONTENTS)
667 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
668 return_response=True)
669 def _got((res, status, headers)):
670 self.failUnlessReallyEqual(int(status), 206)
671 self.failUnless(headers.has_key("content-range"))
672 self.failUnlessReallyEqual(headers["content-range"][0],
673 "bytes %d-%d/%d" % (length-5, length-1, length))
674 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
678 def test_GET_FILEURL_partial_range_overrun(self):
679 headers = {"range": "bytes=100-200"}
680 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
681 "416 Requested Range not satisfiable",
682 "First beyond end of file",
683 self.GET, self.public_url + "/foo/bar.txt",
687 def test_HEAD_FILEURL_range(self):
688 headers = {"range": "bytes=1-10"}
689 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
690 return_response=True)
691 def _got((res, status, headers)):
692 self.failUnlessReallyEqual(res, "")
693 self.failUnlessReallyEqual(int(status), 206)
694 self.failUnless(headers.has_key("content-range"))
695 self.failUnlessReallyEqual(headers["content-range"][0],
696 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
700 def test_HEAD_FILEURL_partial_range(self):
701 headers = {"range": "bytes=5-"}
702 length = len(self.BAR_CONTENTS)
703 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
704 return_response=True)
705 def _got((res, status, headers)):
706 self.failUnlessReallyEqual(int(status), 206)
707 self.failUnless(headers.has_key("content-range"))
708 self.failUnlessReallyEqual(headers["content-range"][0],
709 "bytes 5-%d/%d" % (length-1, length))
713 def test_HEAD_FILEURL_partial_end_range(self):
714 headers = {"range": "bytes=-5"}
715 length = len(self.BAR_CONTENTS)
716 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
717 return_response=True)
718 def _got((res, status, headers)):
719 self.failUnlessReallyEqual(int(status), 206)
720 self.failUnless(headers.has_key("content-range"))
721 self.failUnlessReallyEqual(headers["content-range"][0],
722 "bytes %d-%d/%d" % (length-5, length-1, length))
726 def test_HEAD_FILEURL_partial_range_overrun(self):
727 headers = {"range": "bytes=100-200"}
728 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
729 "416 Requested Range not satisfiable",
731 self.HEAD, self.public_url + "/foo/bar.txt",
735 def test_GET_FILEURL_range_bad(self):
736 headers = {"range": "BOGUS=fizbop-quarnak"}
737 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
738 return_response=True)
739 def _got((res, status, headers)):
740 self.failUnlessReallyEqual(int(status), 200)
741 self.failUnless(not headers.has_key("content-range"))
742 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
746 def test_HEAD_FILEURL(self):
747 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
748 def _got((res, status, headers)):
749 self.failUnlessReallyEqual(res, "")
750 self.failUnlessReallyEqual(headers["content-length"][0],
751 str(len(self.BAR_CONTENTS)))
752 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
756 def test_GET_FILEURL_named(self):
757 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
758 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
759 d = self.GET(base + "/@@name=/blah.txt")
760 d.addCallback(self.failUnlessIsBarDotTxt)
761 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
762 d.addCallback(self.failUnlessIsBarDotTxt)
763 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
764 d.addCallback(self.failUnlessIsBarDotTxt)
765 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
766 d.addCallback(self.failUnlessIsBarDotTxt)
767 save_url = base + "?save=true&filename=blah.txt"
768 d.addCallback(lambda res: self.GET(save_url))
769 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
770 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
771 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
772 u_url = base + "?save=true&filename=" + u_fn_e
773 d.addCallback(lambda res: self.GET(u_url))
774 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
777 def test_PUT_FILEURL_named_bad(self):
778 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
779 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
781 "/file can only be used with GET or HEAD",
782 self.PUT, base + "/@@name=/blah.txt", "")
786 def test_GET_DIRURL_named_bad(self):
787 base = "/file/%s" % urllib.quote(self._foo_uri)
788 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
791 self.GET, base + "/@@name=/blah.txt")
794 def test_GET_slash_file_bad(self):
795 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
797 "/file must be followed by a file-cap and a name",
801 def test_GET_unhandled_URI_named(self):
802 contents, n, newuri = self.makefile(12)
803 verifier_cap = n.get_verify_cap().to_string()
804 base = "/file/%s" % urllib.quote(verifier_cap)
805 # client.create_node_from_uri() can't handle verify-caps
806 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
807 "400 Bad Request", "is not a file-cap",
811 def test_GET_unhandled_URI(self):
812 contents, n, newuri = self.makefile(12)
813 verifier_cap = n.get_verify_cap().to_string()
814 base = "/uri/%s" % urllib.quote(verifier_cap)
815 # client.create_node_from_uri() can't handle verify-caps
816 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
818 "GET unknown URI type: can only do t=info",
822 def test_GET_FILE_URI(self):
823 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
825 d.addCallback(self.failUnlessIsBarDotTxt)
828 def test_GET_FILE_URI_mdmf(self):
829 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
831 d.addCallback(self.failUnlessIsQuuxDotTxt)
834 def test_GET_FILE_URI_mdmf_extensions(self):
835 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
837 d.addCallback(self.failUnlessIsQuuxDotTxt)
840 def test_GET_FILE_URI_mdmf_readonly(self):
841 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
843 d.addCallback(self.failUnlessIsQuuxDotTxt)
846 def test_GET_FILE_URI_badchild(self):
847 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
848 errmsg = "Files have no children, certainly not named 'boguschild'"
849 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
850 "400 Bad Request", errmsg,
854 def test_PUT_FILE_URI_badchild(self):
855 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
856 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
857 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
858 "400 Bad Request", errmsg,
862 def test_PUT_FILE_URI_mdmf(self):
863 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
864 self._quux_new_contents = "new_contents"
866 d.addCallback(lambda res:
867 self.failUnlessIsQuuxDotTxt(res))
868 d.addCallback(lambda ignored:
869 self.PUT(base, self._quux_new_contents))
870 d.addCallback(lambda ignored:
872 d.addCallback(lambda res:
873 self.failUnlessReallyEqual(res, self._quux_new_contents))
876 def test_PUT_FILE_URI_mdmf_extensions(self):
877 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
878 self._quux_new_contents = "new_contents"
880 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
881 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
882 d.addCallback(lambda ignored: self.GET(base))
883 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
887 def test_PUT_FILE_URI_mdmf_readonly(self):
888 # We're not allowed to PUT things to a readonly cap.
889 base = "/uri/%s" % self._quux_txt_readonly_uri
891 d.addCallback(lambda res:
892 self.failUnlessIsQuuxDotTxt(res))
893 # What should we get here? We get a 500 error now; that's not right.
894 d.addCallback(lambda ignored:
895 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
896 "400 Bad Request", "read-only cap",
897 self.PUT, base, "new data"))
900 def test_PUT_FILE_URI_sdmf_readonly(self):
901 # We're not allowed to put things to a readonly cap.
902 base = "/uri/%s" % self._baz_txt_readonly_uri
904 d.addCallback(lambda res:
905 self.failUnlessIsBazDotTxt(res))
906 d.addCallback(lambda ignored:
907 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
908 "400 Bad Request", "read-only cap",
909 self.PUT, base, "new_data"))
912 def test_GET_etags(self):
914 def _check_etags(uri):
916 d2 = _get_etag(uri, 'json')
917 d = defer.DeferredList([d1, d2], consumeErrors=True)
919 # All deferred must succeed
920 self.failUnless(all([r[0] for r in results]))
921 # the etag for the t=json form should be just like the etag
922 # fo the default t='' form, but with a 'json' suffix
923 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
924 d.addCallback(_check)
927 def _get_etag(uri, t=''):
928 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
929 d = self.GET(targetbase, return_response=True, followRedirect=True)
930 def _just_the_etag(result):
931 data, response, headers = result
932 etag = headers['etag'][0]
933 if uri.startswith('URI:DIR'):
934 self.failUnless(etag.startswith('DIR:'), etag)
936 return d.addCallback(_just_the_etag)
938 # Check that etags work with immutable directories
939 (newkids, caps) = self._create_immutable_children()
940 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
941 simplejson.dumps(newkids))
942 def _stash_immdir_uri(uri):
943 self._immdir_uri = uri
945 d.addCallback(_stash_immdir_uri)
946 d.addCallback(_check_etags)
948 # Check that etags work with immutable files
949 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
951 # use the ETag on GET
952 def _check_match(ign):
953 uri = "/uri/%s" % self._bar_txt_uri
954 d = self.GET(uri, return_response=True)
956 d.addCallback(lambda (data, code, headers):
958 # do a GET that's supposed to match the ETag
959 d.addCallback(lambda etag:
960 self.GET(uri, return_response=True,
961 headers={"If-None-Match": etag}))
962 # make sure it short-circuited (304 instead of 200)
963 d.addCallback(lambda (data, code, headers):
964 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
966 d.addCallback(_check_match)
968 def _no_etag(uri, t):
969 target = "/uri/%s?t=%s" % (uri, t)
970 d = self.GET(target, return_response=True, followRedirect=True)
971 d.addCallback(lambda (data, code, headers):
972 self.failIf("etag" in headers, target))
974 def _yes_etag(uri, t):
975 target = "/uri/%s?t=%s" % (uri, t)
976 d = self.GET(target, return_response=True, followRedirect=True)
977 d.addCallback(lambda (data, code, headers):
978 self.failUnless("etag" in headers, target))
981 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
982 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
983 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
984 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
985 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
987 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
988 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
989 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
990 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
991 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
992 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
996 # TODO: version of this with a Unicode filename
997 def test_GET_FILEURL_save(self):
998 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
999 return_response=True)
1000 def _got((res, statuscode, headers)):
1001 content_disposition = headers["content-disposition"][0]
1002 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1003 self.failUnlessIsBarDotTxt(res)
1007 def test_GET_FILEURL_missing(self):
1008 d = self.GET(self.public_url + "/foo/missing")
1009 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1012 def test_GET_FILEURL_info_mdmf(self):
1013 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1015 self.failUnlessIn("mutable file (mdmf)", res)
1016 self.failUnlessIn(self._quux_txt_uri, res)
1017 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1021 def test_GET_FILEURL_info_mdmf_readonly(self):
1022 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1024 self.failUnlessIn("mutable file (mdmf)", res)
1025 self.failIfIn(self._quux_txt_uri, res)
1026 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1030 def test_GET_FILEURL_info_sdmf(self):
1031 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1033 self.failUnlessIn("mutable file (sdmf)", res)
1034 self.failUnlessIn(self._baz_txt_uri, res)
1038 def test_GET_FILEURL_info_mdmf_extensions(self):
1039 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1041 self.failUnlessIn("mutable file (mdmf)", res)
1042 self.failUnlessIn(self._quux_txt_uri, res)
1043 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1047 def test_PUT_overwrite_only_files(self):
1048 # create a directory, put a file in that directory.
1049 contents, n, filecap = self.makefile(8)
1050 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1051 d.addCallback(lambda res:
1052 self.PUT(self.public_url + "/foo/dir/file1.txt",
1053 self.NEWFILE_CONTENTS))
1054 # try to overwrite the file with replace=only-files
1055 # (this should work)
1056 d.addCallback(lambda res:
1057 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1059 d.addCallback(lambda res:
1060 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1061 "There was already a child by that name, and you asked me "
1062 "to not replace it",
1063 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1067 def test_PUT_NEWFILEURL(self):
1068 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1069 # TODO: we lose the response code, so we can't check this
1070 #self.failUnlessReallyEqual(responsecode, 201)
1071 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1072 d.addCallback(lambda res:
1073 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1074 self.NEWFILE_CONTENTS))
1077 def test_PUT_NEWFILEURL_not_mutable(self):
1078 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1079 self.NEWFILE_CONTENTS)
1080 # TODO: we lose the response code, so we can't check this
1081 #self.failUnlessReallyEqual(responsecode, 201)
1082 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1083 d.addCallback(lambda res:
1084 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1085 self.NEWFILE_CONTENTS))
1088 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1089 # this should get us a few segments of an MDMF mutable file,
1090 # which we can then test for.
1091 contents = self.NEWFILE_CONTENTS * 300000
1092 d = self.PUT("/uri?format=mdmf",
1094 def _got_filecap(filecap):
1095 self.failUnless(filecap.startswith("URI:MDMF"))
1097 d.addCallback(_got_filecap)
1098 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1099 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1102 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1103 contents = self.NEWFILE_CONTENTS * 300000
1104 d = self.PUT("/uri?format=sdmf",
1106 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1107 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1110 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1111 contents = self.NEWFILE_CONTENTS * 300000
1112 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1113 400, "Bad Request", "Unknown format: foo",
1114 self.PUT, "/uri?format=foo",
1117 def test_PUT_NEWFILEURL_range_bad(self):
1118 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1119 target = self.public_url + "/foo/new.txt"
1120 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1121 "501 Not Implemented",
1122 "Content-Range in PUT not yet supported",
1123 # (and certainly not for immutable files)
1124 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1126 d.addCallback(lambda res:
1127 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1130 def test_PUT_NEWFILEURL_mutable(self):
1131 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1132 self.NEWFILE_CONTENTS)
1133 # TODO: we lose the response code, so we can't check this
1134 #self.failUnlessReallyEqual(responsecode, 201)
1135 def _check_uri(res):
1136 u = uri.from_string_mutable_filenode(res)
1137 self.failUnless(u.is_mutable())
1138 self.failIf(u.is_readonly())
1140 d.addCallback(_check_uri)
1141 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1142 d.addCallback(lambda res:
1143 self.failUnlessMutableChildContentsAre(self._foo_node,
1145 self.NEWFILE_CONTENTS))
1148 def test_PUT_NEWFILEURL_mutable_toobig(self):
1149 # It is okay to upload large mutable files, so we should be able
1151 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1152 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1155 def test_PUT_NEWFILEURL_replace(self):
1156 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1157 # TODO: we lose the response code, so we can't check this
1158 #self.failUnlessReallyEqual(responsecode, 200)
1159 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1160 d.addCallback(lambda res:
1161 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1162 self.NEWFILE_CONTENTS))
1165 def test_PUT_NEWFILEURL_bad_t(self):
1166 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1167 "PUT to a file: bad t=bogus",
1168 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1172 def test_PUT_NEWFILEURL_no_replace(self):
1173 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1174 self.NEWFILE_CONTENTS)
1175 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1177 "There was already a child by that name, and you asked me "
1178 "to not replace it")
1181 def test_PUT_NEWFILEURL_mkdirs(self):
1182 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1184 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1185 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1186 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1187 d.addCallback(lambda res:
1188 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1189 self.NEWFILE_CONTENTS))
1192 def test_PUT_NEWFILEURL_blocked(self):
1193 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1194 self.NEWFILE_CONTENTS)
1195 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1197 "Unable to create directory 'blockingfile': a file was in the way")
1200 def test_PUT_NEWFILEURL_emptyname(self):
1201 # an empty pathname component (i.e. a double-slash) is disallowed
1202 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1204 "The webapi does not allow empty pathname components",
1205 self.PUT, self.public_url + "/foo//new.txt", "")
1208 def test_DELETE_FILEURL(self):
1209 d = self.DELETE(self.public_url + "/foo/bar.txt")
1210 d.addCallback(lambda res:
1211 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1214 def test_DELETE_FILEURL_missing(self):
1215 d = self.DELETE(self.public_url + "/foo/missing")
1216 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1219 def test_DELETE_FILEURL_missing2(self):
1220 d = self.DELETE(self.public_url + "/missing/missing")
1221 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1224 def failUnlessHasBarDotTxtMetadata(self, res):
1225 data = simplejson.loads(res)
1226 self.failUnless(isinstance(data, list))
1227 self.failUnlessIn("metadata", data[1])
1228 self.failUnlessIn("tahoe", data[1]["metadata"])
1229 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1230 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1231 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1232 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1234 def test_GET_FILEURL_json(self):
1235 # twisted.web.http.parse_qs ignores any query args without an '=', so
1236 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1237 # instead. This may make it tricky to emulate the S3 interface
1239 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1241 self.failUnlessIsBarJSON(data)
1242 self.failUnlessHasBarDotTxtMetadata(data)
1244 d.addCallback(_check1)
1247 def test_GET_FILEURL_json_mutable_type(self):
1248 # The JSON should include format, which says whether the
1249 # file is SDMF or MDMF
1250 d = self.PUT("/uri?format=mdmf",
1251 self.NEWFILE_CONTENTS * 300000)
1252 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1253 def _got_json(json, version):
1254 data = simplejson.loads(json)
1255 assert "filenode" == data[0]
1257 assert isinstance(data, dict)
1259 self.failUnlessIn("format", data)
1260 self.failUnlessEqual(data["format"], version)
1262 d.addCallback(_got_json, "MDMF")
1263 # Now make an SDMF file and check that it is reported correctly.
1264 d.addCallback(lambda ignored:
1265 self.PUT("/uri?format=sdmf",
1266 self.NEWFILE_CONTENTS * 300000))
1267 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1268 d.addCallback(_got_json, "SDMF")
1271 def test_GET_FILEURL_json_mdmf(self):
1272 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1273 d.addCallback(self.failUnlessIsQuuxJSON)
1276 def test_GET_FILEURL_json_missing(self):
1277 d = self.GET(self.public_url + "/foo/missing?json")
1278 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1281 def test_GET_FILEURL_uri(self):
1282 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1284 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1285 d.addCallback(_check)
1286 d.addCallback(lambda res:
1287 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1289 # for now, for files, uris and readonly-uris are the same
1290 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1291 d.addCallback(_check2)
1294 def test_GET_FILEURL_badtype(self):
1295 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1298 self.public_url + "/foo/bar.txt?t=bogus")
1301 def test_CSS_FILE(self):
1302 d = self.GET("/tahoe.css", followRedirect=True)
1304 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1305 self.failUnless(CSS_STYLE.search(res), res)
1306 d.addCallback(_check)
1309 def test_GET_FILEURL_uri_missing(self):
1310 d = self.GET(self.public_url + "/foo/missing?t=uri")
1311 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1314 def _check_upload_and_mkdir_forms(self, html):
1315 # We should have a form to create a file, with radio buttons that allow
1316 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1317 self.failUnlessIn('name="t" value="upload"', html)
1318 self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1319 self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1320 self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1322 # We should also have the ability to create a mutable directory, with
1323 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1324 # or MDMF directory.
1325 self.failUnlessIn('name="t" value="mkdir"', html)
1326 self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1327 self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1329 self.failUnlessIn(FAVICON_MARKUP, html)
1331 def test_GET_DIRECTORY_html(self):
1332 d = self.GET(self.public_url + "/foo", followRedirect=True)
1334 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1335 self._check_upload_and_mkdir_forms(html)
1336 self.failUnlessIn("quux", html)
1337 d.addCallback(_check)
1340 def test_GET_root_html(self):
1342 d.addCallback(self._check_upload_and_mkdir_forms)
1345 def test_GET_DIRURL(self):
1346 # the addSlash means we get a redirect here
1347 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1349 d = self.GET(self.public_url + "/foo", followRedirect=True)
1351 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1353 # the FILE reference points to a URI, but it should end in bar.txt
1354 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1355 (ROOT, urllib.quote(self._bar_txt_uri)))
1356 get_bar = "".join([r'<td>FILE</td>',
1358 r'<a href="%s">bar.txt</a>' % bar_url,
1360 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1362 self.failUnless(re.search(get_bar, res), res)
1363 for label in ['unlink', 'rename/move']:
1364 for line in res.split("\n"):
1365 # find the line that contains the relevant button for bar.txt
1366 if ("form action" in line and
1367 ('value="%s"' % (label,)) in line and
1368 'value="bar.txt"' in line):
1369 # the form target should use a relative URL
1370 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1371 self.failUnlessIn('action="%s"' % foo_url, line)
1372 # and the when_done= should too
1373 #done_url = urllib.quote(???)
1374 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1376 # 'unlink' needs to use POST because it directly has a side effect
1377 if label == 'unlink':
1378 self.failUnlessIn('method="post"', line)
1381 self.fail("unable to find '%s bar.txt' line" % (label,))
1383 # the DIR reference just points to a URI
1384 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1385 get_sub = ((r'<td>DIR</td>')
1386 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1387 self.failUnless(re.search(get_sub, res), res)
1388 d.addCallback(_check)
1390 # look at a readonly directory
1391 d.addCallback(lambda res:
1392 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1394 self.failUnlessIn("(read-only)", res)
1395 self.failIfIn("Upload a file", res)
1396 d.addCallback(_check2)
1398 # and at a directory that contains a readonly directory
1399 d.addCallback(lambda res:
1400 self.GET(self.public_url, followRedirect=True))
1402 self.failUnless(re.search('<td>DIR-RO</td>'
1403 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1404 d.addCallback(_check3)
1406 # and an empty directory
1407 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1409 self.failUnlessIn("directory is empty", res)
1410 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1411 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1412 d.addCallback(_check4)
1414 # and at a literal directory
1415 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1416 d.addCallback(lambda res:
1417 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1419 self.failUnlessIn('(immutable)', res)
1420 self.failUnless(re.search('<td>FILE</td>'
1421 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1422 d.addCallback(_check5)
1425 def test_GET_DIRURL_badtype(self):
1426 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1430 self.public_url + "/foo?t=bogus")
1433 def test_GET_DIRURL_json(self):
1434 d = self.GET(self.public_url + "/foo?t=json")
1435 d.addCallback(self.failUnlessIsFooJSON)
1438 def test_GET_DIRURL_json_format(self):
1439 d = self.PUT(self.public_url + \
1440 "/foo/sdmf.txt?format=sdmf",
1441 self.NEWFILE_CONTENTS * 300000)
1442 d.addCallback(lambda ignored:
1443 self.PUT(self.public_url + \
1444 "/foo/mdmf.txt?format=mdmf",
1445 self.NEWFILE_CONTENTS * 300000))
1446 # Now we have an MDMF and SDMF file in the directory. If we GET
1447 # its JSON, we should see their encodings.
1448 d.addCallback(lambda ignored:
1449 self.GET(self.public_url + "/foo?t=json"))
1450 def _got_json(json):
1451 data = simplejson.loads(json)
1452 assert data[0] == "dirnode"
1455 kids = data['children']
1457 mdmf_data = kids['mdmf.txt'][1]
1458 self.failUnlessIn("format", mdmf_data)
1459 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1461 sdmf_data = kids['sdmf.txt'][1]
1462 self.failUnlessIn("format", sdmf_data)
1463 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1464 d.addCallback(_got_json)
1468 def test_POST_DIRURL_manifest_no_ophandle(self):
1469 d = self.shouldFail2(error.Error,
1470 "test_POST_DIRURL_manifest_no_ophandle",
1472 "slow operation requires ophandle=",
1473 self.POST, self.public_url, t="start-manifest")
1476 def test_POST_DIRURL_manifest(self):
1477 d = defer.succeed(None)
1478 def getman(ignored, output):
1479 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1480 followRedirect=True)
1481 d.addCallback(self.wait_for_operation, "125")
1482 d.addCallback(self.get_operation_results, "125", output)
1484 d.addCallback(getman, None)
1485 def _got_html(manifest):
1486 self.failUnlessIn("Manifest of SI=", manifest)
1487 self.failUnlessIn("<td>sub</td>", manifest)
1488 self.failUnlessIn(self._sub_uri, manifest)
1489 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1490 self.failUnlessIn(FAVICON_MARKUP, manifest)
1491 d.addCallback(_got_html)
1493 # both t=status and unadorned GET should be identical
1494 d.addCallback(lambda res: self.GET("/operations/125"))
1495 d.addCallback(_got_html)
1497 d.addCallback(getman, "html")
1498 d.addCallback(_got_html)
1499 d.addCallback(getman, "text")
1500 def _got_text(manifest):
1501 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1502 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1503 d.addCallback(_got_text)
1504 d.addCallback(getman, "JSON")
1506 data = res["manifest"]
1508 for (path_list, cap) in data:
1509 got[tuple(path_list)] = cap
1510 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1511 self.failUnlessIn((u"sub", u"baz.txt"), got)
1512 self.failUnlessIn("finished", res)
1513 self.failUnlessIn("origin", res)
1514 self.failUnlessIn("storage-index", res)
1515 self.failUnlessIn("verifycaps", res)
1516 self.failUnlessIn("stats", res)
1517 d.addCallback(_got_json)
1520 def test_POST_DIRURL_deepsize_no_ophandle(self):
1521 d = self.shouldFail2(error.Error,
1522 "test_POST_DIRURL_deepsize_no_ophandle",
1524 "slow operation requires ophandle=",
1525 self.POST, self.public_url, t="start-deep-size")
1528 def test_POST_DIRURL_deepsize(self):
1529 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1530 followRedirect=True)
1531 d.addCallback(self.wait_for_operation, "126")
1532 d.addCallback(self.get_operation_results, "126", "json")
1533 def _got_json(data):
1534 self.failUnlessReallyEqual(data["finished"], True)
1536 self.failUnless(size > 1000)
1537 d.addCallback(_got_json)
1538 d.addCallback(self.get_operation_results, "126", "text")
1540 mo = re.search(r'^size: (\d+)$', res, re.M)
1541 self.failUnless(mo, res)
1542 size = int(mo.group(1))
1543 # with directories, the size varies.
1544 self.failUnless(size > 1000)
1545 d.addCallback(_got_text)
1548 def test_POST_DIRURL_deepstats_no_ophandle(self):
1549 d = self.shouldFail2(error.Error,
1550 "test_POST_DIRURL_deepstats_no_ophandle",
1552 "slow operation requires ophandle=",
1553 self.POST, self.public_url, t="start-deep-stats")
1556 def test_POST_DIRURL_deepstats(self):
1557 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1558 followRedirect=True)
1559 d.addCallback(self.wait_for_operation, "127")
1560 d.addCallback(self.get_operation_results, "127", "json")
1561 def _got_json(stats):
1562 expected = {"count-immutable-files": 3,
1563 "count-mutable-files": 2,
1564 "count-literal-files": 0,
1566 "count-directories": 3,
1567 "size-immutable-files": 57,
1568 "size-literal-files": 0,
1569 #"size-directories": 1912, # varies
1570 #"largest-directory": 1590,
1571 "largest-directory-children": 7,
1572 "largest-immutable-file": 19,
1574 for k,v in expected.iteritems():
1575 self.failUnlessReallyEqual(stats[k], v,
1576 "stats[%s] was %s, not %s" %
1578 self.failUnlessReallyEqual(stats["size-files-histogram"],
1580 d.addCallback(_got_json)
1583 def test_POST_DIRURL_stream_manifest(self):
1584 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1586 self.failUnless(res.endswith("\n"))
1587 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1588 self.failUnlessReallyEqual(len(units), 9)
1589 self.failUnlessEqual(units[-1]["type"], "stats")
1591 self.failUnlessEqual(first["path"], [])
1592 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1593 self.failUnlessEqual(first["type"], "directory")
1594 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1595 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1596 self.failIfEqual(baz["storage-index"], None)
1597 self.failIfEqual(baz["verifycap"], None)
1598 self.failIfEqual(baz["repaircap"], None)
1599 # XXX: Add quux and baz to this test.
1601 d.addCallback(_check)
1604 def test_GET_DIRURL_uri(self):
1605 d = self.GET(self.public_url + "/foo?t=uri")
1607 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1608 d.addCallback(_check)
1611 def test_GET_DIRURL_readonly_uri(self):
1612 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1614 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1615 d.addCallback(_check)
1618 def test_PUT_NEWDIRURL(self):
1619 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1620 d.addCallback(lambda res:
1621 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1622 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1623 d.addCallback(self.failUnlessNodeKeysAre, [])
1626 def test_PUT_NEWDIRURL_mdmf(self):
1627 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1628 d.addCallback(lambda res:
1629 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1630 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1631 d.addCallback(lambda node:
1632 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1635 def test_PUT_NEWDIRURL_sdmf(self):
1636 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1638 d.addCallback(lambda res:
1639 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1640 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1641 d.addCallback(lambda node:
1642 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1645 def test_PUT_NEWDIRURL_bad_format(self):
1646 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1647 400, "Bad Request", "Unknown format: foo",
1648 self.PUT, self.public_url +
1649 "/foo/newdir=?t=mkdir&format=foo", "")
1651 def test_POST_NEWDIRURL(self):
1652 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1653 d.addCallback(lambda res:
1654 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1655 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1656 d.addCallback(self.failUnlessNodeKeysAre, [])
1659 def test_POST_NEWDIRURL_mdmf(self):
1660 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1661 d.addCallback(lambda res:
1662 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1663 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1664 d.addCallback(lambda node:
1665 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1668 def test_POST_NEWDIRURL_sdmf(self):
1669 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1670 d.addCallback(lambda res:
1671 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1672 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1673 d.addCallback(lambda node:
1674 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1677 def test_POST_NEWDIRURL_bad_format(self):
1678 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1679 400, "Bad Request", "Unknown format: foo",
1680 self.POST2, self.public_url + \
1681 "/foo/newdir?t=mkdir&format=foo", "")
1683 def test_POST_NEWDIRURL_emptyname(self):
1684 # an empty pathname component (i.e. a double-slash) is disallowed
1685 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1687 "The webapi does not allow empty pathname components, i.e. a double slash",
1688 self.POST, self.public_url + "//?t=mkdir")
1691 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1692 (newkids, caps) = self._create_initial_children()
1693 query = "/foo/newdir?t=mkdir-with-children"
1694 if version == MDMF_VERSION:
1695 query += "&format=mdmf"
1696 elif version == SDMF_VERSION:
1697 query += "&format=sdmf"
1699 version = SDMF_VERSION # for later
1700 d = self.POST2(self.public_url + query,
1701 simplejson.dumps(newkids))
1703 n = self.s.create_node_from_uri(uri.strip())
1704 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1705 self.failUnlessEqual(n._node.get_version(), version)
1706 d2.addCallback(lambda ign:
1707 self.failUnlessROChildURIIs(n, u"child-imm",
1709 d2.addCallback(lambda ign:
1710 self.failUnlessRWChildURIIs(n, u"child-mutable",
1712 d2.addCallback(lambda ign:
1713 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1715 d2.addCallback(lambda ign:
1716 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1717 caps['unknown_rocap']))
1718 d2.addCallback(lambda ign:
1719 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1720 caps['unknown_rwcap']))
1721 d2.addCallback(lambda ign:
1722 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1723 caps['unknown_immcap']))
1724 d2.addCallback(lambda ign:
1725 self.failUnlessRWChildURIIs(n, u"dirchild",
1727 d2.addCallback(lambda ign:
1728 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1730 d2.addCallback(lambda ign:
1731 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1732 caps['emptydircap']))
1734 d.addCallback(_check)
1735 d.addCallback(lambda res:
1736 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1737 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1738 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1739 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1740 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1743 def test_POST_NEWDIRURL_initial_children(self):
1744 return self._do_POST_NEWDIRURL_initial_children_test()
1746 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1747 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1749 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1750 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1752 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1753 (newkids, caps) = self._create_initial_children()
1754 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1755 400, "Bad Request", "Unknown format: foo",
1756 self.POST2, self.public_url + \
1757 "/foo/newdir?t=mkdir-with-children&format=foo",
1758 simplejson.dumps(newkids))
1760 def test_POST_NEWDIRURL_immutable(self):
1761 (newkids, caps) = self._create_immutable_children()
1762 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1763 simplejson.dumps(newkids))
1765 n = self.s.create_node_from_uri(uri.strip())
1766 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1767 d2.addCallback(lambda ign:
1768 self.failUnlessROChildURIIs(n, u"child-imm",
1770 d2.addCallback(lambda ign:
1771 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1772 caps['unknown_immcap']))
1773 d2.addCallback(lambda ign:
1774 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1776 d2.addCallback(lambda ign:
1777 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1779 d2.addCallback(lambda ign:
1780 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1781 caps['emptydircap']))
1783 d.addCallback(_check)
1784 d.addCallback(lambda res:
1785 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1786 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1787 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1788 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1789 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1790 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1791 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1792 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1793 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1794 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1795 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1796 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1797 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1798 d.addErrback(self.explain_web_error)
1801 def test_POST_NEWDIRURL_immutable_bad(self):
1802 (newkids, caps) = self._create_initial_children()
1803 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1805 "needed to be immutable but was not",
1807 self.public_url + "/foo/newdir?t=mkdir-immutable",
1808 simplejson.dumps(newkids))
1811 def test_PUT_NEWDIRURL_exists(self):
1812 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1813 d.addCallback(lambda res:
1814 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1815 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1816 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1819 def test_PUT_NEWDIRURL_blocked(self):
1820 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1821 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1823 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1824 d.addCallback(lambda res:
1825 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1826 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1827 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1830 def test_PUT_NEWDIRURL_mkdirs(self):
1831 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1832 d.addCallback(lambda res:
1833 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1834 d.addCallback(lambda res:
1835 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1836 d.addCallback(lambda res:
1837 self._foo_node.get_child_at_path(u"subdir/newdir"))
1838 d.addCallback(self.failUnlessNodeKeysAre, [])
1841 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1842 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1843 d.addCallback(lambda ignored:
1844 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1845 d.addCallback(lambda ignored:
1846 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1847 d.addCallback(lambda ignored:
1848 self._foo_node.get_child_at_path(u"subdir"))
1849 def _got_subdir(subdir):
1850 # XXX: What we want?
1851 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1852 self.failUnlessNodeHasChild(subdir, u"newdir")
1853 return subdir.get_child_at_path(u"newdir")
1854 d.addCallback(_got_subdir)
1855 d.addCallback(lambda newdir:
1856 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1859 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1860 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1861 d.addCallback(lambda ignored:
1862 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1863 d.addCallback(lambda ignored:
1864 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1865 d.addCallback(lambda ignored:
1866 self._foo_node.get_child_at_path(u"subdir"))
1867 def _got_subdir(subdir):
1868 # XXX: What we want?
1869 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1870 self.failUnlessNodeHasChild(subdir, u"newdir")
1871 return subdir.get_child_at_path(u"newdir")
1872 d.addCallback(_got_subdir)
1873 d.addCallback(lambda newdir:
1874 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1877 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1878 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1879 400, "Bad Request", "Unknown format: foo",
1880 self.PUT, self.public_url + \
1881 "/foo/subdir/newdir?t=mkdir&format=foo",
1884 def test_DELETE_DIRURL(self):
1885 d = self.DELETE(self.public_url + "/foo")
1886 d.addCallback(lambda res:
1887 self.failIfNodeHasChild(self.public_root, u"foo"))
1890 def test_DELETE_DIRURL_missing(self):
1891 d = self.DELETE(self.public_url + "/foo/missing")
1892 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1893 d.addCallback(lambda res:
1894 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1897 def test_DELETE_DIRURL_missing2(self):
1898 d = self.DELETE(self.public_url + "/missing")
1899 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1902 def dump_root(self):
1904 w = webish.DirnodeWalkerMixin()
1905 def visitor(childpath, childnode, metadata):
1907 d = w.walk(self.public_root, visitor)
1910 def failUnlessNodeKeysAre(self, node, expected_keys):
1911 for k in expected_keys:
1912 assert isinstance(k, unicode)
1914 def _check(children):
1915 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1916 d.addCallback(_check)
1918 def failUnlessNodeHasChild(self, node, name):
1919 assert isinstance(name, unicode)
1921 def _check(children):
1922 self.failUnlessIn(name, children)
1923 d.addCallback(_check)
1925 def failIfNodeHasChild(self, node, name):
1926 assert isinstance(name, unicode)
1928 def _check(children):
1929 self.failIfIn(name, children)
1930 d.addCallback(_check)
1933 def failUnlessChildContentsAre(self, node, name, expected_contents):
1934 assert isinstance(name, unicode)
1935 d = node.get_child_at_path(name)
1936 d.addCallback(lambda node: download_to_data(node))
1937 def _check(contents):
1938 self.failUnlessReallyEqual(contents, expected_contents)
1939 d.addCallback(_check)
1942 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1943 assert isinstance(name, unicode)
1944 d = node.get_child_at_path(name)
1945 d.addCallback(lambda node: node.download_best_version())
1946 def _check(contents):
1947 self.failUnlessReallyEqual(contents, expected_contents)
1948 d.addCallback(_check)
1951 def failUnlessRWChildURIIs(self, node, name, expected_uri):
1952 assert isinstance(name, unicode)
1953 d = node.get_child_at_path(name)
1955 self.failUnless(child.is_unknown() or not child.is_readonly())
1956 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1957 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1958 expected_ro_uri = self._make_readonly(expected_uri)
1960 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1961 d.addCallback(_check)
1964 def failUnlessROChildURIIs(self, node, name, expected_uri):
1965 assert isinstance(name, unicode)
1966 d = node.get_child_at_path(name)
1968 self.failUnless(child.is_unknown() or child.is_readonly())
1969 self.failUnlessReallyEqual(child.get_write_uri(), None)
1970 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1971 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1972 d.addCallback(_check)
1975 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1976 assert isinstance(name, unicode)
1977 d = node.get_child_at_path(name)
1979 self.failUnless(child.is_unknown() or not child.is_readonly())
1980 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1981 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1982 expected_ro_uri = self._make_readonly(got_uri)
1984 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1985 d.addCallback(_check)
1988 def failUnlessURIMatchesROChild(self, got_uri, node, name):
1989 assert isinstance(name, unicode)
1990 d = node.get_child_at_path(name)
1992 self.failUnless(child.is_unknown() or child.is_readonly())
1993 self.failUnlessReallyEqual(child.get_write_uri(), None)
1994 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1995 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1996 d.addCallback(_check)
1999 def failUnlessCHKURIHasContents(self, got_uri, contents):
2000 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2002 def test_POST_upload(self):
2003 d = self.POST(self.public_url + "/foo", t="upload",
2004 file=("new.txt", self.NEWFILE_CONTENTS))
2006 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2007 d.addCallback(lambda res:
2008 self.failUnlessChildContentsAre(fn, u"new.txt",
2009 self.NEWFILE_CONTENTS))
2012 def test_POST_upload_unicode(self):
2013 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2014 d = self.POST(self.public_url + "/foo", t="upload",
2015 file=(filename, self.NEWFILE_CONTENTS))
2017 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2018 d.addCallback(lambda res:
2019 self.failUnlessChildContentsAre(fn, filename,
2020 self.NEWFILE_CONTENTS))
2021 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2022 d.addCallback(lambda res: self.GET(target_url))
2023 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2024 self.NEWFILE_CONTENTS,
2028 def test_POST_upload_unicode_named(self):
2029 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2030 d = self.POST(self.public_url + "/foo", t="upload",
2032 file=("overridden", self.NEWFILE_CONTENTS))
2034 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2035 d.addCallback(lambda res:
2036 self.failUnlessChildContentsAre(fn, filename,
2037 self.NEWFILE_CONTENTS))
2038 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2039 d.addCallback(lambda res: self.GET(target_url))
2040 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2041 self.NEWFILE_CONTENTS,
2045 def test_POST_upload_no_link(self):
2046 d = self.POST("/uri", t="upload",
2047 file=("new.txt", self.NEWFILE_CONTENTS))
2048 def _check_upload_results(page):
2049 # this should be a page which describes the results of the upload
2050 # that just finished.
2051 self.failUnlessIn("Upload Results:", page)
2052 self.failUnlessIn("URI:", page)
2053 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2054 mo = uri_re.search(page)
2055 self.failUnless(mo, page)
2056 new_uri = mo.group(1)
2058 d.addCallback(_check_upload_results)
2059 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2062 def test_POST_upload_no_link_whendone(self):
2063 d = self.POST("/uri", t="upload", when_done="/",
2064 file=("new.txt", self.NEWFILE_CONTENTS))
2065 d.addBoth(self.shouldRedirect, "/")
2068 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2069 d = defer.maybeDeferred(callable, *args, **kwargs)
2071 if isinstance(res, failure.Failure):
2072 res.trap(error.PageRedirect)
2073 statuscode = res.value.status
2074 target = res.value.location
2075 return checker(statuscode, target)
2076 self.fail("%s: callable was supposed to redirect, not return '%s'"
2081 def test_POST_upload_no_link_whendone_results(self):
2082 def check(statuscode, target):
2083 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2084 self.failUnless(target.startswith(self.webish_url), target)
2085 return client.getPage(target, method="GET")
2086 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2088 self.POST, "/uri", t="upload",
2089 when_done="/uri/%(uri)s",
2090 file=("new.txt", self.NEWFILE_CONTENTS))
2091 d.addCallback(lambda res:
2092 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2095 def test_POST_upload_no_link_mutable(self):
2096 d = self.POST("/uri", t="upload", mutable="true",
2097 file=("new.txt", self.NEWFILE_CONTENTS))
2098 def _check(filecap):
2099 filecap = filecap.strip()
2100 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2101 self.filecap = filecap
2102 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2103 self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2104 n = self.s.create_node_from_uri(filecap)
2105 return n.download_best_version()
2106 d.addCallback(_check)
2108 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2109 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2110 d.addCallback(_check2)
2112 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2113 return self.GET("/file/%s" % urllib.quote(self.filecap))
2114 d.addCallback(_check3)
2116 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2117 d.addCallback(_check4)
2120 def test_POST_upload_no_link_mutable_toobig(self):
2121 # The SDMF size limit is no longer in place, so we should be
2122 # able to upload mutable files that are as large as we want them
2124 d = self.POST("/uri", t="upload", mutable="true",
2125 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2129 def test_POST_upload_format_unlinked(self):
2130 def _check_upload_unlinked(ign, format, uri_prefix):
2131 filename = format + ".txt"
2132 d = self.POST("/uri?t=upload&format=" + format,
2133 file=(filename, self.NEWFILE_CONTENTS * 300000))
2134 def _got_results(results):
2135 if format.upper() in ("SDMF", "MDMF"):
2136 # webapi.rst says this returns a filecap
2139 # for immutable, it returns an "upload results page", and
2140 # the filecap is buried inside
2141 line = [l for l in results.split("\n") if "URI: " in l][0]
2142 mo = re.search(r'<span>([^<]+)</span>', line)
2143 filecap = mo.group(1)
2144 self.failUnless(filecap.startswith(uri_prefix),
2145 (uri_prefix, filecap))
2146 return self.GET("/uri/%s?t=json" % filecap)
2147 d.addCallback(_got_results)
2148 def _got_json(json):
2149 data = simplejson.loads(json)
2151 self.failUnlessIn("format", data)
2152 self.failUnlessEqual(data["format"], format.upper())
2153 d.addCallback(_got_json)
2155 d = defer.succeed(None)
2156 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2157 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2158 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2159 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2162 def test_POST_upload_bad_format_unlinked(self):
2163 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2164 400, "Bad Request", "Unknown format: foo",
2166 "/uri?t=upload&format=foo",
2167 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2169 def test_POST_upload_format(self):
2170 def _check_upload(ign, format, uri_prefix, fn=None):
2171 filename = format + ".txt"
2172 d = self.POST(self.public_url +
2173 "/foo?t=upload&format=" + format,
2174 file=(filename, self.NEWFILE_CONTENTS * 300000))
2175 def _got_filecap(filecap):
2177 filenameu = unicode(filename)
2178 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2179 self.failUnless(filecap.startswith(uri_prefix))
2180 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2181 d.addCallback(_got_filecap)
2182 def _got_json(json):
2183 data = simplejson.loads(json)
2185 self.failUnlessIn("format", data)
2186 self.failUnlessEqual(data["format"], format.upper())
2187 d.addCallback(_got_json)
2190 d = defer.succeed(None)
2191 d.addCallback(_check_upload, "chk", "URI:CHK")
2192 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2193 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2194 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2197 def test_POST_upload_bad_format(self):
2198 return self.shouldHTTPError("POST_upload_bad_format",
2199 400, "Bad Request", "Unknown format: foo",
2200 self.POST, self.public_url + \
2201 "/foo?t=upload&format=foo",
2202 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2204 def test_POST_upload_mutable(self):
2205 # this creates a mutable file
2206 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2207 file=("new.txt", self.NEWFILE_CONTENTS))
2209 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2210 d.addCallback(lambda res:
2211 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2212 self.NEWFILE_CONTENTS))
2213 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2215 self.failUnless(IMutableFileNode.providedBy(newnode))
2216 self.failUnless(newnode.is_mutable())
2217 self.failIf(newnode.is_readonly())
2218 self._mutable_node = newnode
2219 self._mutable_uri = newnode.get_uri()
2222 # now upload it again and make sure that the URI doesn't change
2223 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2224 d.addCallback(lambda res:
2225 self.POST(self.public_url + "/foo", t="upload",
2227 file=("new.txt", NEWER_CONTENTS)))
2228 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2229 d.addCallback(lambda res:
2230 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2232 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2234 self.failUnless(IMutableFileNode.providedBy(newnode))
2235 self.failUnless(newnode.is_mutable())
2236 self.failIf(newnode.is_readonly())
2237 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2238 d.addCallback(_got2)
2240 # upload a second time, using PUT instead of POST
2241 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2242 d.addCallback(lambda res:
2243 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2244 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2245 d.addCallback(lambda res:
2246 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2249 # finally list the directory, since mutable files are displayed
2250 # slightly differently
2252 d.addCallback(lambda res:
2253 self.GET(self.public_url + "/foo/",
2254 followRedirect=True))
2255 def _check_page(res):
2256 # TODO: assert more about the contents
2257 self.failUnlessIn("SSK", res)
2259 d.addCallback(_check_page)
2261 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2263 self.failUnless(IMutableFileNode.providedBy(newnode))
2264 self.failUnless(newnode.is_mutable())
2265 self.failIf(newnode.is_readonly())
2266 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2267 d.addCallback(_got3)
2269 # look at the JSON form of the enclosing directory
2270 d.addCallback(lambda res:
2271 self.GET(self.public_url + "/foo/?t=json",
2272 followRedirect=True))
2273 def _check_page_json(res):
2274 parsed = simplejson.loads(res)
2275 self.failUnlessEqual(parsed[0], "dirnode")
2276 children = dict( [(unicode(name),value)
2278 in parsed[1]["children"].iteritems()] )
2279 self.failUnlessIn(u"new.txt", children)
2280 new_json = children[u"new.txt"]
2281 self.failUnlessEqual(new_json[0], "filenode")
2282 self.failUnless(new_json[1]["mutable"])
2283 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2284 ro_uri = self._mutable_node.get_readonly().to_string()
2285 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2286 d.addCallback(_check_page_json)
2288 # and the JSON form of the file
2289 d.addCallback(lambda res:
2290 self.GET(self.public_url + "/foo/new.txt?t=json"))
2291 def _check_file_json(res):
2292 parsed = simplejson.loads(res)
2293 self.failUnlessEqual(parsed[0], "filenode")
2294 self.failUnless(parsed[1]["mutable"])
2295 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2296 ro_uri = self._mutable_node.get_readonly().to_string()
2297 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2298 d.addCallback(_check_file_json)
2300 # and look at t=uri and t=readonly-uri
2301 d.addCallback(lambda res:
2302 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2303 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2304 d.addCallback(lambda res:
2305 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2306 def _check_ro_uri(res):
2307 ro_uri = self._mutable_node.get_readonly().to_string()
2308 self.failUnlessReallyEqual(res, ro_uri)
2309 d.addCallback(_check_ro_uri)
2311 # make sure we can get to it from /uri/URI
2312 d.addCallback(lambda res:
2313 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2314 d.addCallback(lambda res:
2315 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2317 # and that HEAD computes the size correctly
2318 d.addCallback(lambda res:
2319 self.HEAD(self.public_url + "/foo/new.txt",
2320 return_response=True))
2321 def _got_headers((res, status, headers)):
2322 self.failUnlessReallyEqual(res, "")
2323 self.failUnlessReallyEqual(headers["content-length"][0],
2324 str(len(NEW2_CONTENTS)))
2325 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2326 d.addCallback(_got_headers)
2328 # make sure that outdated size limits aren't enforced anymore.
2329 d.addCallback(lambda ignored:
2330 self.POST(self.public_url + "/foo", t="upload",
2333 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2334 d.addErrback(self.dump_error)
2337 def test_POST_upload_mutable_toobig(self):
2338 # SDMF had a size limti that was removed a while ago. MDMF has
2339 # never had a size limit. Test to make sure that we do not
2340 # encounter errors when trying to upload large mutable files,
2341 # since there should be no coded prohibitions regarding large
2343 d = self.POST(self.public_url + "/foo",
2344 t="upload", mutable="true",
2345 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2348 def dump_error(self, f):
2349 # if the web server returns an error code (like 400 Bad Request),
2350 # web.client.getPage puts the HTTP response body into the .response
2351 # attribute of the exception object that it gives back. It does not
2352 # appear in the Failure's repr(), so the ERROR that trial displays
2353 # will be rather terse and unhelpful. addErrback this method to the
2354 # end of your chain to get more information out of these errors.
2355 if f.check(error.Error):
2356 print "web.error.Error:"
2358 print f.value.response
2361 def test_POST_upload_replace(self):
2362 d = self.POST(self.public_url + "/foo", t="upload",
2363 file=("bar.txt", self.NEWFILE_CONTENTS))
2365 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2366 d.addCallback(lambda res:
2367 self.failUnlessChildContentsAre(fn, u"bar.txt",
2368 self.NEWFILE_CONTENTS))
2371 def test_POST_upload_no_replace_ok(self):
2372 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2373 file=("new.txt", self.NEWFILE_CONTENTS))
2374 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2375 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2376 self.NEWFILE_CONTENTS))
2379 def test_POST_upload_no_replace_queryarg(self):
2380 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2381 file=("bar.txt", self.NEWFILE_CONTENTS))
2382 d.addBoth(self.shouldFail, error.Error,
2383 "POST_upload_no_replace_queryarg",
2385 "There was already a child by that name, and you asked me "
2386 "to not replace it")
2387 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2388 d.addCallback(self.failUnlessIsBarDotTxt)
2391 def test_POST_upload_no_replace_field(self):
2392 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2393 file=("bar.txt", self.NEWFILE_CONTENTS))
2394 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2396 "There was already a child by that name, and you asked me "
2397 "to not replace it")
2398 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2399 d.addCallback(self.failUnlessIsBarDotTxt)
2402 def test_POST_upload_whendone(self):
2403 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2404 file=("new.txt", self.NEWFILE_CONTENTS))
2405 d.addBoth(self.shouldRedirect, "/THERE")
2407 d.addCallback(lambda res:
2408 self.failUnlessChildContentsAre(fn, u"new.txt",
2409 self.NEWFILE_CONTENTS))
2412 def test_POST_upload_named(self):
2414 d = self.POST(self.public_url + "/foo", t="upload",
2415 name="new.txt", file=self.NEWFILE_CONTENTS)
2416 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2417 d.addCallback(lambda res:
2418 self.failUnlessChildContentsAre(fn, u"new.txt",
2419 self.NEWFILE_CONTENTS))
2422 def test_POST_upload_named_badfilename(self):
2423 d = self.POST(self.public_url + "/foo", t="upload",
2424 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2425 d.addBoth(self.shouldFail, error.Error,
2426 "test_POST_upload_named_badfilename",
2428 "name= may not contain a slash",
2430 # make sure that nothing was added
2431 d.addCallback(lambda res:
2432 self.failUnlessNodeKeysAre(self._foo_node,
2433 [u"bar.txt", u"baz.txt", u"blockingfile",
2434 u"empty", u"n\u00fc.txt", u"quux.txt",
2438 def test_POST_FILEURL_check(self):
2439 bar_url = self.public_url + "/foo/bar.txt"
2440 d = self.POST(bar_url, t="check")
2442 self.failUnlessIn("Healthy :", res)
2443 d.addCallback(_check)
2444 redir_url = "http://allmydata.org/TARGET"
2445 def _check2(statuscode, target):
2446 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2447 self.failUnlessReallyEqual(target, redir_url)
2448 d.addCallback(lambda res:
2449 self.shouldRedirect2("test_POST_FILEURL_check",
2453 when_done=redir_url))
2454 d.addCallback(lambda res:
2455 self.POST(bar_url, t="check", return_to=redir_url))
2457 self.failUnlessIn("Healthy :", res)
2458 self.failUnlessIn("Return to file", res)
2459 self.failUnlessIn(redir_url, res)
2460 d.addCallback(_check3)
2462 d.addCallback(lambda res:
2463 self.POST(bar_url, t="check", output="JSON"))
2464 def _check_json(res):
2465 data = simplejson.loads(res)
2466 self.failUnlessIn("storage-index", data)
2467 self.failUnless(data["results"]["healthy"])
2468 d.addCallback(_check_json)
2472 def test_POST_FILEURL_check_and_repair(self):
2473 bar_url = self.public_url + "/foo/bar.txt"
2474 d = self.POST(bar_url, t="check", repair="true")
2476 self.failUnlessIn("Healthy :", res)
2477 d.addCallback(_check)
2478 redir_url = "http://allmydata.org/TARGET"
2479 def _check2(statuscode, target):
2480 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2481 self.failUnlessReallyEqual(target, redir_url)
2482 d.addCallback(lambda res:
2483 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2486 t="check", repair="true",
2487 when_done=redir_url))
2488 d.addCallback(lambda res:
2489 self.POST(bar_url, t="check", return_to=redir_url))
2491 self.failUnlessIn("Healthy :", res)
2492 self.failUnlessIn("Return to file", res)
2493 self.failUnlessIn(redir_url, res)
2494 d.addCallback(_check3)
2497 def test_POST_DIRURL_check(self):
2498 foo_url = self.public_url + "/foo/"
2499 d = self.POST(foo_url, t="check")
2501 self.failUnlessIn("Healthy :", res)
2502 d.addCallback(_check)
2503 redir_url = "http://allmydata.org/TARGET"
2504 def _check2(statuscode, target):
2505 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2506 self.failUnlessReallyEqual(target, redir_url)
2507 d.addCallback(lambda res:
2508 self.shouldRedirect2("test_POST_DIRURL_check",
2512 when_done=redir_url))
2513 d.addCallback(lambda res:
2514 self.POST(foo_url, t="check", return_to=redir_url))
2516 self.failUnlessIn("Healthy :", res)
2517 self.failUnlessIn("Return to file/directory", res)
2518 self.failUnlessIn(redir_url, res)
2519 d.addCallback(_check3)
2521 d.addCallback(lambda res:
2522 self.POST(foo_url, t="check", output="JSON"))
2523 def _check_json(res):
2524 data = simplejson.loads(res)
2525 self.failUnlessIn("storage-index", data)
2526 self.failUnless(data["results"]["healthy"])
2527 d.addCallback(_check_json)
2531 def test_POST_DIRURL_check_and_repair(self):
2532 foo_url = self.public_url + "/foo/"
2533 d = self.POST(foo_url, t="check", repair="true")
2535 self.failUnlessIn("Healthy :", res)
2536 d.addCallback(_check)
2537 redir_url = "http://allmydata.org/TARGET"
2538 def _check2(statuscode, target):
2539 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2540 self.failUnlessReallyEqual(target, redir_url)
2541 d.addCallback(lambda res:
2542 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2545 t="check", repair="true",
2546 when_done=redir_url))
2547 d.addCallback(lambda res:
2548 self.POST(foo_url, t="check", return_to=redir_url))
2550 self.failUnlessIn("Healthy :", res)
2551 self.failUnlessIn("Return to file/directory", res)
2552 self.failUnlessIn(redir_url, res)
2553 d.addCallback(_check3)
2556 def test_POST_FILEURL_mdmf_check(self):
2557 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2558 d = self.POST(quux_url, t="check")
2560 self.failUnlessIn("Healthy", res)
2561 d.addCallback(_check)
2562 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2563 d.addCallback(lambda ignored:
2564 self.POST(quux_extension_url, t="check"))
2565 d.addCallback(_check)
2568 def test_POST_FILEURL_mdmf_check_and_repair(self):
2569 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2570 d = self.POST(quux_url, t="check", repair="true")
2572 self.failUnlessIn("Healthy", res)
2573 d.addCallback(_check)
2574 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2575 d.addCallback(lambda ignored:
2576 self.POST(quux_extension_url, t="check", repair="true"))
2577 d.addCallback(_check)
2580 def wait_for_operation(self, ignored, ophandle):
2581 url = "/operations/" + ophandle
2582 url += "?t=status&output=JSON"
2585 data = simplejson.loads(res)
2586 if not data["finished"]:
2587 d = self.stall(delay=1.0)
2588 d.addCallback(self.wait_for_operation, ophandle)
2594 def get_operation_results(self, ignored, ophandle, output=None):
2595 url = "/operations/" + ophandle
2598 url += "&output=" + output
2601 if output and output.lower() == "json":
2602 return simplejson.loads(res)
2607 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2608 d = self.shouldFail2(error.Error,
2609 "test_POST_DIRURL_deepcheck_no_ophandle",
2611 "slow operation requires ophandle=",
2612 self.POST, self.public_url, t="start-deep-check")
2615 def test_POST_DIRURL_deepcheck(self):
2616 def _check_redirect(statuscode, target):
2617 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2618 self.failUnless(target.endswith("/operations/123"))
2619 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2620 self.POST, self.public_url,
2621 t="start-deep-check", ophandle="123")
2622 d.addCallback(self.wait_for_operation, "123")
2623 def _check_json(data):
2624 self.failUnlessReallyEqual(data["finished"], True)
2625 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2626 self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2627 d.addCallback(_check_json)
2628 d.addCallback(self.get_operation_results, "123", "html")
2629 def _check_html(res):
2630 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2631 self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2632 self.failUnlessIn(FAVICON_MARKUP, res)
2633 d.addCallback(_check_html)
2635 d.addCallback(lambda res:
2636 self.GET("/operations/123/"))
2637 d.addCallback(_check_html) # should be the same as without the slash
2639 d.addCallback(lambda res:
2640 self.shouldFail2(error.Error, "one", "404 Not Found",
2641 "No detailed results for SI bogus",
2642 self.GET, "/operations/123/bogus"))
2644 foo_si = self._foo_node.get_storage_index()
2645 foo_si_s = base32.b2a(foo_si)
2646 d.addCallback(lambda res:
2647 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2648 def _check_foo_json(res):
2649 data = simplejson.loads(res)
2650 self.failUnlessEqual(data["storage-index"], foo_si_s)
2651 self.failUnless(data["results"]["healthy"])
2652 d.addCallback(_check_foo_json)
2655 def test_POST_DIRURL_deepcheck_and_repair(self):
2656 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2657 ophandle="124", output="json", followRedirect=True)
2658 d.addCallback(self.wait_for_operation, "124")
2659 def _check_json(data):
2660 self.failUnlessReallyEqual(data["finished"], True)
2661 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2662 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2663 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2664 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2665 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2666 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2667 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2668 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2669 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2670 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2671 d.addCallback(_check_json)
2672 d.addCallback(self.get_operation_results, "124", "html")
2673 def _check_html(res):
2674 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2676 self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2677 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2678 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2680 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2681 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2682 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2684 self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2685 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2686 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2688 self.failUnlessIn(FAVICON_MARKUP, res)
2689 d.addCallback(_check_html)
2692 def test_POST_FILEURL_bad_t(self):
2693 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2694 "POST to file: bad t=bogus",
2695 self.POST, self.public_url + "/foo/bar.txt",
2699 def test_POST_mkdir(self): # return value?
2700 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2701 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2702 d.addCallback(self.failUnlessNodeKeysAre, [])
2705 def test_POST_mkdir_mdmf(self):
2706 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2707 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708 d.addCallback(lambda node:
2709 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2712 def test_POST_mkdir_sdmf(self):
2713 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2714 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715 d.addCallback(lambda node:
2716 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2719 def test_POST_mkdir_bad_format(self):
2720 return self.shouldHTTPError("POST_mkdir_bad_format",
2721 400, "Bad Request", "Unknown format: foo",
2722 self.POST, self.public_url +
2723 "/foo?t=mkdir&name=newdir&format=foo")
2725 def test_POST_mkdir_initial_children(self):
2726 (newkids, caps) = self._create_initial_children()
2727 d = self.POST2(self.public_url +
2728 "/foo?t=mkdir-with-children&name=newdir",
2729 simplejson.dumps(newkids))
2730 d.addCallback(lambda res:
2731 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2732 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2733 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2734 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2735 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2738 def test_POST_mkdir_initial_children_mdmf(self):
2739 (newkids, caps) = self._create_initial_children()
2740 d = self.POST2(self.public_url +
2741 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2742 simplejson.dumps(newkids))
2743 d.addCallback(lambda res:
2744 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2745 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2746 d.addCallback(lambda node:
2747 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2748 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2749 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2754 def test_POST_mkdir_initial_children_sdmf(self):
2755 (newkids, caps) = self._create_initial_children()
2756 d = self.POST2(self.public_url +
2757 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2758 simplejson.dumps(newkids))
2759 d.addCallback(lambda res:
2760 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2761 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2762 d.addCallback(lambda node:
2763 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2764 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2765 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2769 def test_POST_mkdir_initial_children_bad_format(self):
2770 (newkids, caps) = self._create_initial_children()
2771 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2772 400, "Bad Request", "Unknown format: foo",
2773 self.POST, self.public_url + \
2774 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2775 simplejson.dumps(newkids))
2777 def test_POST_mkdir_immutable(self):
2778 (newkids, caps) = self._create_immutable_children()
2779 d = self.POST2(self.public_url +
2780 "/foo?t=mkdir-immutable&name=newdir",
2781 simplejson.dumps(newkids))
2782 d.addCallback(lambda res:
2783 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2784 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2785 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2786 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2787 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2788 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2789 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2790 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2791 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2792 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2793 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2794 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2795 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2798 def test_POST_mkdir_immutable_bad(self):
2799 (newkids, caps) = self._create_initial_children()
2800 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2802 "needed to be immutable but was not",
2805 "/foo?t=mkdir-immutable&name=newdir",
2806 simplejson.dumps(newkids))
2809 def test_POST_mkdir_2(self):
2810 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2811 d.addCallback(lambda res:
2812 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2813 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2814 d.addCallback(self.failUnlessNodeKeysAre, [])
2817 def test_POST_mkdirs_2(self):
2818 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2819 d.addCallback(lambda res:
2820 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2821 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2822 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2823 d.addCallback(self.failUnlessNodeKeysAre, [])
2826 def test_POST_mkdir_no_parentdir_noredirect(self):
2827 d = self.POST("/uri?t=mkdir")
2828 def _after_mkdir(res):
2829 uri.DirectoryURI.init_from_string(res)
2830 d.addCallback(_after_mkdir)
2833 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2834 d = self.POST("/uri?t=mkdir&format=mdmf")
2835 def _after_mkdir(res):
2836 u = uri.from_string(res)
2837 # Check that this is an MDMF writecap
2838 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2839 d.addCallback(_after_mkdir)
2842 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2843 d = self.POST("/uri?t=mkdir&format=sdmf")
2844 def _after_mkdir(res):
2845 u = uri.from_string(res)
2846 self.failUnlessIsInstance(u, uri.DirectoryURI)
2847 d.addCallback(_after_mkdir)
2850 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2851 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2852 400, "Bad Request", "Unknown format: foo",
2853 self.POST, self.public_url +
2854 "/uri?t=mkdir&format=foo")
2856 def test_POST_mkdir_no_parentdir_noredirect2(self):
2857 # make sure form-based arguments (as on the welcome page) still work
2858 d = self.POST("/uri", t="mkdir")
2859 def _after_mkdir(res):
2860 uri.DirectoryURI.init_from_string(res)
2861 d.addCallback(_after_mkdir)
2862 d.addErrback(self.explain_web_error)
2865 def test_POST_mkdir_no_parentdir_redirect(self):
2866 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2867 d.addBoth(self.shouldRedirect, None, statuscode='303')
2868 def _check_target(target):
2869 target = urllib.unquote(target)
2870 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2871 d.addCallback(_check_target)
2874 def test_POST_mkdir_no_parentdir_redirect2(self):
2875 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2876 d.addBoth(self.shouldRedirect, None, statuscode='303')
2877 def _check_target(target):
2878 target = urllib.unquote(target)
2879 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2880 d.addCallback(_check_target)
2881 d.addErrback(self.explain_web_error)
2884 def _make_readonly(self, u):
2885 ro_uri = uri.from_string(u).get_readonly()
2888 return ro_uri.to_string()
2890 def _create_initial_children(self):
2891 contents, n, filecap1 = self.makefile(12)
2892 md1 = {"metakey1": "metavalue1"}
2893 filecap2 = make_mutable_file_uri()
2894 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2895 filecap3 = node3.get_readonly_uri()
2896 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2897 dircap = DirectoryNode(node4, None, None).get_uri()
2898 mdmfcap = make_mutable_file_uri(mdmf=True)
2899 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2900 emptydircap = "URI:DIR2-LIT:"
2901 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
2902 "ro_uri": self._make_readonly(filecap1),
2903 "metadata": md1, }],
2904 u"child-mutable": ["filenode", {"rw_uri": filecap2,
2905 "ro_uri": self._make_readonly(filecap2)}],
2906 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2907 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
2908 "ro_uri": unknown_rocap}],
2909 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
2910 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2911 u"dirchild": ["dirnode", {"rw_uri": dircap,
2912 "ro_uri": self._make_readonly(dircap)}],
2913 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2914 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2915 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2916 "ro_uri": self._make_readonly(mdmfcap)}],
2918 return newkids, {'filecap1': filecap1,
2919 'filecap2': filecap2,
2920 'filecap3': filecap3,
2921 'unknown_rwcap': unknown_rwcap,
2922 'unknown_rocap': unknown_rocap,
2923 'unknown_immcap': unknown_immcap,
2925 'litdircap': litdircap,
2926 'emptydircap': emptydircap,
2929 def _create_immutable_children(self):
2930 contents, n, filecap1 = self.makefile(12)
2931 md1 = {"metakey1": "metavalue1"}
2932 tnode = create_chk_filenode("immutable directory contents\n"*10)
2933 dnode = DirectoryNode(tnode, None, None)
2934 assert not dnode.is_mutable()
2935 immdircap = dnode.get_uri()
2936 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2937 emptydircap = "URI:DIR2-LIT:"
2938 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2939 "metadata": md1, }],
2940 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2941 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2942 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2943 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2945 return newkids, {'filecap1': filecap1,
2946 'unknown_immcap': unknown_immcap,
2947 'immdircap': immdircap,
2948 'litdircap': litdircap,
2949 'emptydircap': emptydircap}
2951 def test_POST_mkdir_no_parentdir_initial_children(self):
2952 (newkids, caps) = self._create_initial_children()
2953 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2954 def _after_mkdir(res):
2955 self.failUnless(res.startswith("URI:DIR"), res)
2956 n = self.s.create_node_from_uri(res)
2957 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2958 d2.addCallback(lambda ign:
2959 self.failUnlessROChildURIIs(n, u"child-imm",
2961 d2.addCallback(lambda ign:
2962 self.failUnlessRWChildURIIs(n, u"child-mutable",
2964 d2.addCallback(lambda ign:
2965 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2967 d2.addCallback(lambda ign:
2968 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2969 caps['unknown_rwcap']))
2970 d2.addCallback(lambda ign:
2971 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2972 caps['unknown_rocap']))
2973 d2.addCallback(lambda ign:
2974 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2975 caps['unknown_immcap']))
2976 d2.addCallback(lambda ign:
2977 self.failUnlessRWChildURIIs(n, u"dirchild",
2980 d.addCallback(_after_mkdir)
2983 def test_POST_mkdir_no_parentdir_unexpected_children(self):
2984 # the regular /uri?t=mkdir operation is specified to ignore its body.
2985 # Only t=mkdir-with-children pays attention to it.
2986 (newkids, caps) = self._create_initial_children()
2987 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2989 "t=mkdir does not accept children=, "
2990 "try t=mkdir-with-children instead",
2991 self.POST2, "/uri?t=mkdir", # without children
2992 simplejson.dumps(newkids))
2995 def test_POST_noparent_bad(self):
2996 d = self.shouldHTTPError("POST_noparent_bad",
2998 "/uri accepts only PUT, PUT?t=mkdir, "
2999 "POST?t=upload, and POST?t=mkdir",
3000 self.POST, "/uri?t=bogus")
3003 def test_POST_mkdir_no_parentdir_immutable(self):
3004 (newkids, caps) = self._create_immutable_children()
3005 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3006 def _after_mkdir(res):
3007 self.failUnless(res.startswith("URI:DIR"), res)
3008 n = self.s.create_node_from_uri(res)
3009 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3010 d2.addCallback(lambda ign:
3011 self.failUnlessROChildURIIs(n, u"child-imm",
3013 d2.addCallback(lambda ign:
3014 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3015 caps['unknown_immcap']))
3016 d2.addCallback(lambda ign:
3017 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3019 d2.addCallback(lambda ign:
3020 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3022 d2.addCallback(lambda ign:
3023 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3024 caps['emptydircap']))
3026 d.addCallback(_after_mkdir)
3029 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3030 (newkids, caps) = self._create_initial_children()
3031 d = self.shouldFail2(error.Error,
3032 "test_POST_mkdir_no_parentdir_immutable_bad",
3034 "needed to be immutable but was not",
3036 "/uri?t=mkdir-immutable",
3037 simplejson.dumps(newkids))
3040 def test_welcome_page_mkdir_button(self):
3041 # Fetch the welcome page.
3043 def _after_get_welcome_page(res):
3044 MKDIR_BUTTON_RE = re.compile(
3045 '<form action="([^"]*)" method="post".*?'
3046 '<input type="hidden" name="t" value="([^"]*)" />'
3047 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3048 '<input type="submit" value="Create a directory" />',
3050 mo = MKDIR_BUTTON_RE.search(res)
3051 formaction = mo.group(1)
3053 formaname = mo.group(3)
3054 formavalue = mo.group(4)
3055 return (formaction, formt, formaname, formavalue)
3056 d.addCallback(_after_get_welcome_page)
3057 def _after_parse_form(res):
3058 (formaction, formt, formaname, formavalue) = res
3059 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3060 d.addCallback(_after_parse_form)
3061 d.addBoth(self.shouldRedirect, None, statuscode='303')
3064 def test_POST_mkdir_replace(self): # return value?
3065 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3066 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3067 d.addCallback(self.failUnlessNodeKeysAre, [])
3070 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3071 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3072 d.addBoth(self.shouldFail, error.Error,
3073 "POST_mkdir_no_replace_queryarg",
3075 "There was already a child by that name, and you asked me "
3076 "to not replace it")
3077 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3078 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3081 def test_POST_mkdir_no_replace_field(self): # return value?
3082 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3084 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3086 "There was already a child by that name, and you asked me "
3087 "to not replace it")
3088 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3089 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3092 def test_POST_mkdir_whendone_field(self):
3093 d = self.POST(self.public_url + "/foo",
3094 t="mkdir", name="newdir", when_done="/THERE")
3095 d.addBoth(self.shouldRedirect, "/THERE")
3096 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3097 d.addCallback(self.failUnlessNodeKeysAre, [])
3100 def test_POST_mkdir_whendone_queryarg(self):
3101 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3102 t="mkdir", name="newdir")
3103 d.addBoth(self.shouldRedirect, "/THERE")
3104 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3105 d.addCallback(self.failUnlessNodeKeysAre, [])
3108 def test_POST_bad_t(self):
3109 d = self.shouldFail2(error.Error, "POST_bad_t",
3111 "POST to a directory with bad t=BOGUS",
3112 self.POST, self.public_url + "/foo", t="BOGUS")
3115 def test_POST_set_children(self, command_name="set_children"):
3116 contents9, n9, newuri9 = self.makefile(9)
3117 contents10, n10, newuri10 = self.makefile(10)
3118 contents11, n11, newuri11 = self.makefile(11)
3121 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3124 "ctime": 1002777696.7564139,
3125 "mtime": 1002777696.7564139
3128 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3131 "ctime": 1002777696.7564139,
3132 "mtime": 1002777696.7564139
3135 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3138 "ctime": 1002777696.7564139,
3139 "mtime": 1002777696.7564139
3142 }""" % (newuri9, newuri10, newuri11)
3144 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3146 d = client.getPage(url, method="POST", postdata=reqbody)
3148 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3149 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3150 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3152 d.addCallback(_then)
3153 d.addErrback(self.dump_error)
3156 def test_POST_set_children_with_hyphen(self):
3157 return self.test_POST_set_children(command_name="set-children")
3159 def test_POST_link_uri(self):
3160 contents, n, newuri = self.makefile(8)
3161 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3162 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3163 d.addCallback(lambda res:
3164 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3168 def test_POST_link_uri_replace(self):
3169 contents, n, newuri = self.makefile(8)
3170 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3171 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3172 d.addCallback(lambda res:
3173 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3177 def test_POST_link_uri_unknown_bad(self):
3178 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3179 d.addBoth(self.shouldFail, error.Error,
3180 "POST_link_uri_unknown_bad",
3182 "unknown cap in a write slot")
3185 def test_POST_link_uri_unknown_ro_good(self):
3186 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3187 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3190 def test_POST_link_uri_unknown_imm_good(self):
3191 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3192 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3195 def test_POST_link_uri_no_replace_queryarg(self):
3196 contents, n, newuri = self.makefile(8)
3197 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3198 name="bar.txt", uri=newuri)
3199 d.addBoth(self.shouldFail, error.Error,
3200 "POST_link_uri_no_replace_queryarg",
3202 "There was already a child by that name, and you asked me "
3203 "to not replace it")
3204 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3205 d.addCallback(self.failUnlessIsBarDotTxt)
3208 def test_POST_link_uri_no_replace_field(self):
3209 contents, n, newuri = self.makefile(8)
3210 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3211 name="bar.txt", uri=newuri)
3212 d.addBoth(self.shouldFail, error.Error,
3213 "POST_link_uri_no_replace_field",
3215 "There was already a child by that name, and you asked me "
3216 "to not replace it")
3217 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3218 d.addCallback(self.failUnlessIsBarDotTxt)
3221 def test_POST_delete(self, command_name='delete'):
3222 d = self._foo_node.list()
3223 def _check_before(children):
3224 self.failUnlessIn(u"bar.txt", children)
3225 d.addCallback(_check_before)
3226 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3227 d.addCallback(lambda res: self._foo_node.list())
3228 def _check_after(children):
3229 self.failIfIn(u"bar.txt", children)
3230 d.addCallback(_check_after)
3233 def test_POST_unlink(self):
3234 return self.test_POST_delete(command_name='unlink')
3236 def test_POST_rename_file(self):
3237 d = self.POST(self.public_url + "/foo", t="rename",
3238 from_name="bar.txt", to_name='wibble.txt')
3239 d.addCallback(lambda res:
3240 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3241 d.addCallback(lambda res:
3242 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3243 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3244 d.addCallback(self.failUnlessIsBarDotTxt)
3245 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3246 d.addCallback(self.failUnlessIsBarJSON)
3249 def test_POST_rename_file_redundant(self):
3250 d = self.POST(self.public_url + "/foo", t="rename",
3251 from_name="bar.txt", to_name='bar.txt')
3252 d.addCallback(lambda res:
3253 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3254 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3255 d.addCallback(self.failUnlessIsBarDotTxt)
3256 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3257 d.addCallback(self.failUnlessIsBarJSON)
3260 def test_POST_rename_file_replace(self):
3261 # rename a file and replace a directory with it
3262 d = self.POST(self.public_url + "/foo", t="rename",
3263 from_name="bar.txt", to_name='empty')
3264 d.addCallback(lambda res:
3265 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3266 d.addCallback(lambda res:
3267 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3268 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3269 d.addCallback(self.failUnlessIsBarDotTxt)
3270 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3271 d.addCallback(self.failUnlessIsBarJSON)
3274 def test_POST_rename_file_no_replace_queryarg(self):
3275 # rename a file and replace a directory with it
3276 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3277 from_name="bar.txt", to_name='empty')
3278 d.addBoth(self.shouldFail, error.Error,
3279 "POST_rename_file_no_replace_queryarg",
3281 "There was already a child by that name, and you asked me "
3282 "to not replace it")
3283 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3284 d.addCallback(self.failUnlessIsEmptyJSON)
3287 def test_POST_rename_file_no_replace_field(self):
3288 # rename a file and replace a directory with it
3289 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3290 from_name="bar.txt", to_name='empty')
3291 d.addBoth(self.shouldFail, error.Error,
3292 "POST_rename_file_no_replace_field",
3294 "There was already a child by that name, and you asked me "
3295 "to not replace it")
3296 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3297 d.addCallback(self.failUnlessIsEmptyJSON)
3300 def failUnlessIsEmptyJSON(self, res):
3301 data = simplejson.loads(res)
3302 self.failUnlessEqual(data[0], "dirnode", data)
3303 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3305 def test_POST_rename_file_slash_fail(self):
3306 d = self.POST(self.public_url + "/foo", t="rename",
3307 from_name="bar.txt", to_name='kirk/spock.txt')
3308 d.addBoth(self.shouldFail, error.Error,
3309 "test_POST_rename_file_slash_fail",
3311 "to_name= may not contain a slash",
3313 d.addCallback(lambda res:
3314 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3317 def test_POST_rename_dir(self):
3318 d = self.POST(self.public_url, t="rename",
3319 from_name="foo", to_name='plunk')
3320 d.addCallback(lambda res:
3321 self.failIfNodeHasChild(self.public_root, u"foo"))
3322 d.addCallback(lambda res:
3323 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3324 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3325 d.addCallback(self.failUnlessIsFooJSON)
3328 def test_POST_move_file(self):
3329 d = self.POST(self.public_url + "/foo", t="move",
3330 from_name="bar.txt", to_dir="sub")
3331 d.addCallback(lambda res:
3332 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3333 d.addCallback(lambda res:
3334 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3335 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3336 d.addCallback(self.failUnlessIsBarDotTxt)
3337 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3338 d.addCallback(self.failUnlessIsBarJSON)
3341 def test_POST_move_file_new_name(self):
3342 d = self.POST(self.public_url + "/foo", t="move",
3343 from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3344 d.addCallback(lambda res:
3345 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3346 d.addCallback(lambda res:
3347 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3348 d.addCallback(lambda res:
3349 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3350 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3351 d.addCallback(self.failUnlessIsBarDotTxt)
3352 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3353 d.addCallback(self.failUnlessIsBarJSON)
3356 def test_POST_move_file_replace(self):
3357 d = self.POST(self.public_url + "/foo", t="move",
3358 from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3359 d.addCallback(lambda res:
3360 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3361 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3362 d.addCallback(self.failUnlessIsBarDotTxt)
3363 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3364 d.addCallback(self.failUnlessIsBarJSON)
3367 def test_POST_move_file_no_replace(self):
3368 d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3370 "There was already a child by that name, and you asked me to not replace it",
3371 self.POST, self.public_url + "/foo", t="move",
3372 replace="false", from_name="bar.txt",
3373 to_name="baz.txt", to_dir="sub")
3374 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3375 d.addCallback(self.failUnlessIsBarDotTxt)
3376 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3377 d.addCallback(self.failUnlessIsBarJSON)
3378 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3379 d.addCallback(self.failUnlessIsSubBazDotTxt)
3382 def test_POST_move_file_slash_fail(self):
3383 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3385 "to_name= may not contain a slash",
3386 self.POST, self.public_url + "/foo", t="move",
3387 from_name="bar.txt",
3388 to_name="slash/fail.txt", to_dir="sub")
3389 d.addCallback(lambda res:
3390 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3391 d.addCallback(lambda res:
3392 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3393 d.addCallback(lambda ign:
3394 self.shouldFail2(error.Error,
3395 "test_POST_rename_file_slash_fail2",
3397 "from_name= may not contain a slash",
3398 self.POST, self.public_url + "/foo",
3400 from_name="nope/bar.txt",
3401 to_name="fail.txt", to_dir="sub"))
3404 def test_POST_move_file_no_target(self):
3405 d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3407 "move requires from_name and to_dir",
3408 self.POST, self.public_url + "/foo", t="move",
3409 from_name="bar.txt", to_name="baz.txt")
3410 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3411 d.addCallback(self.failUnlessIsBarDotTxt)
3412 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3413 d.addCallback(self.failUnlessIsBarJSON)
3414 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3415 d.addCallback(self.failUnlessIsBazDotTxt)
3418 def test_POST_move_file_bad_target_type(self):
3419 d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3420 "400 Bad Request", "invalid target_type parameter",
3422 self.public_url + "/foo", t="move",
3423 target_type="*D", from_name="bar.txt",
3427 def test_POST_move_file_multi_level(self):
3428 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3429 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3430 from_name="bar.txt", to_dir="sub/level2"))
3431 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3432 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3433 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3434 d.addCallback(self.failUnlessIsBarDotTxt)
3435 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3436 d.addCallback(self.failUnlessIsBarJSON)
3439 def test_POST_move_file_to_uri(self):
3440 d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3441 from_name="bar.txt", to_dir=self._sub_uri)
3442 d.addCallback(lambda res:
3443 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3444 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3445 d.addCallback(self.failUnlessIsBarDotTxt)
3446 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3447 d.addCallback(self.failUnlessIsBarJSON)
3450 def test_POST_move_file_to_nonexist_dir(self):
3451 d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3452 "404 Not Found", "No such child: nopechucktesta",
3453 self.POST, self.public_url + "/foo", t="move",
3454 from_name="bar.txt", to_dir="nopechucktesta")
3457 def test_POST_move_file_into_file(self):
3458 d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3459 "400 Bad Request", "to_dir is not a directory",
3460 self.POST, self.public_url + "/foo", t="move",
3461 from_name="bar.txt", to_dir="baz.txt")
3462 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3463 d.addCallback(self.failUnlessIsBazDotTxt)
3464 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3465 d.addCallback(self.failUnlessIsBarDotTxt)
3466 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3467 d.addCallback(self.failUnlessIsBarJSON)
3470 def test_POST_move_file_to_bad_uri(self):
3471 d = self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3472 "400 Bad Request", "to_dir is not a directory",
3473 self.POST, self.public_url + "/foo", t="move",
3474 from_name="bar.txt", target_type="uri",
3475 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3476 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3477 d.addCallback(self.failUnlessIsBarDotTxt)
3478 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3479 d.addCallback(self.failUnlessIsBarJSON)
3482 def test_POST_move_dir(self):
3483 d = self.POST(self.public_url + "/foo", t="move",
3484 from_name="bar.txt", to_dir="empty")
3485 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3486 t="move", from_name="empty", to_dir="sub"))
3487 d.addCallback(lambda res:
3488 self.failIfNodeHasChild(self._foo_node, u"empty"))
3489 d.addCallback(lambda res:
3490 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3491 d.addCallback(lambda res:
3492 self._sub_node.get_child_at_path(u"empty"))
3493 d.addCallback(lambda node:
3494 self.failUnlessNodeHasChild(node, u"bar.txt"))
3495 d.addCallback(lambda res:
3496 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3497 d.addCallback(self.failUnlessIsBarDotTxt)
3500 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3501 """ If target is not None then the redirection has to go to target. If
3502 statuscode is not None then the redirection has to be accomplished with
3503 that HTTP status code."""
3504 if not isinstance(res, failure.Failure):
3505 to_where = (target is None) and "somewhere" or ("to " + target)
3506 self.fail("%s: we were expecting to get redirected %s, not get an"
3507 " actual page: %s" % (which, to_where, res))
3508 res.trap(error.PageRedirect)
3509 if statuscode is not None:
3510 self.failUnlessReallyEqual(res.value.status, statuscode,
3511 "%s: not a redirect" % which)
3512 if target is not None:
3513 # the PageRedirect does not seem to capture the uri= query arg
3514 # properly, so we can't check for it.
3515 realtarget = self.webish_url + target
3516 self.failUnlessReallyEqual(res.value.location, realtarget,
3517 "%s: wrong target" % which)
3518 return res.value.location
3520 def test_GET_URI_form(self):
3521 base = "/uri?uri=%s" % self._bar_txt_uri
3522 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3523 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3525 d.addBoth(self.shouldRedirect, targetbase)
3526 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3527 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3528 d.addCallback(lambda res: self.GET(base+"&t=json"))
3529 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3530 d.addCallback(self.log, "about to get file by uri")
3531 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3532 d.addCallback(self.failUnlessIsBarDotTxt)
3533 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3534 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3535 followRedirect=True))
3536 d.addCallback(self.failUnlessIsFooJSON)
3537 d.addCallback(self.log, "got dir by uri")
3541 def test_GET_URI_form_bad(self):
3542 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3543 "400 Bad Request", "GET /uri requires uri=",
3547 def test_GET_rename_form(self):
3548 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3549 followRedirect=True)
3551 self.failUnlessIn('name="when_done" value="."', res)
3552 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3553 self.failUnlessIn(FAVICON_MARKUP, res)
3554 d.addCallback(_check)
3557 def log(self, res, msg):
3558 #print "MSG: %s RES: %s" % (msg, res)
3562 def test_GET_URI_URL(self):
3563 base = "/uri/%s" % self._bar_txt_uri
3565 d.addCallback(self.failUnlessIsBarDotTxt)
3566 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3567 d.addCallback(self.failUnlessIsBarDotTxt)
3568 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3569 d.addCallback(self.failUnlessIsBarDotTxt)
3572 def test_GET_URI_URL_dir(self):
3573 base = "/uri/%s?t=json" % self._foo_uri
3575 d.addCallback(self.failUnlessIsFooJSON)
3578 def test_GET_URI_URL_missing(self):
3579 base = "/uri/%s" % self._bad_file_uri
3580 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3581 http.GONE, None, "NotEnoughSharesError",
3583 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3584 # here? we must arrange for a download to fail after target.open()
3585 # has been called, and then inspect the response to see that it is
3586 # shorter than we expected.
3589 def test_PUT_DIRURL_uri(self):
3590 d = self.s.create_dirnode()
3592 new_uri = dn.get_uri()
3593 # replace /foo with a new (empty) directory
3594 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3595 d.addCallback(lambda res:
3596 self.failUnlessReallyEqual(res.strip(), new_uri))
3597 d.addCallback(lambda res:
3598 self.failUnlessRWChildURIIs(self.public_root,
3602 d.addCallback(_made_dir)
3605 def test_PUT_DIRURL_uri_noreplace(self):
3606 d = self.s.create_dirnode()
3608 new_uri = dn.get_uri()
3609 # replace /foo with a new (empty) directory, but ask that
3610 # replace=false, so it should fail
3611 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3612 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3614 self.public_url + "/foo?t=uri&replace=false",
3616 d.addCallback(lambda res:
3617 self.failUnlessRWChildURIIs(self.public_root,
3621 d.addCallback(_made_dir)
3624 def test_PUT_DIRURL_bad_t(self):
3625 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3626 "400 Bad Request", "PUT to a directory",
3627 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3628 d.addCallback(lambda res:
3629 self.failUnlessRWChildURIIs(self.public_root,
3634 def test_PUT_NEWFILEURL_uri(self):
3635 contents, n, new_uri = self.makefile(8)
3636 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3637 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3638 d.addCallback(lambda res:
3639 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3643 def test_PUT_NEWFILEURL_mdmf(self):
3644 new_contents = self.NEWFILE_CONTENTS * 300000
3645 d = self.PUT(self.public_url + \
3646 "/foo/mdmf.txt?format=mdmf",
3648 d.addCallback(lambda ignored:
3649 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3650 def _got_json(json):
3651 data = simplejson.loads(json)
3653 self.failUnlessIn("format", data)
3654 self.failUnlessEqual(data["format"], "MDMF")
3655 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3656 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3657 d.addCallback(_got_json)
3660 def test_PUT_NEWFILEURL_sdmf(self):
3661 new_contents = self.NEWFILE_CONTENTS * 300000
3662 d = self.PUT(self.public_url + \
3663 "/foo/sdmf.txt?format=sdmf",
3665 d.addCallback(lambda ignored:
3666 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3667 def _got_json(json):
3668 data = simplejson.loads(json)
3670 self.failUnlessIn("format", data)
3671 self.failUnlessEqual(data["format"], "SDMF")
3672 d.addCallback(_got_json)
3675 def test_PUT_NEWFILEURL_bad_format(self):
3676 new_contents = self.NEWFILE_CONTENTS * 300000
3677 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3678 400, "Bad Request", "Unknown format: foo",
3679 self.PUT, self.public_url + \
3680 "/foo/foo.txt?format=foo",
3683 def test_PUT_NEWFILEURL_uri_replace(self):
3684 contents, n, new_uri = self.makefile(8)
3685 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3686 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3687 d.addCallback(lambda res:
3688 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3692 def test_PUT_NEWFILEURL_uri_no_replace(self):
3693 contents, n, new_uri = self.makefile(8)
3694 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3695 d.addBoth(self.shouldFail, error.Error,
3696 "PUT_NEWFILEURL_uri_no_replace",
3698 "There was already a child by that name, and you asked me "
3699 "to not replace it")
3702 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3703 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3704 d.addBoth(self.shouldFail, error.Error,
3705 "POST_put_uri_unknown_bad",
3707 "unknown cap in a write slot")
3710 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3711 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3712 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3713 u"put-future-ro.txt")
3716 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3717 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3718 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3719 u"put-future-imm.txt")
3722 def test_PUT_NEWFILE_URI(self):
3723 file_contents = "New file contents here\n"
3724 d = self.PUT("/uri", file_contents)
3726 assert isinstance(uri, str), uri
3727 self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3728 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3730 return self.GET("/uri/%s" % uri)
3731 d.addCallback(_check)
3733 self.failUnlessReallyEqual(res, file_contents)
3734 d.addCallback(_check2)
3737 def test_PUT_NEWFILE_URI_not_mutable(self):
3738 file_contents = "New file contents here\n"
3739 d = self.PUT("/uri?mutable=false", file_contents)
3741 assert isinstance(uri, str), uri
3742 self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3743 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3745 return self.GET("/uri/%s" % uri)
3746 d.addCallback(_check)
3748 self.failUnlessReallyEqual(res, file_contents)
3749 d.addCallback(_check2)
3752 def test_PUT_NEWFILE_URI_only_PUT(self):
3753 d = self.PUT("/uri?t=bogus", "")
3754 d.addBoth(self.shouldFail, error.Error,
3755 "PUT_NEWFILE_URI_only_PUT",
3757 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3760 def test_PUT_NEWFILE_URI_mutable(self):
3761 file_contents = "New file contents here\n"
3762 d = self.PUT("/uri?mutable=true", file_contents)
3763 def _check1(filecap):
3764 filecap = filecap.strip()
3765 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3766 self.filecap = filecap
3767 u = uri.WriteableSSKFileURI.init_from_string(filecap)
3768 self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3769 n = self.s.create_node_from_uri(filecap)
3770 return n.download_best_version()
3771 d.addCallback(_check1)
3773 self.failUnlessReallyEqual(data, file_contents)
3774 return self.GET("/uri/%s" % urllib.quote(self.filecap))
3775 d.addCallback(_check2)
3777 self.failUnlessReallyEqual(res, file_contents)
3778 d.addCallback(_check3)
3781 def test_PUT_mkdir(self):
3782 d = self.PUT("/uri?t=mkdir", "")
3784 n = self.s.create_node_from_uri(uri.strip())
3785 d2 = self.failUnlessNodeKeysAre(n, [])
3786 d2.addCallback(lambda res:
3787 self.GET("/uri/%s?t=json" % uri))
3789 d.addCallback(_check)
3790 d.addCallback(self.failUnlessIsEmptyJSON)
3793 def test_PUT_mkdir_mdmf(self):
3794 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3796 u = uri.from_string(res)
3797 # Check that this is an MDMF writecap
3798 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3802 def test_PUT_mkdir_sdmf(self):
3803 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3805 u = uri.from_string(res)
3806 self.failUnlessIsInstance(u, uri.DirectoryURI)
3810 def test_PUT_mkdir_bad_format(self):
3811 return self.shouldHTTPError("PUT_mkdir_bad_format",
3812 400, "Bad Request", "Unknown format: foo",
3813 self.PUT, "/uri?t=mkdir&format=foo",
3816 def test_POST_check(self):
3817 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3819 # this returns a string form of the results, which are probably
3820 # None since we're using fake filenodes.
3821 # TODO: verify that the check actually happened, by changing
3822 # FakeCHKFileNode to count how many times .check() has been
3825 d.addCallback(_done)
3829 def test_PUT_update_at_offset(self):
3830 file_contents = "test file" * 100000 # about 900 KiB
3831 d = self.PUT("/uri?mutable=true", file_contents)
3833 self.filecap = filecap
3834 new_data = file_contents[:100]
3835 new = "replaced and so on"
3837 new_data += file_contents[len(new_data):]
3838 assert len(new_data) == len(file_contents)
3839 self.new_data = new_data
3840 d.addCallback(_then)
3841 d.addCallback(lambda ignored:
3842 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3843 "replaced and so on"))
3844 def _get_data(filecap):
3845 n = self.s.create_node_from_uri(filecap)
3846 return n.download_best_version()
3847 d.addCallback(_get_data)
3848 d.addCallback(lambda results:
3849 self.failUnlessEqual(results, self.new_data))
3850 # Now try appending things to the file
3851 d.addCallback(lambda ignored:
3852 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3854 d.addCallback(_get_data)
3855 d.addCallback(lambda results:
3856 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3857 # and try replacing the beginning of the file
3858 d.addCallback(lambda ignored:
3859 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3860 d.addCallback(_get_data)
3861 d.addCallback(lambda results:
3862 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3865 def test_PUT_update_at_invalid_offset(self):
3866 file_contents = "test file" * 100000 # about 900 KiB
3867 d = self.PUT("/uri?mutable=true", file_contents)
3869 self.filecap = filecap
3870 d.addCallback(_then)
3871 # Negative offsets should cause an error.
3872 d.addCallback(lambda ignored:
3873 self.shouldHTTPError("PUT_update_at_invalid_offset",
3877 "/uri/%s?offset=-1" % self.filecap,
3881 def test_PUT_update_at_offset_immutable(self):
3882 file_contents = "Test file" * 100000
3883 d = self.PUT("/uri", file_contents)
3885 self.filecap = filecap
3886 d.addCallback(_then)
3887 d.addCallback(lambda ignored:
3888 self.shouldHTTPError("PUT_update_at_offset_immutable",
3892 "/uri/%s?offset=50" % self.filecap,
3897 def test_bad_method(self):
3898 url = self.webish_url + self.public_url + "/foo/bar.txt"
3899 d = self.shouldHTTPError("bad_method",
3900 501, "Not Implemented",
3901 "I don't know how to treat a BOGUS request.",
3902 client.getPage, url, method="BOGUS")
3905 def test_short_url(self):
3906 url = self.webish_url + "/uri"
3907 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3908 "I don't know how to treat a DELETE request.",
3909 client.getPage, url, method="DELETE")
3912 def test_ophandle_bad(self):
3913 url = self.webish_url + "/operations/bogus?t=status"
3914 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3915 "unknown/expired handle 'bogus'",
3916 client.getPage, url)
3919 def test_ophandle_cancel(self):
3920 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3921 followRedirect=True)
3922 d.addCallback(lambda ignored:
3923 self.GET("/operations/128?t=status&output=JSON"))
3925 data = simplejson.loads(res)
3926 self.failUnless("finished" in data, res)
3927 monitor = self.ws.root.child_operations.handles["128"][0]
3928 d = self.POST("/operations/128?t=cancel&output=JSON")
3930 data = simplejson.loads(res)
3931 self.failUnless("finished" in data, res)
3932 # t=cancel causes the handle to be forgotten
3933 self.failUnless(monitor.is_cancelled())
3934 d.addCallback(_check2)
3936 d.addCallback(_check1)
3937 d.addCallback(lambda ignored:
3938 self.shouldHTTPError("ophandle_cancel",
3939 404, "404 Not Found",
3940 "unknown/expired handle '128'",
3942 "/operations/128?t=status&output=JSON"))
3945 def test_ophandle_retainfor(self):
3946 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3947 followRedirect=True)
3948 d.addCallback(lambda ignored:
3949 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3951 data = simplejson.loads(res)
3952 self.failUnless("finished" in data, res)
3953 d.addCallback(_check1)
3954 # the retain-for=0 will cause the handle to be expired very soon
3955 d.addCallback(lambda ign:
3956 self.clock.advance(2.0))
3957 d.addCallback(lambda ignored:
3958 self.shouldHTTPError("ophandle_retainfor",
3959 404, "404 Not Found",
3960 "unknown/expired handle '129'",
3962 "/operations/129?t=status&output=JSON"))
3965 def test_ophandle_release_after_complete(self):
3966 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3967 followRedirect=True)
3968 d.addCallback(self.wait_for_operation, "130")
3969 d.addCallback(lambda ignored:
3970 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3971 # the release-after-complete=true will cause the handle to be expired
3972 d.addCallback(lambda ignored:
3973 self.shouldHTTPError("ophandle_release_after_complete",
3974 404, "404 Not Found",
3975 "unknown/expired handle '130'",
3977 "/operations/130?t=status&output=JSON"))
3980 def test_uncollected_ophandle_expiration(self):
3981 # uncollected ophandles should expire after 4 days
3982 def _make_uncollected_ophandle(ophandle):
3983 d = self.POST(self.public_url +
3984 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3985 followRedirect=False)
3986 # When we start the operation, the webapi server will want
3987 # to redirect us to the page for the ophandle, so we get
3988 # confirmation that the operation has started. If the
3989 # manifest operation has finished by the time we get there,
3990 # following that redirect (by setting followRedirect=True
3991 # above) has the side effect of collecting the ophandle that
3992 # we've just created, which means that we can't use the
3993 # ophandle to test the uncollected timeout anymore. So,
3994 # instead, catch the 302 here and don't follow it.
3995 d.addBoth(self.should302, "uncollected_ophandle_creation")
3997 # Create an ophandle, don't collect it, then advance the clock by
3998 # 4 days - 1 second and make sure that the ophandle is still there.
3999 d = _make_uncollected_ophandle(131)
4000 d.addCallback(lambda ign:
4001 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4002 d.addCallback(lambda ign:
4003 self.GET("/operations/131?t=status&output=JSON"))
4005 data = simplejson.loads(res)
4006 self.failUnless("finished" in data, res)
4007 d.addCallback(_check1)
4008 # Create an ophandle, don't collect it, then try to collect it
4009 # after 4 days. It should be gone.
4010 d.addCallback(lambda ign:
4011 _make_uncollected_ophandle(132))
4012 d.addCallback(lambda ign:
4013 self.clock.advance(96*60*60))
4014 d.addCallback(lambda ign:
4015 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4016 404, "404 Not Found",
4017 "unknown/expired handle '132'",
4019 "/operations/132?t=status&output=JSON"))
4022 def test_collected_ophandle_expiration(self):
4023 # collected ophandles should expire after 1 day
4024 def _make_collected_ophandle(ophandle):
4025 d = self.POST(self.public_url +
4026 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4027 followRedirect=True)
4028 # By following the initial redirect, we collect the ophandle
4029 # we've just created.
4031 # Create a collected ophandle, then collect it after 23 hours
4032 # and 59 seconds to make sure that it is still there.
4033 d = _make_collected_ophandle(133)
4034 d.addCallback(lambda ign:
4035 self.clock.advance((24*60*60) - 1))
4036 d.addCallback(lambda ign:
4037 self.GET("/operations/133?t=status&output=JSON"))
4039 data = simplejson.loads(res)
4040 self.failUnless("finished" in data, res)
4041 d.addCallback(_check1)
4042 # Create another uncollected ophandle, then try to collect it
4043 # after 24 hours to make sure that it is gone.
4044 d.addCallback(lambda ign:
4045 _make_collected_ophandle(134))
4046 d.addCallback(lambda ign:
4047 self.clock.advance(24*60*60))
4048 d.addCallback(lambda ign:
4049 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4050 404, "404 Not Found",
4051 "unknown/expired handle '134'",
4053 "/operations/134?t=status&output=JSON"))
4056 def test_incident(self):
4057 d = self.POST("/report_incident", details="eek")
4059 self.failIfIn("<html>", res)
4060 self.failUnlessIn("Thank you for your report!", res)
4061 d.addCallback(_done)
4064 def test_static(self):
4065 webdir = os.path.join(self.staticdir, "subdir")
4066 fileutil.make_dirs(webdir)
4067 f = open(os.path.join(webdir, "hello.txt"), "wb")
4071 d = self.GET("/static/subdir/hello.txt")
4073 self.failUnlessReallyEqual(res, "hello")
4074 d.addCallback(_check)
4078 class IntroducerWeb(unittest.TestCase):
4083 d = defer.succeed(None)
4085 d.addCallback(lambda ign: self.node.stopService())
4086 d.addCallback(flushEventualQueue)
4089 def test_welcome(self):
4090 basedir = "web.IntroducerWeb.test_welcome"
4092 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4093 self.node = IntroducerNode(basedir)
4094 self.ws = self.node.getServiceNamed("webish")
4096 d = fireEventually(None)
4097 d.addCallback(lambda ign: self.node.startService())
4098 d.addCallback(lambda ign: self.node.when_tub_ready())
4100 d.addCallback(lambda ign: self.GET("/"))
4102 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4103 self.failUnlessIn(FAVICON_MARKUP, res)
4104 d.addCallback(_check)
4107 def GET(self, urlpath, followRedirect=False, return_response=False,
4109 # if return_response=True, this fires with (data, statuscode,
4110 # respheaders) instead of just data.
4111 assert not isinstance(urlpath, unicode)
4112 url = self.ws.getURL().rstrip('/') + urlpath
4113 factory = HTTPClientGETFactory(url, method="GET",
4114 followRedirect=followRedirect, **kwargs)
4115 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4116 d = factory.deferred
4117 def _got_data(data):
4118 return (data, factory.status, factory.response_headers)
4120 d.addCallback(_got_data)
4121 return factory.deferred
4124 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4125 def test_load_file(self):
4126 # This will raise an exception unless a well-formed XML file is found under that name.
4127 common.getxmlfile('directory.xhtml').load()
4129 def test_parse_replace_arg(self):
4130 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4131 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4132 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4134 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4135 common.parse_replace_arg, "only_fles")
4137 def test_abbreviate_time(self):
4138 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4139 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4140 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4141 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4142 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4143 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4145 def test_compute_rate(self):
4146 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4147 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4148 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4149 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4150 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4151 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4152 self.shouldFail(AssertionError, "test_compute_rate", "",
4153 common.compute_rate, -100, 10)
4154 self.shouldFail(AssertionError, "test_compute_rate", "",
4155 common.compute_rate, 100, -10)
4158 rate = common.compute_rate(10*1000*1000, 1)
4159 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4161 def test_abbreviate_rate(self):
4162 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4163 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4164 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4165 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4167 def test_abbreviate_size(self):
4168 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4169 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4170 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4171 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4172 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4174 def test_plural(self):
4176 return "%d second%s" % (s, status.plural(s))
4177 self.failUnlessReallyEqual(convert(0), "0 seconds")
4178 self.failUnlessReallyEqual(convert(1), "1 second")
4179 self.failUnlessReallyEqual(convert(2), "2 seconds")
4181 return "has share%s: %s" % (status.plural(s), ",".join(s))
4182 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4183 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4184 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4187 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4189 def CHECK(self, ign, which, args, clientnum=0):
4190 fileurl = self.fileurls[which]
4191 url = fileurl + "?" + args
4192 return self.GET(url, method="POST", clientnum=clientnum)
4194 def test_filecheck(self):
4195 self.basedir = "web/Grid/filecheck"
4197 c0 = self.g.clients[0]
4200 d = c0.upload(upload.Data(DATA, convergence=""))
4201 def _stash_uri(ur, which):
4202 self.uris[which] = ur.uri
4203 d.addCallback(_stash_uri, "good")
4204 d.addCallback(lambda ign:
4205 c0.upload(upload.Data(DATA+"1", convergence="")))
4206 d.addCallback(_stash_uri, "sick")
4207 d.addCallback(lambda ign:
4208 c0.upload(upload.Data(DATA+"2", convergence="")))
4209 d.addCallback(_stash_uri, "dead")
4210 def _stash_mutable_uri(n, which):
4211 self.uris[which] = n.get_uri()
4212 assert isinstance(self.uris[which], str)
4213 d.addCallback(lambda ign:
4214 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4215 d.addCallback(_stash_mutable_uri, "corrupt")
4216 d.addCallback(lambda ign:
4217 c0.upload(upload.Data("literal", convergence="")))
4218 d.addCallback(_stash_uri, "small")
4219 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4220 d.addCallback(_stash_mutable_uri, "smalldir")
4222 def _compute_fileurls(ignored):
4224 for which in self.uris:
4225 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4226 d.addCallback(_compute_fileurls)
4228 def _clobber_shares(ignored):
4229 good_shares = self.find_uri_shares(self.uris["good"])
4230 self.failUnlessReallyEqual(len(good_shares), 10)
4231 sick_shares = self.find_uri_shares(self.uris["sick"])
4232 os.unlink(sick_shares[0][2])
4233 dead_shares = self.find_uri_shares(self.uris["dead"])
4234 for i in range(1, 10):
4235 os.unlink(dead_shares[i][2])
4236 c_shares = self.find_uri_shares(self.uris["corrupt"])
4237 cso = CorruptShareOptions()
4238 cso.stdout = StringIO()
4239 cso.parseOptions([c_shares[0][2]])
4241 d.addCallback(_clobber_shares)
4243 d.addCallback(self.CHECK, "good", "t=check")
4244 def _got_html_good(res):
4245 self.failUnlessIn("Healthy", res)
4246 self.failIfIn("Not Healthy", res)
4247 self.failUnlessIn(FAVICON_MARKUP, res)
4248 d.addCallback(_got_html_good)
4249 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4250 def _got_html_good_return_to(res):
4251 self.failUnlessIn("Healthy", res)
4252 self.failIfIn("Not Healthy", res)
4253 self.failUnlessIn('<a href="somewhere">Return to file', res)
4254 d.addCallback(_got_html_good_return_to)
4255 d.addCallback(self.CHECK, "good", "t=check&output=json")
4256 def _got_json_good(res):
4257 r = simplejson.loads(res)
4258 self.failUnlessEqual(r["summary"], "Healthy")
4259 self.failUnless(r["results"]["healthy"])
4260 self.failIf(r["results"]["needs-rebalancing"])
4261 self.failUnless(r["results"]["recoverable"])
4262 d.addCallback(_got_json_good)
4264 d.addCallback(self.CHECK, "small", "t=check")
4265 def _got_html_small(res):
4266 self.failUnlessIn("Literal files are always healthy", res)
4267 self.failIfIn("Not Healthy", res)
4268 d.addCallback(_got_html_small)
4269 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4270 def _got_html_small_return_to(res):
4271 self.failUnlessIn("Literal files are always healthy", res)
4272 self.failIfIn("Not Healthy", res)
4273 self.failUnlessIn('<a href="somewhere">Return to file', res)
4274 d.addCallback(_got_html_small_return_to)
4275 d.addCallback(self.CHECK, "small", "t=check&output=json")
4276 def _got_json_small(res):
4277 r = simplejson.loads(res)
4278 self.failUnlessEqual(r["storage-index"], "")
4279 self.failUnless(r["results"]["healthy"])
4280 d.addCallback(_got_json_small)
4282 d.addCallback(self.CHECK, "smalldir", "t=check")
4283 def _got_html_smalldir(res):
4284 self.failUnlessIn("Literal files are always healthy", res)
4285 self.failIfIn("Not Healthy", res)
4286 d.addCallback(_got_html_smalldir)
4287 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4288 def _got_json_smalldir(res):
4289 r = simplejson.loads(res)
4290 self.failUnlessEqual(r["storage-index"], "")
4291 self.failUnless(r["results"]["healthy"])
4292 d.addCallback(_got_json_smalldir)
4294 d.addCallback(self.CHECK, "sick", "t=check")
4295 def _got_html_sick(res):
4296 self.failUnlessIn("Not Healthy", res)
4297 d.addCallback(_got_html_sick)
4298 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4299 def _got_json_sick(res):
4300 r = simplejson.loads(res)
4301 self.failUnlessEqual(r["summary"],
4302 "Not Healthy: 9 shares (enc 3-of-10)")
4303 self.failIf(r["results"]["healthy"])
4304 self.failIf(r["results"]["needs-rebalancing"])
4305 self.failUnless(r["results"]["recoverable"])
4306 d.addCallback(_got_json_sick)
4308 d.addCallback(self.CHECK, "dead", "t=check")
4309 def _got_html_dead(res):
4310 self.failUnlessIn("Not Healthy", res)
4311 d.addCallback(_got_html_dead)
4312 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4313 def _got_json_dead(res):
4314 r = simplejson.loads(res)
4315 self.failUnlessEqual(r["summary"],
4316 "Not Healthy: 1 shares (enc 3-of-10)")
4317 self.failIf(r["results"]["healthy"])
4318 self.failIf(r["results"]["needs-rebalancing"])
4319 self.failIf(r["results"]["recoverable"])
4320 d.addCallback(_got_json_dead)
4322 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4323 def _got_html_corrupt(res):
4324 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4325 d.addCallback(_got_html_corrupt)
4326 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4327 def _got_json_corrupt(res):
4328 r = simplejson.loads(res)
4329 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4330 self.failIf(r["results"]["healthy"])
4331 self.failUnless(r["results"]["recoverable"])
4332 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4333 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4334 d.addCallback(_got_json_corrupt)
4336 d.addErrback(self.explain_web_error)
4339 def test_repair_html(self):
4340 self.basedir = "web/Grid/repair_html"
4342 c0 = self.g.clients[0]
4345 d = c0.upload(upload.Data(DATA, convergence=""))
4346 def _stash_uri(ur, which):
4347 self.uris[which] = ur.uri
4348 d.addCallback(_stash_uri, "good")
4349 d.addCallback(lambda ign:
4350 c0.upload(upload.Data(DATA+"1", convergence="")))
4351 d.addCallback(_stash_uri, "sick")
4352 d.addCallback(lambda ign:
4353 c0.upload(upload.Data(DATA+"2", convergence="")))
4354 d.addCallback(_stash_uri, "dead")
4355 def _stash_mutable_uri(n, which):
4356 self.uris[which] = n.get_uri()
4357 assert isinstance(self.uris[which], str)
4358 d.addCallback(lambda ign:
4359 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4360 d.addCallback(_stash_mutable_uri, "corrupt")
4362 def _compute_fileurls(ignored):
4364 for which in self.uris:
4365 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4366 d.addCallback(_compute_fileurls)
4368 def _clobber_shares(ignored):
4369 good_shares = self.find_uri_shares(self.uris["good"])
4370 self.failUnlessReallyEqual(len(good_shares), 10)
4371 sick_shares = self.find_uri_shares(self.uris["sick"])
4372 os.unlink(sick_shares[0][2])
4373 dead_shares = self.find_uri_shares(self.uris["dead"])
4374 for i in range(1, 10):
4375 os.unlink(dead_shares[i][2])
4376 c_shares = self.find_uri_shares(self.uris["corrupt"])
4377 cso = CorruptShareOptions()
4378 cso.stdout = StringIO()
4379 cso.parseOptions([c_shares[0][2]])
4381 d.addCallback(_clobber_shares)
4383 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4384 def _got_html_good(res):
4385 self.failUnlessIn("Healthy", res)
4386 self.failIfIn("Not Healthy", res)
4387 self.failUnlessIn("No repair necessary", res)
4388 self.failUnlessIn(FAVICON_MARKUP, res)
4389 d.addCallback(_got_html_good)
4391 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4392 def _got_html_sick(res):
4393 self.failUnlessIn("Healthy : healthy", res)
4394 self.failIfIn("Not Healthy", res)
4395 self.failUnlessIn("Repair successful", res)
4396 d.addCallback(_got_html_sick)
4398 # repair of a dead file will fail, of course, but it isn't yet
4399 # clear how this should be reported. Right now it shows up as
4402 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4403 #def _got_html_dead(res):
4405 # self.failUnlessIn("Healthy : healthy", res)
4406 # self.failIfIn("Not Healthy", res)
4407 # self.failUnlessIn("No repair necessary", res)
4408 #d.addCallback(_got_html_dead)
4410 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4411 def _got_html_corrupt(res):
4412 self.failUnlessIn("Healthy : Healthy", res)
4413 self.failIfIn("Not Healthy", res)
4414 self.failUnlessIn("Repair successful", res)
4415 d.addCallback(_got_html_corrupt)
4417 d.addErrback(self.explain_web_error)
4420 def test_repair_json(self):
4421 self.basedir = "web/Grid/repair_json"
4423 c0 = self.g.clients[0]
4426 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4427 def _stash_uri(ur, which):
4428 self.uris[which] = ur.uri
4429 d.addCallback(_stash_uri, "sick")
4431 def _compute_fileurls(ignored):
4433 for which in self.uris:
4434 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4435 d.addCallback(_compute_fileurls)
4437 def _clobber_shares(ignored):
4438 sick_shares = self.find_uri_shares(self.uris["sick"])
4439 os.unlink(sick_shares[0][2])
4440 d.addCallback(_clobber_shares)
4442 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4443 def _got_json_sick(res):
4444 r = simplejson.loads(res)
4445 self.failUnlessReallyEqual(r["repair-attempted"], True)
4446 self.failUnlessReallyEqual(r["repair-successful"], True)
4447 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4448 "Not Healthy: 9 shares (enc 3-of-10)")
4449 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4450 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4451 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4452 d.addCallback(_got_json_sick)
4454 d.addErrback(self.explain_web_error)
4457 def test_unknown(self, immutable=False):
4458 self.basedir = "web/Grid/unknown"
4460 self.basedir = "web/Grid/unknown-immutable"
4463 c0 = self.g.clients[0]
4467 # the future cap format may contain slashes, which must be tolerated
4468 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4472 name = u"future-imm"
4473 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4474 d = c0.create_immutable_dirnode({name: (future_node, {})})
4477 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4478 d = c0.create_dirnode()
4480 def _stash_root_and_create_file(n):
4482 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4483 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4485 return self.rootnode.set_node(name, future_node)
4486 d.addCallback(_stash_root_and_create_file)
4488 # make sure directory listing tolerates unknown nodes
4489 d.addCallback(lambda ign: self.GET(self.rooturl))
4490 def _check_directory_html(res, expected_type_suffix):
4491 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4492 '<td>%s</td>' % (expected_type_suffix, str(name)),
4494 self.failUnless(re.search(pattern, res), res)
4495 # find the More Info link for name, should be relative
4496 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4497 info_url = mo.group(1)
4498 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4500 d.addCallback(_check_directory_html, "-IMM")
4502 d.addCallback(_check_directory_html, "")
4504 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4505 def _check_directory_json(res, expect_rw_uri):
4506 data = simplejson.loads(res)
4507 self.failUnlessEqual(data[0], "dirnode")
4508 f = data[1]["children"][name]
4509 self.failUnlessEqual(f[0], "unknown")
4511 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4513 self.failIfIn("rw_uri", f[1])
4515 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4517 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4518 self.failUnlessIn("metadata", f[1])
4519 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4521 def _check_info(res, expect_rw_uri, expect_ro_uri):
4522 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4524 self.failUnlessIn(unknown_rwcap, res)
4527 self.failUnlessIn(unknown_immcap, res)
4529 self.failUnlessIn(unknown_rocap, res)
4531 self.failIfIn(unknown_rocap, res)
4532 self.failIfIn("Raw data as", res)
4533 self.failIfIn("Directory writecap", res)
4534 self.failIfIn("Checker Operations", res)
4535 self.failIfIn("Mutable File Operations", res)
4536 self.failIfIn("Directory Operations", res)
4538 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4539 # why they fail. Possibly related to ticket #922.
4541 d.addCallback(lambda ign: self.GET(expected_info_url))
4542 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4543 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4544 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4546 def _check_json(res, expect_rw_uri):
4547 data = simplejson.loads(res)
4548 self.failUnlessEqual(data[0], "unknown")
4550 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4552 self.failIfIn("rw_uri", data[1])
4555 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4556 self.failUnlessReallyEqual(data[1]["mutable"], False)
4558 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4559 self.failUnlessReallyEqual(data[1]["mutable"], True)
4561 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4562 self.failIfIn("mutable", data[1])
4564 # TODO: check metadata contents
4565 self.failUnlessIn("metadata", data[1])
4567 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4568 d.addCallback(_check_json, expect_rw_uri=not immutable)
4570 # and make sure that a read-only version of the directory can be
4571 # rendered too. This version will not have unknown_rwcap, whether
4572 # or not future_node was immutable.
4573 d.addCallback(lambda ign: self.GET(self.rourl))
4575 d.addCallback(_check_directory_html, "-IMM")
4577 d.addCallback(_check_directory_html, "-RO")
4579 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4580 d.addCallback(_check_directory_json, expect_rw_uri=False)
4582 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4583 d.addCallback(_check_json, expect_rw_uri=False)
4585 # TODO: check that getting t=info from the Info link in the ro directory
4586 # works, and does not include the writecap URI.
4589 def test_immutable_unknown(self):
4590 return self.test_unknown(immutable=True)
4592 def test_mutant_dirnodes_are_omitted(self):
4593 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4596 c = self.g.clients[0]
4601 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4602 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4603 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4605 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4606 # test the dirnode and web layers separately.
4608 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4609 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4610 # When the directory is read, the mutants should be silently disposed of, leaving
4611 # their lonely sibling.
4612 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4613 # because immutable directories don't have a writecap and therefore that field
4614 # isn't (and can't be) decrypted.
4615 # TODO: The field still exists in the netstring. Technically we should check what
4616 # happens if something is put there (_unpack_contents should raise ValueError),
4617 # but that can wait.
4619 lonely_child = nm.create_from_cap(lonely_uri)
4620 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4621 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4623 def _by_hook_or_by_crook():
4625 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4626 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4628 mutant_write_in_ro_child.get_write_uri = lambda: None
4629 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4631 kids = {u"lonely": (lonely_child, {}),
4632 u"ro": (mutant_ro_child, {}),
4633 u"write-in-ro": (mutant_write_in_ro_child, {}),
4635 d = c.create_immutable_dirnode(kids)
4638 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4639 self.failIf(dn.is_mutable())
4640 self.failUnless(dn.is_readonly())
4641 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4642 self.failIf(hasattr(dn._node, 'get_writekey'))
4644 self.failUnlessIn("RO-IMM", rep)
4646 self.failUnlessIn("CHK", cap.to_string())
4649 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4650 return download_to_data(dn._node)
4651 d.addCallback(_created)
4653 def _check_data(data):
4654 # Decode the netstring representation of the directory to check that all children
4655 # are present. This is a bit of an abstraction violation, but there's not really
4656 # any other way to do it given that the real DirectoryNode._unpack_contents would
4657 # strip the mutant children out (which is what we're trying to test, later).
4660 while position < len(data):
4661 entries, position = split_netstring(data, 1, position)
4663 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4664 name = name_utf8.decode("utf-8")
4665 self.failUnlessEqual(rwcapdata, "")
4666 self.failUnlessIn(name, kids)
4667 (expected_child, ign) = kids[name]
4668 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4671 self.failUnlessReallyEqual(numkids, 3)
4672 return self.rootnode.list()
4673 d.addCallback(_check_data)
4675 # Now when we use the real directory listing code, the mutants should be absent.
4676 def _check_kids(children):
4677 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4678 lonely_node, lonely_metadata = children[u"lonely"]
4680 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4681 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4682 d.addCallback(_check_kids)
4684 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4685 d.addCallback(lambda n: n.list())
4686 d.addCallback(_check_kids) # again with dirnode recreated from cap
4688 # Make sure the lonely child can be listed in HTML...
4689 d.addCallback(lambda ign: self.GET(self.rooturl))
4690 def _check_html(res):
4691 self.failIfIn("URI:SSK", res)
4692 get_lonely = "".join([r'<td>FILE</td>',
4694 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4696 r'\s+<td align="right">%d</td>' % len("one"),
4698 self.failUnless(re.search(get_lonely, res), res)
4700 # find the More Info link for name, should be relative
4701 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4702 info_url = mo.group(1)
4703 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4704 d.addCallback(_check_html)
4707 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4708 def _check_json(res):
4709 data = simplejson.loads(res)
4710 self.failUnlessEqual(data[0], "dirnode")
4711 listed_children = data[1]["children"]
4712 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4713 ll_type, ll_data = listed_children[u"lonely"]
4714 self.failUnlessEqual(ll_type, "filenode")
4715 self.failIfIn("rw_uri", ll_data)
4716 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4717 d.addCallback(_check_json)
4720 def test_deep_check(self):
4721 self.basedir = "web/Grid/deep_check"
4723 c0 = self.g.clients[0]
4727 d = c0.create_dirnode()
4728 def _stash_root_and_create_file(n):
4730 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4731 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4732 d.addCallback(_stash_root_and_create_file)
4733 def _stash_uri(fn, which):
4734 self.uris[which] = fn.get_uri()
4736 d.addCallback(_stash_uri, "good")
4737 d.addCallback(lambda ign:
4738 self.rootnode.add_file(u"small",
4739 upload.Data("literal",
4741 d.addCallback(_stash_uri, "small")
4742 d.addCallback(lambda ign:
4743 self.rootnode.add_file(u"sick",
4744 upload.Data(DATA+"1",
4746 d.addCallback(_stash_uri, "sick")
4748 # this tests that deep-check and stream-manifest will ignore
4749 # UnknownNode instances. Hopefully this will also cover deep-stats.
4750 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4751 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4753 def _clobber_shares(ignored):
4754 self.delete_shares_numbered(self.uris["sick"], [0,1])
4755 d.addCallback(_clobber_shares)
4763 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4766 units = [simplejson.loads(line)
4767 for line in res.splitlines()
4770 print "response is:", res
4771 print "undecodeable line was '%s'" % line
4773 self.failUnlessReallyEqual(len(units), 5+1)
4774 # should be parent-first
4776 self.failUnlessEqual(u0["path"], [])
4777 self.failUnlessEqual(u0["type"], "directory")
4778 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4779 u0cr = u0["check-results"]
4780 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4782 ugood = [u for u in units
4783 if u["type"] == "file" and u["path"] == [u"good"]][0]
4784 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4785 ugoodcr = ugood["check-results"]
4786 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4789 self.failUnlessEqual(stats["type"], "stats")
4791 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4792 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4793 self.failUnlessReallyEqual(s["count-directories"], 1)
4794 self.failUnlessReallyEqual(s["count-unknown"], 1)
4795 d.addCallback(_done)
4797 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4798 def _check_manifest(res):
4799 self.failUnless(res.endswith("\n"))
4800 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4801 self.failUnlessReallyEqual(len(units), 5+1)
4802 self.failUnlessEqual(units[-1]["type"], "stats")
4804 self.failUnlessEqual(first["path"], [])
4805 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4806 self.failUnlessEqual(first["type"], "directory")
4807 stats = units[-1]["stats"]
4808 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4809 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4810 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4811 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4812 self.failUnlessReallyEqual(stats["count-unknown"], 1)
4813 d.addCallback(_check_manifest)
4815 # now add root/subdir and root/subdir/grandchild, then make subdir
4816 # unrecoverable, then see what happens
4818 d.addCallback(lambda ign:
4819 self.rootnode.create_subdirectory(u"subdir"))
4820 d.addCallback(_stash_uri, "subdir")
4821 d.addCallback(lambda subdir_node:
4822 subdir_node.add_file(u"grandchild",
4823 upload.Data(DATA+"2",
4825 d.addCallback(_stash_uri, "grandchild")
4827 d.addCallback(lambda ign:
4828 self.delete_shares_numbered(self.uris["subdir"],
4836 # root/subdir [unrecoverable]
4837 # root/subdir/grandchild
4839 # how should a streaming-JSON API indicate fatal error?
4840 # answer: emit ERROR: instead of a JSON string
4842 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4843 def _check_broken_manifest(res):
4844 lines = res.splitlines()
4846 for (i,line) in enumerate(lines)
4847 if line.startswith("ERROR:")]
4849 self.fail("no ERROR: in output: %s" % (res,))
4850 first_error = error_lines[0]
4851 error_line = lines[first_error]
4852 error_msg = lines[first_error+1:]
4853 error_msg_s = "\n".join(error_msg) + "\n"
4854 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4856 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4857 units = [simplejson.loads(line) for line in lines[:first_error]]
4858 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4859 last_unit = units[-1]
4860 self.failUnlessEqual(last_unit["path"], ["subdir"])
4861 d.addCallback(_check_broken_manifest)
4863 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4864 def _check_broken_deepcheck(res):
4865 lines = res.splitlines()
4867 for (i,line) in enumerate(lines)
4868 if line.startswith("ERROR:")]
4870 self.fail("no ERROR: in output: %s" % (res,))
4871 first_error = error_lines[0]
4872 error_line = lines[first_error]
4873 error_msg = lines[first_error+1:]
4874 error_msg_s = "\n".join(error_msg) + "\n"
4875 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4877 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4878 units = [simplejson.loads(line) for line in lines[:first_error]]
4879 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4880 last_unit = units[-1]
4881 self.failUnlessEqual(last_unit["path"], ["subdir"])
4882 r = last_unit["check-results"]["results"]
4883 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4884 self.failUnlessReallyEqual(r["count-shares-good"], 1)
4885 self.failUnlessReallyEqual(r["recoverable"], False)
4886 d.addCallback(_check_broken_deepcheck)
4888 d.addErrback(self.explain_web_error)
4891 def test_deep_check_and_repair(self):
4892 self.basedir = "web/Grid/deep_check_and_repair"
4894 c0 = self.g.clients[0]
4898 d = c0.create_dirnode()
4899 def _stash_root_and_create_file(n):
4901 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4902 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4903 d.addCallback(_stash_root_and_create_file)
4904 def _stash_uri(fn, which):
4905 self.uris[which] = fn.get_uri()
4906 d.addCallback(_stash_uri, "good")
4907 d.addCallback(lambda ign:
4908 self.rootnode.add_file(u"small",
4909 upload.Data("literal",
4911 d.addCallback(_stash_uri, "small")
4912 d.addCallback(lambda ign:
4913 self.rootnode.add_file(u"sick",
4914 upload.Data(DATA+"1",
4916 d.addCallback(_stash_uri, "sick")
4917 #d.addCallback(lambda ign:
4918 # self.rootnode.add_file(u"dead",
4919 # upload.Data(DATA+"2",
4921 #d.addCallback(_stash_uri, "dead")
4923 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4924 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4925 #d.addCallback(_stash_uri, "corrupt")
4927 def _clobber_shares(ignored):
4928 good_shares = self.find_uri_shares(self.uris["good"])
4929 self.failUnlessReallyEqual(len(good_shares), 10)
4930 sick_shares = self.find_uri_shares(self.uris["sick"])
4931 os.unlink(sick_shares[0][2])
4932 #dead_shares = self.find_uri_shares(self.uris["dead"])
4933 #for i in range(1, 10):
4934 # os.unlink(dead_shares[i][2])
4936 #c_shares = self.find_uri_shares(self.uris["corrupt"])
4937 #cso = CorruptShareOptions()
4938 #cso.stdout = StringIO()
4939 #cso.parseOptions([c_shares[0][2]])
4941 d.addCallback(_clobber_shares)
4944 # root/good CHK, 10 shares
4946 # root/sick CHK, 9 shares
4948 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4950 units = [simplejson.loads(line)
4951 for line in res.splitlines()
4953 self.failUnlessReallyEqual(len(units), 4+1)
4954 # should be parent-first
4956 self.failUnlessEqual(u0["path"], [])
4957 self.failUnlessEqual(u0["type"], "directory")
4958 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4959 u0crr = u0["check-and-repair-results"]
4960 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4961 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4963 ugood = [u for u in units
4964 if u["type"] == "file" and u["path"] == [u"good"]][0]
4965 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4966 ugoodcrr = ugood["check-and-repair-results"]
4967 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4968 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4970 usick = [u for u in units
4971 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4972 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4973 usickcrr = usick["check-and-repair-results"]
4974 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4975 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4976 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4977 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4980 self.failUnlessEqual(stats["type"], "stats")
4982 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4983 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4984 self.failUnlessReallyEqual(s["count-directories"], 1)
4985 d.addCallback(_done)
4987 d.addErrback(self.explain_web_error)
4990 def _count_leases(self, ignored, which):
4991 u = self.uris[which]
4992 shares = self.find_uri_shares(u)
4994 for shnum, serverid, fn in shares:
4995 sf = get_share_file(fn)
4996 num_leases = len(list(sf.get_leases()))
4997 lease_counts.append( (fn, num_leases) )
5000 def _assert_leasecount(self, lease_counts, expected):
5001 for (fn, num_leases) in lease_counts:
5002 if num_leases != expected:
5003 self.fail("expected %d leases, have %d, on %s" %
5004 (expected, num_leases, fn))
5006 def test_add_lease(self):
5007 self.basedir = "web/Grid/add_lease"
5008 self.set_up_grid(num_clients=2)
5009 c0 = self.g.clients[0]
5012 d = c0.upload(upload.Data(DATA, convergence=""))
5013 def _stash_uri(ur, which):
5014 self.uris[which] = ur.uri
5015 d.addCallback(_stash_uri, "one")
5016 d.addCallback(lambda ign:
5017 c0.upload(upload.Data(DATA+"1", convergence="")))
5018 d.addCallback(_stash_uri, "two")
5019 def _stash_mutable_uri(n, which):
5020 self.uris[which] = n.get_uri()
5021 assert isinstance(self.uris[which], str)
5022 d.addCallback(lambda ign:
5023 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5024 d.addCallback(_stash_mutable_uri, "mutable")
5026 def _compute_fileurls(ignored):
5028 for which in self.uris:
5029 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5030 d.addCallback(_compute_fileurls)
5032 d.addCallback(self._count_leases, "one")
5033 d.addCallback(self._assert_leasecount, 1)
5034 d.addCallback(self._count_leases, "two")
5035 d.addCallback(self._assert_leasecount, 1)
5036 d.addCallback(self._count_leases, "mutable")
5037 d.addCallback(self._assert_leasecount, 1)
5039 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5040 def _got_html_good(res):
5041 self.failUnlessIn("Healthy", res)
5042 self.failIfIn("Not Healthy", res)
5043 d.addCallback(_got_html_good)
5045 d.addCallback(self._count_leases, "one")
5046 d.addCallback(self._assert_leasecount, 1)
5047 d.addCallback(self._count_leases, "two")
5048 d.addCallback(self._assert_leasecount, 1)
5049 d.addCallback(self._count_leases, "mutable")
5050 d.addCallback(self._assert_leasecount, 1)
5052 # this CHECK uses the original client, which uses the same
5053 # lease-secrets, so it will just renew the original lease
5054 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5055 d.addCallback(_got_html_good)
5057 d.addCallback(self._count_leases, "one")
5058 d.addCallback(self._assert_leasecount, 1)
5059 d.addCallback(self._count_leases, "two")
5060 d.addCallback(self._assert_leasecount, 1)
5061 d.addCallback(self._count_leases, "mutable")
5062 d.addCallback(self._assert_leasecount, 1)
5064 # this CHECK uses an alternate client, which adds a second lease
5065 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5066 d.addCallback(_got_html_good)
5068 d.addCallback(self._count_leases, "one")
5069 d.addCallback(self._assert_leasecount, 2)
5070 d.addCallback(self._count_leases, "two")
5071 d.addCallback(self._assert_leasecount, 1)
5072 d.addCallback(self._count_leases, "mutable")
5073 d.addCallback(self._assert_leasecount, 1)
5075 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5076 d.addCallback(_got_html_good)
5078 d.addCallback(self._count_leases, "one")
5079 d.addCallback(self._assert_leasecount, 2)
5080 d.addCallback(self._count_leases, "two")
5081 d.addCallback(self._assert_leasecount, 1)
5082 d.addCallback(self._count_leases, "mutable")
5083 d.addCallback(self._assert_leasecount, 1)
5085 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5087 d.addCallback(_got_html_good)
5089 d.addCallback(self._count_leases, "one")
5090 d.addCallback(self._assert_leasecount, 2)
5091 d.addCallback(self._count_leases, "two")
5092 d.addCallback(self._assert_leasecount, 1)
5093 d.addCallback(self._count_leases, "mutable")
5094 d.addCallback(self._assert_leasecount, 2)
5096 d.addErrback(self.explain_web_error)
5099 def test_deep_add_lease(self):
5100 self.basedir = "web/Grid/deep_add_lease"
5101 self.set_up_grid(num_clients=2)
5102 c0 = self.g.clients[0]
5106 d = c0.create_dirnode()
5107 def _stash_root_and_create_file(n):
5109 self.uris["root"] = n.get_uri()
5110 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5111 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5112 d.addCallback(_stash_root_and_create_file)
5113 def _stash_uri(fn, which):
5114 self.uris[which] = fn.get_uri()
5115 d.addCallback(_stash_uri, "one")
5116 d.addCallback(lambda ign:
5117 self.rootnode.add_file(u"small",
5118 upload.Data("literal",
5120 d.addCallback(_stash_uri, "small")
5122 d.addCallback(lambda ign:
5123 c0.create_mutable_file(publish.MutableData("mutable")))
5124 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5125 d.addCallback(_stash_uri, "mutable")
5127 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5129 units = [simplejson.loads(line)
5130 for line in res.splitlines()
5132 # root, one, small, mutable, stats
5133 self.failUnlessReallyEqual(len(units), 4+1)
5134 d.addCallback(_done)
5136 d.addCallback(self._count_leases, "root")
5137 d.addCallback(self._assert_leasecount, 1)
5138 d.addCallback(self._count_leases, "one")
5139 d.addCallback(self._assert_leasecount, 1)
5140 d.addCallback(self._count_leases, "mutable")
5141 d.addCallback(self._assert_leasecount, 1)
5143 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5144 d.addCallback(_done)
5146 d.addCallback(self._count_leases, "root")
5147 d.addCallback(self._assert_leasecount, 1)
5148 d.addCallback(self._count_leases, "one")
5149 d.addCallback(self._assert_leasecount, 1)
5150 d.addCallback(self._count_leases, "mutable")
5151 d.addCallback(self._assert_leasecount, 1)
5153 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5155 d.addCallback(_done)
5157 d.addCallback(self._count_leases, "root")
5158 d.addCallback(self._assert_leasecount, 2)
5159 d.addCallback(self._count_leases, "one")
5160 d.addCallback(self._assert_leasecount, 2)
5161 d.addCallback(self._count_leases, "mutable")
5162 d.addCallback(self._assert_leasecount, 2)
5164 d.addErrback(self.explain_web_error)
5168 def test_exceptions(self):
5169 self.basedir = "web/Grid/exceptions"
5170 self.set_up_grid(num_clients=1, num_servers=2)
5171 c0 = self.g.clients[0]
5172 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5175 d = c0.create_dirnode()
5177 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5178 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5180 d.addCallback(_stash_root)
5181 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5183 self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5184 self.delete_shares_numbered(ur.uri, range(1,10))
5186 u = uri.from_string(ur.uri)
5187 u.key = testutil.flip_bit(u.key, 0)
5188 baduri = u.to_string()
5189 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5190 d.addCallback(_stash_bad)
5191 d.addCallback(lambda ign: c0.create_dirnode())
5192 def _mangle_dirnode_1share(n):
5194 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5195 self.fileurls["dir-1share-json"] = url + "?t=json"
5196 self.delete_shares_numbered(u, range(1,10))
5197 d.addCallback(_mangle_dirnode_1share)
5198 d.addCallback(lambda ign: c0.create_dirnode())
5199 def _mangle_dirnode_0share(n):
5201 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5202 self.fileurls["dir-0share-json"] = url + "?t=json"
5203 self.delete_shares_numbered(u, range(0,10))
5204 d.addCallback(_mangle_dirnode_0share)
5206 # NotEnoughSharesError should be reported sensibly, with a
5207 # text/plain explanation of the problem, and perhaps some
5208 # information on which shares *could* be found.
5210 d.addCallback(lambda ignored:
5211 self.shouldHTTPError("GET unrecoverable",
5212 410, "Gone", "NoSharesError",
5213 self.GET, self.fileurls["0shares"]))
5214 def _check_zero_shares(body):
5215 self.failIfIn("<html>", body)
5216 body = " ".join(body.strip().split())
5217 exp = ("NoSharesError: no shares could be found. "
5218 "Zero shares usually indicates a corrupt URI, or that "
5219 "no servers were connected, but it might also indicate "
5220 "severe corruption. You should perform a filecheck on "
5221 "this object to learn more. The full error message is: "
5222 "no shares (need 3). Last failure: None")
5223 self.failUnlessReallyEqual(exp, body)
5224 d.addCallback(_check_zero_shares)
5227 d.addCallback(lambda ignored:
5228 self.shouldHTTPError("GET 1share",
5229 410, "Gone", "NotEnoughSharesError",
5230 self.GET, self.fileurls["1share"]))
5231 def _check_one_share(body):
5232 self.failIfIn("<html>", body)
5233 body = " ".join(body.strip().split())
5234 msgbase = ("NotEnoughSharesError: This indicates that some "
5235 "servers were unavailable, or that shares have been "
5236 "lost to server departure, hard drive failure, or disk "
5237 "corruption. You should perform a filecheck on "
5238 "this object to learn more. The full error message is:"
5240 msg1 = msgbase + (" ran out of shares:"
5243 " overdue= unused= need 3. Last failure: None")
5244 msg2 = msgbase + (" ran out of shares:"
5246 " pending=Share(sh0-on-xgru5)"
5247 " overdue= unused= need 3. Last failure: None")
5248 self.failUnless(body == msg1 or body == msg2, body)
5249 d.addCallback(_check_one_share)
5251 d.addCallback(lambda ignored:
5252 self.shouldHTTPError("GET imaginary",
5253 404, "Not Found", None,
5254 self.GET, self.fileurls["imaginary"]))
5255 def _missing_child(body):
5256 self.failUnlessIn("No such child: imaginary", body)
5257 d.addCallback(_missing_child)
5259 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5260 def _check_0shares_dir_html(body):
5261 self.failUnlessIn("<html>", body)
5262 # we should see the regular page, but without the child table or
5264 body = " ".join(body.strip().split())
5265 self.failUnlessIn('href="?t=info">More info on this directory',
5267 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5268 "could not be retrieved, because there were insufficient "
5269 "good shares. This might indicate that no servers were "
5270 "connected, insufficient servers were connected, the URI "
5271 "was corrupt, or that shares have been lost due to server "
5272 "departure, hard drive failure, or disk corruption. You "
5273 "should perform a filecheck on this object to learn more.")
5274 self.failUnlessIn(exp, body)
5275 self.failUnlessIn("No upload forms: directory is unreadable", body)
5276 d.addCallback(_check_0shares_dir_html)
5278 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5279 def _check_1shares_dir_html(body):
5280 # at some point, we'll split UnrecoverableFileError into 0-shares
5281 # and some-shares like we did for immutable files (since there
5282 # are different sorts of advice to offer in each case). For now,
5283 # they present the same way.
5284 self.failUnlessIn("<html>", body)
5285 body = " ".join(body.strip().split())
5286 self.failUnlessIn('href="?t=info">More info on this directory',
5288 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5289 "could not be retrieved, because there were insufficient "
5290 "good shares. This might indicate that no servers were "
5291 "connected, insufficient servers were connected, the URI "
5292 "was corrupt, or that shares have been lost due to server "
5293 "departure, hard drive failure, or disk corruption. You "
5294 "should perform a filecheck on this object to learn more.")
5295 self.failUnlessIn(exp, body)
5296 self.failUnlessIn("No upload forms: directory is unreadable", body)
5297 d.addCallback(_check_1shares_dir_html)
5299 d.addCallback(lambda ignored:
5300 self.shouldHTTPError("GET dir-0share-json",
5301 410, "Gone", "UnrecoverableFileError",
5303 self.fileurls["dir-0share-json"]))
5304 def _check_unrecoverable_file(body):
5305 self.failIfIn("<html>", body)
5306 body = " ".join(body.strip().split())
5307 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5308 "could not be retrieved, because there were insufficient "
5309 "good shares. This might indicate that no servers were "
5310 "connected, insufficient servers were connected, the URI "
5311 "was corrupt, or that shares have been lost due to server "
5312 "departure, hard drive failure, or disk corruption. You "
5313 "should perform a filecheck on this object to learn more.")
5314 self.failUnlessReallyEqual(exp, body)
5315 d.addCallback(_check_unrecoverable_file)
5317 d.addCallback(lambda ignored:
5318 self.shouldHTTPError("GET dir-1share-json",
5319 410, "Gone", "UnrecoverableFileError",
5321 self.fileurls["dir-1share-json"]))
5322 d.addCallback(_check_unrecoverable_file)
5324 d.addCallback(lambda ignored:
5325 self.shouldHTTPError("GET imaginary",
5326 404, "Not Found", None,
5327 self.GET, self.fileurls["imaginary"]))
5329 # attach a webapi child that throws a random error, to test how it
5331 w = c0.getServiceNamed("webish")
5332 w.root.putChild("ERRORBOOM", ErrorBoom())
5334 # "Accept: */*" : should get a text/html stack trace
5335 # "Accept: text/plain" : should get a text/plain stack trace
5336 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5337 # no Accept header: should get a text/html stack trace
5339 d.addCallback(lambda ignored:
5340 self.shouldHTTPError("GET errorboom_html",
5341 500, "Internal Server Error", None,
5342 self.GET, "ERRORBOOM",
5343 headers={"accept": "*/*"}))
5344 def _internal_error_html1(body):
5345 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5346 d.addCallback(_internal_error_html1)
5348 d.addCallback(lambda ignored:
5349 self.shouldHTTPError("GET errorboom_text",
5350 500, "Internal Server Error", None,
5351 self.GET, "ERRORBOOM",
5352 headers={"accept": "text/plain"}))
5353 def _internal_error_text2(body):
5354 self.failIfIn("<html>", body)
5355 self.failUnless(body.startswith("Traceback "), body)
5356 d.addCallback(_internal_error_text2)
5358 CLI_accepts = "text/plain, application/octet-stream"
5359 d.addCallback(lambda ignored:
5360 self.shouldHTTPError("GET errorboom_text",
5361 500, "Internal Server Error", None,
5362 self.GET, "ERRORBOOM",
5363 headers={"accept": CLI_accepts}))
5364 def _internal_error_text3(body):
5365 self.failIfIn("<html>", body)
5366 self.failUnless(body.startswith("Traceback "), body)
5367 d.addCallback(_internal_error_text3)
5369 d.addCallback(lambda ignored:
5370 self.shouldHTTPError("GET errorboom_text",
5371 500, "Internal Server Error", None,
5372 self.GET, "ERRORBOOM"))
5373 def _internal_error_html4(body):
5374 self.failUnlessIn("<html>", body)
5375 d.addCallback(_internal_error_html4)
5377 def _flush_errors(res):
5378 # Trial: please ignore the CompletelyUnhandledError in the logs
5379 self.flushLoggedErrors(CompletelyUnhandledError)
5381 d.addBoth(_flush_errors)
5385 def test_blacklist(self):
5386 # download from a blacklisted URI, get an error
5387 self.basedir = "web/Grid/blacklist"
5389 c0 = self.g.clients[0]
5390 c0_basedir = c0.basedir
5391 fn = os.path.join(c0_basedir, "access.blacklist")
5393 DATA = "off-limits " * 50
5395 d = c0.upload(upload.Data(DATA, convergence=""))
5396 def _stash_uri_and_create_dir(ur):
5398 self.url = "uri/"+self.uri
5399 u = uri.from_string_filenode(self.uri)
5400 self.si = u.get_storage_index()
5401 childnode = c0.create_node_from_uri(self.uri, None)
5402 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5403 d.addCallback(_stash_uri_and_create_dir)
5404 def _stash_dir(node):
5405 self.dir_node = node
5406 self.dir_uri = node.get_uri()
5407 self.dir_url = "uri/"+self.dir_uri
5408 d.addCallback(_stash_dir)
5409 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5410 def _check_dir_html(body):
5411 self.failUnlessIn("<html>", body)
5412 self.failUnlessIn("blacklisted.txt</a>", body)
5413 d.addCallback(_check_dir_html)
5414 d.addCallback(lambda ign: self.GET(self.url))
5415 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5417 def _blacklist(ign):
5419 f.write(" # this is a comment\n")
5421 f.write("\n") # also exercise blank lines
5422 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5424 # clients should be checking the blacklist each time, so we don't
5425 # need to restart the client
5426 d.addCallback(_blacklist)
5427 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5429 "Access Prohibited: off-limits",
5430 self.GET, self.url))
5432 # We should still be able to list the parent directory, in HTML...
5433 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5434 def _check_dir_html2(body):
5435 self.failUnlessIn("<html>", body)
5436 self.failUnlessIn("blacklisted.txt</strike>", body)
5437 d.addCallback(_check_dir_html2)
5439 # ... and in JSON (used by CLI).
5440 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5441 def _check_dir_json(res):
5442 data = simplejson.loads(res)
5443 self.failUnless(isinstance(data, list), data)
5444 self.failUnlessEqual(data[0], "dirnode")
5445 self.failUnless(isinstance(data[1], dict), data)
5446 self.failUnlessIn("children", data[1])
5447 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5448 childdata = data[1]["children"]["blacklisted.txt"]
5449 self.failUnless(isinstance(childdata, list), data)
5450 self.failUnlessEqual(childdata[0], "filenode")
5451 self.failUnless(isinstance(childdata[1], dict), data)
5452 d.addCallback(_check_dir_json)
5454 def _unblacklist(ign):
5455 open(fn, "w").close()
5456 # the Blacklist object watches mtime to tell when the file has
5457 # changed, but on windows this test will run faster than the
5458 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5459 # to force a reload.
5460 self.g.clients[0].blacklist.last_mtime -= 2.0
5461 d.addCallback(_unblacklist)
5463 # now a read should work
5464 d.addCallback(lambda ign: self.GET(self.url))
5465 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5467 # read again to exercise the blacklist-is-unchanged logic
5468 d.addCallback(lambda ign: self.GET(self.url))
5469 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5471 # now add a blacklisted directory, and make sure files under it are
5474 childnode = c0.create_node_from_uri(self.uri, None)
5475 return c0.create_dirnode({u"child": (childnode,{}) })
5476 d.addCallback(_add_dir)
5477 def _get_dircap(dn):
5478 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5479 self.dir_url_base = "uri/"+dn.get_write_uri()
5480 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5481 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5482 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5483 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5484 d.addCallback(_get_dircap)
5485 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5486 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5487 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5488 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5489 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5490 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5491 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5492 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5493 d.addCallback(lambda ign: self.GET(self.child_url))
5494 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5496 def _block_dir(ign):
5498 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5500 self.g.clients[0].blacklist.last_mtime -= 2.0
5501 d.addCallback(_block_dir)
5502 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5504 "Access Prohibited: dir-off-limits",
5505 self.GET, self.dir_url_base))
5506 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5508 "Access Prohibited: dir-off-limits",
5509 self.GET, self.dir_url_json1))
5510 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5512 "Access Prohibited: dir-off-limits",
5513 self.GET, self.dir_url_json2))
5514 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5516 "Access Prohibited: dir-off-limits",
5517 self.GET, self.dir_url_json_ro))
5518 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5520 "Access Prohibited: dir-off-limits",
5521 self.GET, self.child_url))
5525 class CompletelyUnhandledError(Exception):
5527 class ErrorBoom(rend.Page):
5528 def beforeRender(self, ctx):
5529 raise CompletelyUnhandledError("whoops")