1 import os.path, re, urllib, time
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow import rend
16 from allmydata import interfaces, uri, webish, dirnode
17 from allmydata.storage.shares import get_share_file
18 from allmydata.storage_client import StorageFarmBroker
19 from allmydata.immutable import upload
20 from allmydata.immutable.downloader.status import DownloadStatus
21 from allmydata.dirnode import DirectoryNode
22 from allmydata.nodemaker import NodeMaker
23 from allmydata.unknown import UnknownNode
24 from allmydata.web import status, common
25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
26 from allmydata.util import fileutil, base32, hashutil
27 from allmydata.util.consumer import download_to_data
28 from allmydata.util.netstring import split_netstring
29 from allmydata.util.encodingutil import to_str
30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
31 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
32 make_mutable_file_uri, create_mutable_filenode
33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
34 from allmydata.mutable import servermap, publish, retrieve
35 import allmydata.test.common_util as testutil
36 from allmydata.test.no_network import GridTestMixin
37 from allmydata.test.common_web import HTTPClientGETFactory, \
39 from allmydata.client import Client, SecretHolder
40 from allmydata.introducer import IntroducerNode
42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
43 # create a webserver that works against them
45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
54 class FakeStatsProvider:
56 stats = {'stats': {}, 'counters': {}}
59 class FakeNodeMaker(NodeMaker):
64 'max_segment_size':128*1024 # 1024=KiB
66 def _create_lit(self, cap):
67 return FakeCHKFileNode(cap)
68 def _create_immutable(self, cap):
69 return FakeCHKFileNode(cap)
70 def _create_mutable(self, cap):
71 return FakeMutableFileNode(None,
73 self.encoding_params, None).init_from_cap(cap)
74 def create_mutable_file(self, contents="", keysize=None,
75 version=SDMF_VERSION):
76 n = FakeMutableFileNode(None, None, self.encoding_params, None)
77 return n.create(contents, version=version)
79 class FakeUploader(service.Service):
81 def upload(self, uploadable):
82 d = uploadable.get_size()
83 d.addCallback(lambda size: uploadable.read(size))
86 n = create_chk_filenode(data)
87 results = upload.UploadResults()
88 results.uri = n.get_uri()
90 d.addCallback(_got_data)
92 def get_helper_info(self):
96 def __init__(self, binaryserverid):
97 self.binaryserverid = binaryserverid
98 def get_name(self): return "short"
99 def get_longname(self): return "long"
100 def get_serverid(self): return self.binaryserverid
103 ds = DownloadStatus("storage_index", 1234)
106 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
107 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
108 storage_index = hashutil.storage_index_hash("SI")
109 e0 = ds.add_segment_request(0, now)
111 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
112 e1 = ds.add_segment_request(1, now+2)
114 # two outstanding requests
115 e2 = ds.add_segment_request(2, now+4)
116 e3 = ds.add_segment_request(3, now+5)
117 del e2,e3 # hush pyflakes
119 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
120 e = ds.add_segment_request(4, now)
122 e.deliver(now, 0, 140, 0.5)
124 e = ds.add_dyhb_request(serverA, now)
125 e.finished([1,2], now+1)
126 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
128 e = ds.add_read_event(0, 120, now)
129 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
131 e = ds.add_read_event(120, 30, now+2) # left unfinished
133 e = ds.add_block_request(serverA, 1, 100, 20, now)
134 e.finished(20, now+1)
135 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
137 # make sure that add_read_event() can come first too
138 ds1 = DownloadStatus(storage_index, 1234)
139 e = ds1.add_read_event(0, 120, now)
140 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
146 _all_upload_status = [upload.UploadStatus()]
147 _all_download_status = [build_one_ds()]
148 _all_mapupdate_statuses = [servermap.UpdateStatus()]
149 _all_publish_statuses = [publish.PublishStatus()]
150 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
152 def list_all_upload_statuses(self):
153 return self._all_upload_status
154 def list_all_download_statuses(self):
155 return self._all_download_status
156 def list_all_mapupdate_statuses(self):
157 return self._all_mapupdate_statuses
158 def list_all_publish_statuses(self):
159 return self._all_publish_statuses
160 def list_all_retrieve_statuses(self):
161 return self._all_retrieve_statuses
162 def list_all_helper_statuses(self):
165 class FakeClient(Client):
167 # don't upcall to Client.__init__, since we only want to initialize a
169 service.MultiService.__init__(self)
170 self.nodeid = "fake_nodeid"
171 self.nickname = "fake_nickname"
172 self.introducer_furl = "None"
173 self.stats_provider = FakeStatsProvider()
174 self._secret_holder = SecretHolder("lease secret", "convergence secret")
176 self.convergence = "some random string"
177 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
178 self.introducer_client = None
179 self.history = FakeHistory()
180 self.uploader = FakeUploader()
181 self.uploader.setServiceParent(self)
182 self.blacklist = None
183 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
186 self.mutable_file_default = SDMF_VERSION
188 def startService(self):
189 return service.MultiService.startService(self)
190 def stopService(self):
191 return service.MultiService.stopService(self)
193 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
195 class WebMixin(object):
197 self.s = FakeClient()
198 self.s.startService()
199 self.staticdir = self.mktemp()
201 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
203 self.ws.setServiceParent(self.s)
204 self.webish_port = self.ws.getPortnum()
205 self.webish_url = self.ws.getURL()
206 assert self.webish_url.endswith("/")
207 self.webish_url = self.webish_url[:-1] # these tests add their own /
209 l = [ self.s.create_dirnode() for x in range(6) ]
210 d = defer.DeferredList(l)
212 self.public_root = res[0][1]
213 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
214 self.public_url = "/uri/" + self.public_root.get_uri()
215 self.private_root = res[1][1]
219 self._foo_uri = foo.get_uri()
220 self._foo_readonly_uri = foo.get_readonly_uri()
221 self._foo_verifycap = foo.get_verify_cap().to_string()
222 # NOTE: we ignore the deferred on all set_uri() calls, because we
223 # know the fake nodes do these synchronously
224 self.public_root.set_uri(u"foo", foo.get_uri(),
225 foo.get_readonly_uri())
227 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
228 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
229 self._bar_txt_verifycap = n.get_verify_cap().to_string()
232 # XXX: Do we ever use this?
233 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
235 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
238 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
239 assert self._quux_txt_uri.startswith("URI:MDMF")
240 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
242 foo.set_uri(u"empty", res[3][1].get_uri(),
243 res[3][1].get_readonly_uri())
244 sub_uri = res[4][1].get_uri()
245 self._sub_uri = sub_uri
246 foo.set_uri(u"sub", sub_uri, sub_uri)
247 sub = self.s.create_node_from_uri(sub_uri)
250 _ign, n, blocking_uri = self.makefile(1)
251 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
253 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
254 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
255 # still think of it as an umlaut
256 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
258 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
259 self._baz_file_uri = baz_file
260 sub.set_uri(u"baz.txt", baz_file, baz_file)
262 _ign, n, self._bad_file_uri = self.makefile(3)
263 # this uri should not be downloadable
264 del FakeCHKFileNode.all_contents[self._bad_file_uri]
267 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
268 rodir.get_readonly_uri())
269 rodir.set_uri(u"nor", baz_file, baz_file)
275 # public/foo/quux.txt
276 # public/foo/blockingfile
279 # public/foo/sub/baz.txt
281 # public/reedownlee/nor
282 self.NEWFILE_CONTENTS = "newfile contents\n"
284 return foo.get_metadata_for(u"bar.txt")
286 def _got_metadata(metadata):
287 self._bar_txt_metadata = metadata
288 d.addCallback(_got_metadata)
291 def makefile(self, number):
292 contents = "contents of file %s\n" % number
293 n = create_chk_filenode(contents)
294 return contents, n, n.get_uri()
296 def makefile_mutable(self, number, mdmf=False):
297 contents = "contents of mutable file %s\n" % number
298 n = create_mutable_filenode(contents, mdmf)
299 return contents, n, n.get_uri(), n.get_readonly_uri()
302 return self.s.stopService()
304 def failUnlessIsBarDotTxt(self, res):
305 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
307 def failUnlessIsQuuxDotTxt(self, res):
308 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
310 def failUnlessIsBazDotTxt(self, res):
311 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
313 def failUnlessIsSubBazDotTxt(self, res):
314 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
316 def failUnlessIsBarJSON(self, res):
317 data = simplejson.loads(res)
318 self.failUnless(isinstance(data, list))
319 self.failUnlessEqual(data[0], "filenode")
320 self.failUnless(isinstance(data[1], dict))
321 self.failIf(data[1]["mutable"])
322 self.failIfIn("rw_uri", data[1]) # immutable
323 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
324 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
325 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
327 def failUnlessIsQuuxJSON(self, res, readonly=False):
328 data = simplejson.loads(res)
329 self.failUnless(isinstance(data, list))
330 self.failUnlessEqual(data[0], "filenode")
331 self.failUnless(isinstance(data[1], dict))
333 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
335 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
336 self.failUnless(metadata['mutable'])
338 self.failIfIn("rw_uri", metadata)
340 self.failUnlessIn("rw_uri", metadata)
341 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
342 self.failUnlessIn("ro_uri", metadata)
343 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
344 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
346 def failUnlessIsFooJSON(self, res):
347 data = simplejson.loads(res)
348 self.failUnless(isinstance(data, list))
349 self.failUnlessEqual(data[0], "dirnode", res)
350 self.failUnless(isinstance(data[1], dict))
351 self.failUnless(data[1]["mutable"])
352 self.failUnlessIn("rw_uri", data[1]) # mutable
353 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
354 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
355 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
357 kidnames = sorted([unicode(n) for n in data[1]["children"]])
358 self.failUnlessEqual(kidnames,
359 [u"bar.txt", u"baz.txt", u"blockingfile",
360 u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
361 kids = dict( [(unicode(name),value)
363 in data[1]["children"].iteritems()] )
364 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
365 self.failUnlessIn("metadata", kids[u"sub"][1])
366 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
367 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
368 self.failUnlessIn("linkcrtime", tahoe_md)
369 self.failUnlessIn("linkmotime", tahoe_md)
370 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
371 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
372 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
373 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
374 self._bar_txt_verifycap)
375 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
376 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
377 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
378 self._bar_txt_metadata["tahoe"]["linkcrtime"])
379 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
381 self.failUnlessIn("quux.txt", kids)
382 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
384 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
385 self._quux_txt_readonly_uri)
387 def GET(self, urlpath, followRedirect=False, return_response=False,
389 # if return_response=True, this fires with (data, statuscode,
390 # respheaders) instead of just data.
391 assert not isinstance(urlpath, unicode)
392 url = self.webish_url + urlpath
393 factory = HTTPClientGETFactory(url, method="GET",
394 followRedirect=followRedirect, **kwargs)
395 reactor.connectTCP("localhost", self.webish_port, factory)
398 return (data, factory.status, factory.response_headers)
400 d.addCallback(_got_data)
401 return factory.deferred
403 def HEAD(self, urlpath, return_response=False, **kwargs):
404 # this requires some surgery, because twisted.web.client doesn't want
405 # to give us back the response headers.
406 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
407 reactor.connectTCP("localhost", self.webish_port, factory)
410 return (data, factory.status, factory.response_headers)
412 d.addCallback(_got_data)
413 return factory.deferred
415 def PUT(self, urlpath, data, **kwargs):
416 url = self.webish_url + urlpath
417 return client.getPage(url, method="PUT", postdata=data, **kwargs)
419 def DELETE(self, urlpath):
420 url = self.webish_url + urlpath
421 return client.getPage(url, method="DELETE")
423 def POST(self, urlpath, followRedirect=False, **fields):
424 sepbase = "boogabooga"
428 form.append('Content-Disposition: form-data; name="_charset"')
432 for name, value in fields.iteritems():
433 if isinstance(value, tuple):
434 filename, value = value
435 form.append('Content-Disposition: form-data; name="%s"; '
436 'filename="%s"' % (name, filename.encode("utf-8")))
438 form.append('Content-Disposition: form-data; name="%s"' % name)
440 if isinstance(value, unicode):
441 value = value.encode("utf-8")
444 assert isinstance(value, str)
451 body = "\r\n".join(form) + "\r\n"
452 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
453 return self.POST2(urlpath, body, headers, followRedirect)
455 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
456 url = self.webish_url + urlpath
457 return client.getPage(url, method="POST", postdata=body,
458 headers=headers, followRedirect=followRedirect)
460 def shouldFail(self, res, expected_failure, which,
461 substring=None, response_substring=None):
462 if isinstance(res, failure.Failure):
463 res.trap(expected_failure)
465 self.failUnlessIn(substring, str(res), which)
466 if response_substring:
467 self.failUnlessIn(response_substring, res.value.response, which)
469 self.fail("%s was supposed to raise %s, not get '%s'" %
470 (which, expected_failure, res))
472 def shouldFail2(self, expected_failure, which, substring,
474 callable, *args, **kwargs):
475 assert substring is None or isinstance(substring, str)
476 assert response_substring is None or isinstance(response_substring, str)
477 d = defer.maybeDeferred(callable, *args, **kwargs)
479 if isinstance(res, failure.Failure):
480 res.trap(expected_failure)
482 self.failUnlessIn(substring, str(res),
483 "'%s' not in '%s' for test '%s'" % \
484 (substring, str(res), which))
485 if response_substring:
486 self.failUnlessIn(response_substring, res.value.response,
487 "'%s' not in '%s' for test '%s'" % \
488 (response_substring, res.value.response,
491 self.fail("%s was supposed to raise %s, not get '%s'" %
492 (which, expected_failure, res))
496 def should404(self, res, which):
497 if isinstance(res, failure.Failure):
498 res.trap(error.Error)
499 self.failUnlessReallyEqual(res.value.status, "404")
501 self.fail("%s was supposed to Error(404), not get '%s'" %
504 def should302(self, res, which):
505 if isinstance(res, failure.Failure):
506 res.trap(error.Error)
507 self.failUnlessReallyEqual(res.value.status, "302")
509 self.fail("%s was supposed to Error(302), not get '%s'" %
513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
514 def test_create(self):
517 def test_welcome(self):
520 self.failUnlessIn('Welcome to Tahoe-LAFS', res)
521 self.failUnlessIn(FAVICON_MARKUP, res)
522 self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
524 self.s.basedir = 'web/test_welcome'
525 fileutil.make_dirs("web/test_welcome")
526 fileutil.make_dirs("web/test_welcome/private")
528 d.addCallback(_check)
531 def test_status(self):
532 h = self.s.get_history()
533 dl_num = h.list_all_download_statuses()[0].get_counter()
534 ul_num = h.list_all_upload_statuses()[0].get_counter()
535 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
536 pub_num = h.list_all_publish_statuses()[0].get_counter()
537 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
538 d = self.GET("/status", followRedirect=True)
540 self.failUnlessIn('Upload and Download Status', res)
541 self.failUnlessIn('"down-%d"' % dl_num, res)
542 self.failUnlessIn('"up-%d"' % ul_num, res)
543 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
544 self.failUnlessIn('"publish-%d"' % pub_num, res)
545 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
546 d.addCallback(_check)
547 d.addCallback(lambda res: self.GET("/status/?t=json"))
548 def _check_json(res):
549 data = simplejson.loads(res)
550 self.failUnless(isinstance(data, dict))
551 #active = data["active"]
552 # TODO: test more. We need a way to fake an active operation
554 d.addCallback(_check_json)
556 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
558 self.failUnlessIn("File Download Status", res)
559 d.addCallback(_check_dl)
560 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
561 def _check_dl_json(res):
562 data = simplejson.loads(res)
563 self.failUnless(isinstance(data, dict))
564 self.failUnlessIn("read", data)
565 self.failUnlessEqual(data["read"][0]["length"], 120)
566 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
567 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
568 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
569 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
570 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
571 # serverids[] keys are strings, since that's what JSON does, but
572 # we'd really like them to be ints
573 self.failUnlessEqual(data["serverids"]["0"], "phwr")
574 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
575 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
576 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
577 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
578 self.failUnlessIn("dyhb", data)
579 self.failUnlessIn("misc", data)
580 d.addCallback(_check_dl_json)
581 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
583 self.failUnlessIn("File Upload Status", res)
584 d.addCallback(_check_ul)
585 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
586 def _check_mapupdate(res):
587 self.failUnlessIn("Mutable File Servermap Update Status", res)
588 d.addCallback(_check_mapupdate)
589 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
590 def _check_publish(res):
591 self.failUnlessIn("Mutable File Publish Status", res)
592 d.addCallback(_check_publish)
593 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
594 def _check_retrieve(res):
595 self.failUnlessIn("Mutable File Retrieve Status", res)
596 d.addCallback(_check_retrieve)
600 def test_status_numbers(self):
601 drrm = status.DownloadResultsRendererMixin()
602 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
603 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
604 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
605 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
606 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
607 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
608 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
609 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
610 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
612 urrm = status.UploadResultsRendererMixin()
613 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
614 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
615 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
616 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
617 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
618 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
619 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
620 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
621 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
623 def test_GET_FILEURL(self):
624 d = self.GET(self.public_url + "/foo/bar.txt")
625 d.addCallback(self.failUnlessIsBarDotTxt)
628 def test_GET_FILEURL_range(self):
629 headers = {"range": "bytes=1-10"}
630 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
631 return_response=True)
632 def _got((res, status, headers)):
633 self.failUnlessReallyEqual(int(status), 206)
634 self.failUnless(headers.has_key("content-range"))
635 self.failUnlessReallyEqual(headers["content-range"][0],
636 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
637 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
641 def test_GET_FILEURL_partial_range(self):
642 headers = {"range": "bytes=5-"}
643 length = len(self.BAR_CONTENTS)
644 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
645 return_response=True)
646 def _got((res, status, headers)):
647 self.failUnlessReallyEqual(int(status), 206)
648 self.failUnless(headers.has_key("content-range"))
649 self.failUnlessReallyEqual(headers["content-range"][0],
650 "bytes 5-%d/%d" % (length-1, length))
651 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
655 def test_GET_FILEURL_partial_end_range(self):
656 headers = {"range": "bytes=-5"}
657 length = len(self.BAR_CONTENTS)
658 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
659 return_response=True)
660 def _got((res, status, headers)):
661 self.failUnlessReallyEqual(int(status), 206)
662 self.failUnless(headers.has_key("content-range"))
663 self.failUnlessReallyEqual(headers["content-range"][0],
664 "bytes %d-%d/%d" % (length-5, length-1, length))
665 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
669 def test_GET_FILEURL_partial_range_overrun(self):
670 headers = {"range": "bytes=100-200"}
671 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
672 "416 Requested Range not satisfiable",
673 "First beyond end of file",
674 self.GET, self.public_url + "/foo/bar.txt",
678 def test_HEAD_FILEURL_range(self):
679 headers = {"range": "bytes=1-10"}
680 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
681 return_response=True)
682 def _got((res, status, headers)):
683 self.failUnlessReallyEqual(res, "")
684 self.failUnlessReallyEqual(int(status), 206)
685 self.failUnless(headers.has_key("content-range"))
686 self.failUnlessReallyEqual(headers["content-range"][0],
687 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
691 def test_HEAD_FILEURL_partial_range(self):
692 headers = {"range": "bytes=5-"}
693 length = len(self.BAR_CONTENTS)
694 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
695 return_response=True)
696 def _got((res, status, headers)):
697 self.failUnlessReallyEqual(int(status), 206)
698 self.failUnless(headers.has_key("content-range"))
699 self.failUnlessReallyEqual(headers["content-range"][0],
700 "bytes 5-%d/%d" % (length-1, length))
704 def test_HEAD_FILEURL_partial_end_range(self):
705 headers = {"range": "bytes=-5"}
706 length = len(self.BAR_CONTENTS)
707 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
708 return_response=True)
709 def _got((res, status, headers)):
710 self.failUnlessReallyEqual(int(status), 206)
711 self.failUnless(headers.has_key("content-range"))
712 self.failUnlessReallyEqual(headers["content-range"][0],
713 "bytes %d-%d/%d" % (length-5, length-1, length))
717 def test_HEAD_FILEURL_partial_range_overrun(self):
718 headers = {"range": "bytes=100-200"}
719 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
720 "416 Requested Range not satisfiable",
722 self.HEAD, self.public_url + "/foo/bar.txt",
726 def test_GET_FILEURL_range_bad(self):
727 headers = {"range": "BOGUS=fizbop-quarnak"}
728 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
729 return_response=True)
730 def _got((res, status, headers)):
731 self.failUnlessReallyEqual(int(status), 200)
732 self.failUnless(not headers.has_key("content-range"))
733 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
737 def test_HEAD_FILEURL(self):
738 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
739 def _got((res, status, headers)):
740 self.failUnlessReallyEqual(res, "")
741 self.failUnlessReallyEqual(headers["content-length"][0],
742 str(len(self.BAR_CONTENTS)))
743 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
747 def test_GET_FILEURL_named(self):
748 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
749 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
750 d = self.GET(base + "/@@name=/blah.txt")
751 d.addCallback(self.failUnlessIsBarDotTxt)
752 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
753 d.addCallback(self.failUnlessIsBarDotTxt)
754 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
755 d.addCallback(self.failUnlessIsBarDotTxt)
756 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
757 d.addCallback(self.failUnlessIsBarDotTxt)
758 save_url = base + "?save=true&filename=blah.txt"
759 d.addCallback(lambda res: self.GET(save_url))
760 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
761 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
762 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
763 u_url = base + "?save=true&filename=" + u_fn_e
764 d.addCallback(lambda res: self.GET(u_url))
765 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
768 def test_PUT_FILEURL_named_bad(self):
769 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
770 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
772 "/file can only be used with GET or HEAD",
773 self.PUT, base + "/@@name=/blah.txt", "")
777 def test_GET_DIRURL_named_bad(self):
778 base = "/file/%s" % urllib.quote(self._foo_uri)
779 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
782 self.GET, base + "/@@name=/blah.txt")
785 def test_GET_slash_file_bad(self):
786 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
788 "/file must be followed by a file-cap and a name",
792 def test_GET_unhandled_URI_named(self):
793 contents, n, newuri = self.makefile(12)
794 verifier_cap = n.get_verify_cap().to_string()
795 base = "/file/%s" % urllib.quote(verifier_cap)
796 # client.create_node_from_uri() can't handle verify-caps
797 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
798 "400 Bad Request", "is not a file-cap",
802 def test_GET_unhandled_URI(self):
803 contents, n, newuri = self.makefile(12)
804 verifier_cap = n.get_verify_cap().to_string()
805 base = "/uri/%s" % urllib.quote(verifier_cap)
806 # client.create_node_from_uri() can't handle verify-caps
807 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
809 "GET unknown URI type: can only do t=info",
813 def test_GET_FILE_URI(self):
814 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
816 d.addCallback(self.failUnlessIsBarDotTxt)
819 def test_GET_FILE_URI_mdmf(self):
820 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
822 d.addCallback(self.failUnlessIsQuuxDotTxt)
825 def test_GET_FILE_URI_mdmf_extensions(self):
826 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
828 d.addCallback(self.failUnlessIsQuuxDotTxt)
831 def test_GET_FILE_URI_mdmf_readonly(self):
832 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
834 d.addCallback(self.failUnlessIsQuuxDotTxt)
837 def test_GET_FILE_URI_badchild(self):
838 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
839 errmsg = "Files have no children, certainly not named 'boguschild'"
840 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
841 "400 Bad Request", errmsg,
845 def test_PUT_FILE_URI_badchild(self):
846 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
847 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
848 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
849 "400 Bad Request", errmsg,
853 def test_PUT_FILE_URI_mdmf(self):
854 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
855 self._quux_new_contents = "new_contents"
857 d.addCallback(lambda res:
858 self.failUnlessIsQuuxDotTxt(res))
859 d.addCallback(lambda ignored:
860 self.PUT(base, self._quux_new_contents))
861 d.addCallback(lambda ignored:
863 d.addCallback(lambda res:
864 self.failUnlessReallyEqual(res, self._quux_new_contents))
867 def test_PUT_FILE_URI_mdmf_extensions(self):
868 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
869 self._quux_new_contents = "new_contents"
871 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
872 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
873 d.addCallback(lambda ignored: self.GET(base))
874 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
878 def test_PUT_FILE_URI_mdmf_readonly(self):
879 # We're not allowed to PUT things to a readonly cap.
880 base = "/uri/%s" % self._quux_txt_readonly_uri
882 d.addCallback(lambda res:
883 self.failUnlessIsQuuxDotTxt(res))
884 # What should we get here? We get a 500 error now; that's not right.
885 d.addCallback(lambda ignored:
886 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
887 "400 Bad Request", "read-only cap",
888 self.PUT, base, "new data"))
891 def test_PUT_FILE_URI_sdmf_readonly(self):
892 # We're not allowed to put things to a readonly cap.
893 base = "/uri/%s" % self._baz_txt_readonly_uri
895 d.addCallback(lambda res:
896 self.failUnlessIsBazDotTxt(res))
897 d.addCallback(lambda ignored:
898 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
899 "400 Bad Request", "read-only cap",
900 self.PUT, base, "new_data"))
903 def test_GET_etags(self):
905 def _check_etags(uri):
907 d2 = _get_etag(uri, 'json')
908 d = defer.DeferredList([d1, d2], consumeErrors=True)
910 assert all([r[0] for r in results]) # All deferred must succeed
911 assert results[0][1] + 'json' == results[1][1]
912 d.addCallback(_check)
915 def _get_etag(uri, t=''):
916 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
917 d = self.GET(targetbase, return_response=True, followRedirect=True)
918 def _just_the_etag(result):
919 data, response, headers = result
920 etag = headers['etag'][0]
921 if uri.startswith('URI:DIR'): assert etag.startswith('DIR:')
923 return d.addCallback(_just_the_etag)
925 # Check that etags work with immutable directories
926 (newkids, caps) = self._create_immutable_children()
927 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
928 simplejson.dumps(newkids))
929 d.addCallback(_check_etags)
931 # Check that etags work with immutable files
932 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
935 # TODO: version of this with a Unicode filename
936 def test_GET_FILEURL_save(self):
937 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
938 return_response=True)
939 def _got((res, statuscode, headers)):
940 content_disposition = headers["content-disposition"][0]
941 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
942 self.failUnlessIsBarDotTxt(res)
946 def test_GET_FILEURL_missing(self):
947 d = self.GET(self.public_url + "/foo/missing")
948 d.addBoth(self.should404, "test_GET_FILEURL_missing")
951 def test_GET_FILEURL_info_mdmf(self):
952 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
954 self.failUnlessIn("mutable file (mdmf)", res)
955 self.failUnlessIn(self._quux_txt_uri, res)
956 self.failUnlessIn(self._quux_txt_readonly_uri, res)
960 def test_GET_FILEURL_info_mdmf_readonly(self):
961 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
963 self.failUnlessIn("mutable file (mdmf)", res)
964 self.failIfIn(self._quux_txt_uri, res)
965 self.failUnlessIn(self._quux_txt_readonly_uri, res)
969 def test_GET_FILEURL_info_sdmf(self):
970 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
972 self.failUnlessIn("mutable file (sdmf)", res)
973 self.failUnlessIn(self._baz_txt_uri, res)
977 def test_GET_FILEURL_info_mdmf_extensions(self):
978 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
980 self.failUnlessIn("mutable file (mdmf)", res)
981 self.failUnlessIn(self._quux_txt_uri, res)
982 self.failUnlessIn(self._quux_txt_readonly_uri, res)
986 def test_PUT_overwrite_only_files(self):
987 # create a directory, put a file in that directory.
988 contents, n, filecap = self.makefile(8)
989 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
990 d.addCallback(lambda res:
991 self.PUT(self.public_url + "/foo/dir/file1.txt",
992 self.NEWFILE_CONTENTS))
993 # try to overwrite the file with replace=only-files
995 d.addCallback(lambda res:
996 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
998 d.addCallback(lambda res:
999 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1000 "There was already a child by that name, and you asked me "
1001 "to not replace it",
1002 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1006 def test_PUT_NEWFILEURL(self):
1007 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1008 # TODO: we lose the response code, so we can't check this
1009 #self.failUnlessReallyEqual(responsecode, 201)
1010 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1011 d.addCallback(lambda res:
1012 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1013 self.NEWFILE_CONTENTS))
1016 def test_PUT_NEWFILEURL_not_mutable(self):
1017 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1018 self.NEWFILE_CONTENTS)
1019 # TODO: we lose the response code, so we can't check this
1020 #self.failUnlessReallyEqual(responsecode, 201)
1021 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1022 d.addCallback(lambda res:
1023 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1024 self.NEWFILE_CONTENTS))
1027 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1028 # this should get us a few segments of an MDMF mutable file,
1029 # which we can then test for.
1030 contents = self.NEWFILE_CONTENTS * 300000
1031 d = self.PUT("/uri?format=mdmf",
1033 def _got_filecap(filecap):
1034 self.failUnless(filecap.startswith("URI:MDMF"))
1036 d.addCallback(_got_filecap)
1037 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1038 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1041 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1042 contents = self.NEWFILE_CONTENTS * 300000
1043 d = self.PUT("/uri?format=sdmf",
1045 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1046 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1049 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1050 contents = self.NEWFILE_CONTENTS * 300000
1051 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1052 400, "Bad Request", "Unknown format: foo",
1053 self.PUT, "/uri?format=foo",
1056 def test_PUT_NEWFILEURL_range_bad(self):
1057 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1058 target = self.public_url + "/foo/new.txt"
1059 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1060 "501 Not Implemented",
1061 "Content-Range in PUT not yet supported",
1062 # (and certainly not for immutable files)
1063 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1065 d.addCallback(lambda res:
1066 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1069 def test_PUT_NEWFILEURL_mutable(self):
1070 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1071 self.NEWFILE_CONTENTS)
1072 # TODO: we lose the response code, so we can't check this
1073 #self.failUnlessReallyEqual(responsecode, 201)
1074 def _check_uri(res):
1075 u = uri.from_string_mutable_filenode(res)
1076 self.failUnless(u.is_mutable())
1077 self.failIf(u.is_readonly())
1079 d.addCallback(_check_uri)
1080 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1081 d.addCallback(lambda res:
1082 self.failUnlessMutableChildContentsAre(self._foo_node,
1084 self.NEWFILE_CONTENTS))
1087 def test_PUT_NEWFILEURL_mutable_toobig(self):
1088 # It is okay to upload large mutable files, so we should be able
1090 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1091 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1094 def test_PUT_NEWFILEURL_replace(self):
1095 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1096 # TODO: we lose the response code, so we can't check this
1097 #self.failUnlessReallyEqual(responsecode, 200)
1098 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1099 d.addCallback(lambda res:
1100 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1101 self.NEWFILE_CONTENTS))
1104 def test_PUT_NEWFILEURL_bad_t(self):
1105 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1106 "PUT to a file: bad t=bogus",
1107 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1111 def test_PUT_NEWFILEURL_no_replace(self):
1112 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1113 self.NEWFILE_CONTENTS)
1114 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1116 "There was already a child by that name, and you asked me "
1117 "to not replace it")
1120 def test_PUT_NEWFILEURL_mkdirs(self):
1121 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1123 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1124 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1125 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1126 d.addCallback(lambda res:
1127 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1128 self.NEWFILE_CONTENTS))
1131 def test_PUT_NEWFILEURL_blocked(self):
1132 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1133 self.NEWFILE_CONTENTS)
1134 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1136 "Unable to create directory 'blockingfile': a file was in the way")
1139 def test_PUT_NEWFILEURL_emptyname(self):
1140 # an empty pathname component (i.e. a double-slash) is disallowed
1141 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1143 "The webapi does not allow empty pathname components",
1144 self.PUT, self.public_url + "/foo//new.txt", "")
1147 def test_DELETE_FILEURL(self):
1148 d = self.DELETE(self.public_url + "/foo/bar.txt")
1149 d.addCallback(lambda res:
1150 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1153 def test_DELETE_FILEURL_missing(self):
1154 d = self.DELETE(self.public_url + "/foo/missing")
1155 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1158 def test_DELETE_FILEURL_missing2(self):
1159 d = self.DELETE(self.public_url + "/missing/missing")
1160 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1163 def failUnlessHasBarDotTxtMetadata(self, res):
1164 data = simplejson.loads(res)
1165 self.failUnless(isinstance(data, list))
1166 self.failUnlessIn("metadata", data[1])
1167 self.failUnlessIn("tahoe", data[1]["metadata"])
1168 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1169 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1170 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1171 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1173 def test_GET_FILEURL_json(self):
1174 # twisted.web.http.parse_qs ignores any query args without an '=', so
1175 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1176 # instead. This may make it tricky to emulate the S3 interface
1178 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1180 self.failUnlessIsBarJSON(data)
1181 self.failUnlessHasBarDotTxtMetadata(data)
1183 d.addCallback(_check1)
1186 def test_GET_FILEURL_json_mutable_type(self):
1187 # The JSON should include format, which says whether the
1188 # file is SDMF or MDMF
1189 d = self.PUT("/uri?format=mdmf",
1190 self.NEWFILE_CONTENTS * 300000)
1191 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1192 def _got_json(json, version):
1193 data = simplejson.loads(json)
1194 assert "filenode" == data[0]
1196 assert isinstance(data, dict)
1198 self.failUnlessIn("format", data)
1199 self.failUnlessEqual(data["format"], version)
1201 d.addCallback(_got_json, "MDMF")
1202 # Now make an SDMF file and check that it is reported correctly.
1203 d.addCallback(lambda ignored:
1204 self.PUT("/uri?format=sdmf",
1205 self.NEWFILE_CONTENTS * 300000))
1206 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1207 d.addCallback(_got_json, "SDMF")
1210 def test_GET_FILEURL_json_mdmf(self):
1211 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1212 d.addCallback(self.failUnlessIsQuuxJSON)
1215 def test_GET_FILEURL_json_missing(self):
1216 d = self.GET(self.public_url + "/foo/missing?json")
1217 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1220 def test_GET_FILEURL_uri(self):
1221 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1223 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1224 d.addCallback(_check)
1225 d.addCallback(lambda res:
1226 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1228 # for now, for files, uris and readonly-uris are the same
1229 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1230 d.addCallback(_check2)
1233 def test_GET_FILEURL_badtype(self):
1234 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1237 self.public_url + "/foo/bar.txt?t=bogus")
1240 def test_CSS_FILE(self):
1241 d = self.GET("/tahoe.css", followRedirect=True)
1243 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1244 self.failUnless(CSS_STYLE.search(res), res)
1245 d.addCallback(_check)
1248 def test_GET_FILEURL_uri_missing(self):
1249 d = self.GET(self.public_url + "/foo/missing?t=uri")
1250 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1253 def _check_upload_and_mkdir_forms(self, html):
1254 # We should have a form to create a file, with radio buttons that allow
1255 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1256 self.failUnlessIn('name="t" value="upload"', html)
1257 self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1258 self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1259 self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1261 # We should also have the ability to create a mutable directory, with
1262 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1263 # or MDMF directory.
1264 self.failUnlessIn('name="t" value="mkdir"', html)
1265 self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1266 self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1268 self.failUnlessIn(FAVICON_MARKUP, html)
1270 def test_GET_DIRECTORY_html(self):
1271 d = self.GET(self.public_url + "/foo", followRedirect=True)
1273 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1274 self._check_upload_and_mkdir_forms(html)
1275 self.failUnlessIn("quux", html)
1276 d.addCallback(_check)
1279 def test_GET_root_html(self):
1281 d.addCallback(self._check_upload_and_mkdir_forms)
1284 def test_GET_DIRURL(self):
1285 # the addSlash means we get a redirect here
1286 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1288 d = self.GET(self.public_url + "/foo", followRedirect=True)
1290 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1292 # the FILE reference points to a URI, but it should end in bar.txt
1293 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1294 (ROOT, urllib.quote(self._bar_txt_uri)))
1295 get_bar = "".join([r'<td>FILE</td>',
1297 r'<a href="%s">bar.txt</a>' % bar_url,
1299 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1301 self.failUnless(re.search(get_bar, res), res)
1302 for label in ['unlink', 'rename/move']:
1303 for line in res.split("\n"):
1304 # find the line that contains the relevant button for bar.txt
1305 if ("form action" in line and
1306 ('value="%s"' % (label,)) in line and
1307 'value="bar.txt"' in line):
1308 # the form target should use a relative URL
1309 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1310 self.failUnlessIn('action="%s"' % foo_url, line)
1311 # and the when_done= should too
1312 #done_url = urllib.quote(???)
1313 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1315 # 'unlink' needs to use POST because it directly has a side effect
1316 if label == 'unlink':
1317 self.failUnlessIn('method="post"', line)
1320 self.fail("unable to find '%s bar.txt' line" % (label,))
1322 # the DIR reference just points to a URI
1323 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1324 get_sub = ((r'<td>DIR</td>')
1325 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1326 self.failUnless(re.search(get_sub, res), res)
1327 d.addCallback(_check)
1329 # look at a readonly directory
1330 d.addCallback(lambda res:
1331 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1333 self.failUnlessIn("(read-only)", res)
1334 self.failIfIn("Upload a file", res)
1335 d.addCallback(_check2)
1337 # and at a directory that contains a readonly directory
1338 d.addCallback(lambda res:
1339 self.GET(self.public_url, followRedirect=True))
1341 self.failUnless(re.search('<td>DIR-RO</td>'
1342 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1343 d.addCallback(_check3)
1345 # and an empty directory
1346 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1348 self.failUnlessIn("directory is empty", res)
1349 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1350 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1351 d.addCallback(_check4)
1353 # and at a literal directory
1354 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1355 d.addCallback(lambda res:
1356 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1358 self.failUnlessIn('(immutable)', res)
1359 self.failUnless(re.search('<td>FILE</td>'
1360 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1361 d.addCallback(_check5)
1364 def test_GET_DIRURL_badtype(self):
1365 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1369 self.public_url + "/foo?t=bogus")
1372 def test_GET_DIRURL_json(self):
1373 d = self.GET(self.public_url + "/foo?t=json")
1374 d.addCallback(self.failUnlessIsFooJSON)
1377 def test_GET_DIRURL_json_format(self):
1378 d = self.PUT(self.public_url + \
1379 "/foo/sdmf.txt?format=sdmf",
1380 self.NEWFILE_CONTENTS * 300000)
1381 d.addCallback(lambda ignored:
1382 self.PUT(self.public_url + \
1383 "/foo/mdmf.txt?format=mdmf",
1384 self.NEWFILE_CONTENTS * 300000))
1385 # Now we have an MDMF and SDMF file in the directory. If we GET
1386 # its JSON, we should see their encodings.
1387 d.addCallback(lambda ignored:
1388 self.GET(self.public_url + "/foo?t=json"))
1389 def _got_json(json):
1390 data = simplejson.loads(json)
1391 assert data[0] == "dirnode"
1394 kids = data['children']
1396 mdmf_data = kids['mdmf.txt'][1]
1397 self.failUnlessIn("format", mdmf_data)
1398 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1400 sdmf_data = kids['sdmf.txt'][1]
1401 self.failUnlessIn("format", sdmf_data)
1402 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1403 d.addCallback(_got_json)
1407 def test_POST_DIRURL_manifest_no_ophandle(self):
1408 d = self.shouldFail2(error.Error,
1409 "test_POST_DIRURL_manifest_no_ophandle",
1411 "slow operation requires ophandle=",
1412 self.POST, self.public_url, t="start-manifest")
1415 def test_POST_DIRURL_manifest(self):
1416 d = defer.succeed(None)
1417 def getman(ignored, output):
1418 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1419 followRedirect=True)
1420 d.addCallback(self.wait_for_operation, "125")
1421 d.addCallback(self.get_operation_results, "125", output)
1423 d.addCallback(getman, None)
1424 def _got_html(manifest):
1425 self.failUnlessIn("Manifest of SI=", manifest)
1426 self.failUnlessIn("<td>sub</td>", manifest)
1427 self.failUnlessIn(self._sub_uri, manifest)
1428 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1429 self.failUnlessIn(FAVICON_MARKUP, manifest)
1430 d.addCallback(_got_html)
1432 # both t=status and unadorned GET should be identical
1433 d.addCallback(lambda res: self.GET("/operations/125"))
1434 d.addCallback(_got_html)
1436 d.addCallback(getman, "html")
1437 d.addCallback(_got_html)
1438 d.addCallback(getman, "text")
1439 def _got_text(manifest):
1440 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1441 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1442 d.addCallback(_got_text)
1443 d.addCallback(getman, "JSON")
1445 data = res["manifest"]
1447 for (path_list, cap) in data:
1448 got[tuple(path_list)] = cap
1449 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1450 self.failUnlessIn((u"sub", u"baz.txt"), got)
1451 self.failUnlessIn("finished", res)
1452 self.failUnlessIn("origin", res)
1453 self.failUnlessIn("storage-index", res)
1454 self.failUnlessIn("verifycaps", res)
1455 self.failUnlessIn("stats", res)
1456 d.addCallback(_got_json)
1459 def test_POST_DIRURL_deepsize_no_ophandle(self):
1460 d = self.shouldFail2(error.Error,
1461 "test_POST_DIRURL_deepsize_no_ophandle",
1463 "slow operation requires ophandle=",
1464 self.POST, self.public_url, t="start-deep-size")
1467 def test_POST_DIRURL_deepsize(self):
1468 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1469 followRedirect=True)
1470 d.addCallback(self.wait_for_operation, "126")
1471 d.addCallback(self.get_operation_results, "126", "json")
1472 def _got_json(data):
1473 self.failUnlessReallyEqual(data["finished"], True)
1475 self.failUnless(size > 1000)
1476 d.addCallback(_got_json)
1477 d.addCallback(self.get_operation_results, "126", "text")
1479 mo = re.search(r'^size: (\d+)$', res, re.M)
1480 self.failUnless(mo, res)
1481 size = int(mo.group(1))
1482 # with directories, the size varies.
1483 self.failUnless(size > 1000)
1484 d.addCallback(_got_text)
1487 def test_POST_DIRURL_deepstats_no_ophandle(self):
1488 d = self.shouldFail2(error.Error,
1489 "test_POST_DIRURL_deepstats_no_ophandle",
1491 "slow operation requires ophandle=",
1492 self.POST, self.public_url, t="start-deep-stats")
1495 def test_POST_DIRURL_deepstats(self):
1496 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1497 followRedirect=True)
1498 d.addCallback(self.wait_for_operation, "127")
1499 d.addCallback(self.get_operation_results, "127", "json")
1500 def _got_json(stats):
1501 expected = {"count-immutable-files": 3,
1502 "count-mutable-files": 2,
1503 "count-literal-files": 0,
1505 "count-directories": 3,
1506 "size-immutable-files": 57,
1507 "size-literal-files": 0,
1508 #"size-directories": 1912, # varies
1509 #"largest-directory": 1590,
1510 "largest-directory-children": 7,
1511 "largest-immutable-file": 19,
1513 for k,v in expected.iteritems():
1514 self.failUnlessReallyEqual(stats[k], v,
1515 "stats[%s] was %s, not %s" %
1517 self.failUnlessReallyEqual(stats["size-files-histogram"],
1519 d.addCallback(_got_json)
1522 def test_POST_DIRURL_stream_manifest(self):
1523 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1525 self.failUnless(res.endswith("\n"))
1526 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1527 self.failUnlessReallyEqual(len(units), 9)
1528 self.failUnlessEqual(units[-1]["type"], "stats")
1530 self.failUnlessEqual(first["path"], [])
1531 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1532 self.failUnlessEqual(first["type"], "directory")
1533 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1534 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1535 self.failIfEqual(baz["storage-index"], None)
1536 self.failIfEqual(baz["verifycap"], None)
1537 self.failIfEqual(baz["repaircap"], None)
1538 # XXX: Add quux and baz to this test.
1540 d.addCallback(_check)
1543 def test_GET_DIRURL_uri(self):
1544 d = self.GET(self.public_url + "/foo?t=uri")
1546 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1547 d.addCallback(_check)
1550 def test_GET_DIRURL_readonly_uri(self):
1551 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1553 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1554 d.addCallback(_check)
1557 def test_PUT_NEWDIRURL(self):
1558 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1559 d.addCallback(lambda res:
1560 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1561 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1562 d.addCallback(self.failUnlessNodeKeysAre, [])
1565 def test_PUT_NEWDIRURL_mdmf(self):
1566 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1567 d.addCallback(lambda res:
1568 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1569 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1570 d.addCallback(lambda node:
1571 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1574 def test_PUT_NEWDIRURL_sdmf(self):
1575 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1577 d.addCallback(lambda res:
1578 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1579 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1580 d.addCallback(lambda node:
1581 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1584 def test_PUT_NEWDIRURL_bad_format(self):
1585 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1586 400, "Bad Request", "Unknown format: foo",
1587 self.PUT, self.public_url +
1588 "/foo/newdir=?t=mkdir&format=foo", "")
1590 def test_POST_NEWDIRURL(self):
1591 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1592 d.addCallback(lambda res:
1593 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1594 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1595 d.addCallback(self.failUnlessNodeKeysAre, [])
1598 def test_POST_NEWDIRURL_mdmf(self):
1599 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1600 d.addCallback(lambda res:
1601 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1602 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1603 d.addCallback(lambda node:
1604 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1607 def test_POST_NEWDIRURL_sdmf(self):
1608 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1609 d.addCallback(lambda res:
1610 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1611 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1612 d.addCallback(lambda node:
1613 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1616 def test_POST_NEWDIRURL_bad_format(self):
1617 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1618 400, "Bad Request", "Unknown format: foo",
1619 self.POST2, self.public_url + \
1620 "/foo/newdir?t=mkdir&format=foo", "")
1622 def test_POST_NEWDIRURL_emptyname(self):
1623 # an empty pathname component (i.e. a double-slash) is disallowed
1624 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1626 "The webapi does not allow empty pathname components, i.e. a double slash",
1627 self.POST, self.public_url + "//?t=mkdir")
1630 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1631 (newkids, caps) = self._create_initial_children()
1632 query = "/foo/newdir?t=mkdir-with-children"
1633 if version == MDMF_VERSION:
1634 query += "&format=mdmf"
1635 elif version == SDMF_VERSION:
1636 query += "&format=sdmf"
1638 version = SDMF_VERSION # for later
1639 d = self.POST2(self.public_url + query,
1640 simplejson.dumps(newkids))
1642 n = self.s.create_node_from_uri(uri.strip())
1643 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1644 self.failUnlessEqual(n._node.get_version(), version)
1645 d2.addCallback(lambda ign:
1646 self.failUnlessROChildURIIs(n, u"child-imm",
1648 d2.addCallback(lambda ign:
1649 self.failUnlessRWChildURIIs(n, u"child-mutable",
1651 d2.addCallback(lambda ign:
1652 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1654 d2.addCallback(lambda ign:
1655 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1656 caps['unknown_rocap']))
1657 d2.addCallback(lambda ign:
1658 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1659 caps['unknown_rwcap']))
1660 d2.addCallback(lambda ign:
1661 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1662 caps['unknown_immcap']))
1663 d2.addCallback(lambda ign:
1664 self.failUnlessRWChildURIIs(n, u"dirchild",
1666 d2.addCallback(lambda ign:
1667 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1669 d2.addCallback(lambda ign:
1670 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1671 caps['emptydircap']))
1673 d.addCallback(_check)
1674 d.addCallback(lambda res:
1675 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1676 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1677 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1678 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1679 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1682 def test_POST_NEWDIRURL_initial_children(self):
1683 return self._do_POST_NEWDIRURL_initial_children_test()
1685 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1686 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1688 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1689 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1691 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1692 (newkids, caps) = self._create_initial_children()
1693 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1694 400, "Bad Request", "Unknown format: foo",
1695 self.POST2, self.public_url + \
1696 "/foo/newdir?t=mkdir-with-children&format=foo",
1697 simplejson.dumps(newkids))
1699 def test_POST_NEWDIRURL_immutable(self):
1700 (newkids, caps) = self._create_immutable_children()
1701 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1702 simplejson.dumps(newkids))
1704 n = self.s.create_node_from_uri(uri.strip())
1705 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1706 d2.addCallback(lambda ign:
1707 self.failUnlessROChildURIIs(n, u"child-imm",
1709 d2.addCallback(lambda ign:
1710 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1711 caps['unknown_immcap']))
1712 d2.addCallback(lambda ign:
1713 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1715 d2.addCallback(lambda ign:
1716 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1718 d2.addCallback(lambda ign:
1719 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1720 caps['emptydircap']))
1722 d.addCallback(_check)
1723 d.addCallback(lambda res:
1724 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1725 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1726 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1727 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1728 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1729 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1730 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1731 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1732 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1733 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1734 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1735 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1736 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1737 d.addErrback(self.explain_web_error)
1740 def test_POST_NEWDIRURL_immutable_bad(self):
1741 (newkids, caps) = self._create_initial_children()
1742 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1744 "needed to be immutable but was not",
1746 self.public_url + "/foo/newdir?t=mkdir-immutable",
1747 simplejson.dumps(newkids))
1750 def test_PUT_NEWDIRURL_exists(self):
1751 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1752 d.addCallback(lambda res:
1753 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1754 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1755 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1758 def test_PUT_NEWDIRURL_blocked(self):
1759 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1760 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1762 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1763 d.addCallback(lambda res:
1764 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1765 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1766 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1769 def test_PUT_NEWDIRURL_mkdirs(self):
1770 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1771 d.addCallback(lambda res:
1772 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1773 d.addCallback(lambda res:
1774 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1775 d.addCallback(lambda res:
1776 self._foo_node.get_child_at_path(u"subdir/newdir"))
1777 d.addCallback(self.failUnlessNodeKeysAre, [])
1780 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1781 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1782 d.addCallback(lambda ignored:
1783 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1784 d.addCallback(lambda ignored:
1785 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1786 d.addCallback(lambda ignored:
1787 self._foo_node.get_child_at_path(u"subdir"))
1788 def _got_subdir(subdir):
1789 # XXX: What we want?
1790 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1791 self.failUnlessNodeHasChild(subdir, u"newdir")
1792 return subdir.get_child_at_path(u"newdir")
1793 d.addCallback(_got_subdir)
1794 d.addCallback(lambda newdir:
1795 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1798 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1799 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1800 d.addCallback(lambda ignored:
1801 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1802 d.addCallback(lambda ignored:
1803 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1804 d.addCallback(lambda ignored:
1805 self._foo_node.get_child_at_path(u"subdir"))
1806 def _got_subdir(subdir):
1807 # XXX: What we want?
1808 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1809 self.failUnlessNodeHasChild(subdir, u"newdir")
1810 return subdir.get_child_at_path(u"newdir")
1811 d.addCallback(_got_subdir)
1812 d.addCallback(lambda newdir:
1813 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1816 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1817 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1818 400, "Bad Request", "Unknown format: foo",
1819 self.PUT, self.public_url + \
1820 "/foo/subdir/newdir?t=mkdir&format=foo",
1823 def test_DELETE_DIRURL(self):
1824 d = self.DELETE(self.public_url + "/foo")
1825 d.addCallback(lambda res:
1826 self.failIfNodeHasChild(self.public_root, u"foo"))
1829 def test_DELETE_DIRURL_missing(self):
1830 d = self.DELETE(self.public_url + "/foo/missing")
1831 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1832 d.addCallback(lambda res:
1833 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1836 def test_DELETE_DIRURL_missing2(self):
1837 d = self.DELETE(self.public_url + "/missing")
1838 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1841 def dump_root(self):
1843 w = webish.DirnodeWalkerMixin()
1844 def visitor(childpath, childnode, metadata):
1846 d = w.walk(self.public_root, visitor)
1849 def failUnlessNodeKeysAre(self, node, expected_keys):
1850 for k in expected_keys:
1851 assert isinstance(k, unicode)
1853 def _check(children):
1854 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1855 d.addCallback(_check)
1857 def failUnlessNodeHasChild(self, node, name):
1858 assert isinstance(name, unicode)
1860 def _check(children):
1861 self.failUnlessIn(name, children)
1862 d.addCallback(_check)
1864 def failIfNodeHasChild(self, node, name):
1865 assert isinstance(name, unicode)
1867 def _check(children):
1868 self.failIfIn(name, children)
1869 d.addCallback(_check)
1872 def failUnlessChildContentsAre(self, node, name, expected_contents):
1873 assert isinstance(name, unicode)
1874 d = node.get_child_at_path(name)
1875 d.addCallback(lambda node: download_to_data(node))
1876 def _check(contents):
1877 self.failUnlessReallyEqual(contents, expected_contents)
1878 d.addCallback(_check)
1881 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1882 assert isinstance(name, unicode)
1883 d = node.get_child_at_path(name)
1884 d.addCallback(lambda node: node.download_best_version())
1885 def _check(contents):
1886 self.failUnlessReallyEqual(contents, expected_contents)
1887 d.addCallback(_check)
1890 def failUnlessRWChildURIIs(self, node, name, expected_uri):
1891 assert isinstance(name, unicode)
1892 d = node.get_child_at_path(name)
1894 self.failUnless(child.is_unknown() or not child.is_readonly())
1895 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1896 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1897 expected_ro_uri = self._make_readonly(expected_uri)
1899 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1900 d.addCallback(_check)
1903 def failUnlessROChildURIIs(self, node, name, expected_uri):
1904 assert isinstance(name, unicode)
1905 d = node.get_child_at_path(name)
1907 self.failUnless(child.is_unknown() or child.is_readonly())
1908 self.failUnlessReallyEqual(child.get_write_uri(), None)
1909 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1910 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1911 d.addCallback(_check)
1914 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1915 assert isinstance(name, unicode)
1916 d = node.get_child_at_path(name)
1918 self.failUnless(child.is_unknown() or not child.is_readonly())
1919 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1920 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1921 expected_ro_uri = self._make_readonly(got_uri)
1923 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1924 d.addCallback(_check)
1927 def failUnlessURIMatchesROChild(self, got_uri, node, name):
1928 assert isinstance(name, unicode)
1929 d = node.get_child_at_path(name)
1931 self.failUnless(child.is_unknown() or child.is_readonly())
1932 self.failUnlessReallyEqual(child.get_write_uri(), None)
1933 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1934 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1935 d.addCallback(_check)
1938 def failUnlessCHKURIHasContents(self, got_uri, contents):
1939 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1941 def test_POST_upload(self):
1942 d = self.POST(self.public_url + "/foo", t="upload",
1943 file=("new.txt", self.NEWFILE_CONTENTS))
1945 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1946 d.addCallback(lambda res:
1947 self.failUnlessChildContentsAre(fn, u"new.txt",
1948 self.NEWFILE_CONTENTS))
1951 def test_POST_upload_unicode(self):
1952 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1953 d = self.POST(self.public_url + "/foo", t="upload",
1954 file=(filename, self.NEWFILE_CONTENTS))
1956 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1957 d.addCallback(lambda res:
1958 self.failUnlessChildContentsAre(fn, filename,
1959 self.NEWFILE_CONTENTS))
1960 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1961 d.addCallback(lambda res: self.GET(target_url))
1962 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1963 self.NEWFILE_CONTENTS,
1967 def test_POST_upload_unicode_named(self):
1968 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1969 d = self.POST(self.public_url + "/foo", t="upload",
1971 file=("overridden", self.NEWFILE_CONTENTS))
1973 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1974 d.addCallback(lambda res:
1975 self.failUnlessChildContentsAre(fn, filename,
1976 self.NEWFILE_CONTENTS))
1977 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1978 d.addCallback(lambda res: self.GET(target_url))
1979 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1980 self.NEWFILE_CONTENTS,
1984 def test_POST_upload_no_link(self):
1985 d = self.POST("/uri", t="upload",
1986 file=("new.txt", self.NEWFILE_CONTENTS))
1987 def _check_upload_results(page):
1988 # this should be a page which describes the results of the upload
1989 # that just finished.
1990 self.failUnlessIn("Upload Results:", page)
1991 self.failUnlessIn("URI:", page)
1992 uri_re = re.compile("URI: <tt><span>(.*)</span>")
1993 mo = uri_re.search(page)
1994 self.failUnless(mo, page)
1995 new_uri = mo.group(1)
1997 d.addCallback(_check_upload_results)
1998 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2001 def test_POST_upload_no_link_whendone(self):
2002 d = self.POST("/uri", t="upload", when_done="/",
2003 file=("new.txt", self.NEWFILE_CONTENTS))
2004 d.addBoth(self.shouldRedirect, "/")
2007 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2008 d = defer.maybeDeferred(callable, *args, **kwargs)
2010 if isinstance(res, failure.Failure):
2011 res.trap(error.PageRedirect)
2012 statuscode = res.value.status
2013 target = res.value.location
2014 return checker(statuscode, target)
2015 self.fail("%s: callable was supposed to redirect, not return '%s'"
2020 def test_POST_upload_no_link_whendone_results(self):
2021 def check(statuscode, target):
2022 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2023 self.failUnless(target.startswith(self.webish_url), target)
2024 return client.getPage(target, method="GET")
2025 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2027 self.POST, "/uri", t="upload",
2028 when_done="/uri/%(uri)s",
2029 file=("new.txt", self.NEWFILE_CONTENTS))
2030 d.addCallback(lambda res:
2031 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2034 def test_POST_upload_no_link_mutable(self):
2035 d = self.POST("/uri", t="upload", mutable="true",
2036 file=("new.txt", self.NEWFILE_CONTENTS))
2037 def _check(filecap):
2038 filecap = filecap.strip()
2039 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2040 self.filecap = filecap
2041 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2042 self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2043 n = self.s.create_node_from_uri(filecap)
2044 return n.download_best_version()
2045 d.addCallback(_check)
2047 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2048 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2049 d.addCallback(_check2)
2051 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2052 return self.GET("/file/%s" % urllib.quote(self.filecap))
2053 d.addCallback(_check3)
2055 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2056 d.addCallback(_check4)
2059 def test_POST_upload_no_link_mutable_toobig(self):
2060 # The SDMF size limit is no longer in place, so we should be
2061 # able to upload mutable files that are as large as we want them
2063 d = self.POST("/uri", t="upload", mutable="true",
2064 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2068 def test_POST_upload_format_unlinked(self):
2069 def _check_upload_unlinked(ign, format, uri_prefix):
2070 filename = format + ".txt"
2071 d = self.POST("/uri?t=upload&format=" + format,
2072 file=(filename, self.NEWFILE_CONTENTS * 300000))
2073 def _got_results(results):
2074 if format.upper() in ("SDMF", "MDMF"):
2075 # webapi.rst says this returns a filecap
2078 # for immutable, it returns an "upload results page", and
2079 # the filecap is buried inside
2080 line = [l for l in results.split("\n") if "URI: " in l][0]
2081 mo = re.search(r'<span>([^<]+)</span>', line)
2082 filecap = mo.group(1)
2083 self.failUnless(filecap.startswith(uri_prefix),
2084 (uri_prefix, filecap))
2085 return self.GET("/uri/%s?t=json" % filecap)
2086 d.addCallback(_got_results)
2087 def _got_json(json):
2088 data = simplejson.loads(json)
2090 self.failUnlessIn("format", data)
2091 self.failUnlessEqual(data["format"], format.upper())
2092 d.addCallback(_got_json)
2094 d = defer.succeed(None)
2095 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2096 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2097 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2098 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2101 def test_POST_upload_bad_format_unlinked(self):
2102 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2103 400, "Bad Request", "Unknown format: foo",
2105 "/uri?t=upload&format=foo",
2106 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2108 def test_POST_upload_format(self):
2109 def _check_upload(ign, format, uri_prefix, fn=None):
2110 filename = format + ".txt"
2111 d = self.POST(self.public_url +
2112 "/foo?t=upload&format=" + format,
2113 file=(filename, self.NEWFILE_CONTENTS * 300000))
2114 def _got_filecap(filecap):
2116 filenameu = unicode(filename)
2117 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2118 self.failUnless(filecap.startswith(uri_prefix))
2119 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2120 d.addCallback(_got_filecap)
2121 def _got_json(json):
2122 data = simplejson.loads(json)
2124 self.failUnlessIn("format", data)
2125 self.failUnlessEqual(data["format"], format.upper())
2126 d.addCallback(_got_json)
2129 d = defer.succeed(None)
2130 d.addCallback(_check_upload, "chk", "URI:CHK")
2131 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2132 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2133 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2136 def test_POST_upload_bad_format(self):
2137 return self.shouldHTTPError("POST_upload_bad_format",
2138 400, "Bad Request", "Unknown format: foo",
2139 self.POST, self.public_url + \
2140 "/foo?t=upload&format=foo",
2141 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2143 def test_POST_upload_mutable(self):
2144 # this creates a mutable file
2145 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2146 file=("new.txt", self.NEWFILE_CONTENTS))
2148 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2149 d.addCallback(lambda res:
2150 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2151 self.NEWFILE_CONTENTS))
2152 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2154 self.failUnless(IMutableFileNode.providedBy(newnode))
2155 self.failUnless(newnode.is_mutable())
2156 self.failIf(newnode.is_readonly())
2157 self._mutable_node = newnode
2158 self._mutable_uri = newnode.get_uri()
2161 # now upload it again and make sure that the URI doesn't change
2162 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2163 d.addCallback(lambda res:
2164 self.POST(self.public_url + "/foo", t="upload",
2166 file=("new.txt", NEWER_CONTENTS)))
2167 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2168 d.addCallback(lambda res:
2169 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2171 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2173 self.failUnless(IMutableFileNode.providedBy(newnode))
2174 self.failUnless(newnode.is_mutable())
2175 self.failIf(newnode.is_readonly())
2176 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2177 d.addCallback(_got2)
2179 # upload a second time, using PUT instead of POST
2180 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2181 d.addCallback(lambda res:
2182 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2183 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2184 d.addCallback(lambda res:
2185 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2188 # finally list the directory, since mutable files are displayed
2189 # slightly differently
2191 d.addCallback(lambda res:
2192 self.GET(self.public_url + "/foo/",
2193 followRedirect=True))
2194 def _check_page(res):
2195 # TODO: assert more about the contents
2196 self.failUnlessIn("SSK", res)
2198 d.addCallback(_check_page)
2200 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2202 self.failUnless(IMutableFileNode.providedBy(newnode))
2203 self.failUnless(newnode.is_mutable())
2204 self.failIf(newnode.is_readonly())
2205 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2206 d.addCallback(_got3)
2208 # look at the JSON form of the enclosing directory
2209 d.addCallback(lambda res:
2210 self.GET(self.public_url + "/foo/?t=json",
2211 followRedirect=True))
2212 def _check_page_json(res):
2213 parsed = simplejson.loads(res)
2214 self.failUnlessEqual(parsed[0], "dirnode")
2215 children = dict( [(unicode(name),value)
2217 in parsed[1]["children"].iteritems()] )
2218 self.failUnlessIn(u"new.txt", children)
2219 new_json = children[u"new.txt"]
2220 self.failUnlessEqual(new_json[0], "filenode")
2221 self.failUnless(new_json[1]["mutable"])
2222 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2223 ro_uri = self._mutable_node.get_readonly().to_string()
2224 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2225 d.addCallback(_check_page_json)
2227 # and the JSON form of the file
2228 d.addCallback(lambda res:
2229 self.GET(self.public_url + "/foo/new.txt?t=json"))
2230 def _check_file_json(res):
2231 parsed = simplejson.loads(res)
2232 self.failUnlessEqual(parsed[0], "filenode")
2233 self.failUnless(parsed[1]["mutable"])
2234 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2235 ro_uri = self._mutable_node.get_readonly().to_string()
2236 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2237 d.addCallback(_check_file_json)
2239 # and look at t=uri and t=readonly-uri
2240 d.addCallback(lambda res:
2241 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2242 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2243 d.addCallback(lambda res:
2244 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2245 def _check_ro_uri(res):
2246 ro_uri = self._mutable_node.get_readonly().to_string()
2247 self.failUnlessReallyEqual(res, ro_uri)
2248 d.addCallback(_check_ro_uri)
2250 # make sure we can get to it from /uri/URI
2251 d.addCallback(lambda res:
2252 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2253 d.addCallback(lambda res:
2254 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2256 # and that HEAD computes the size correctly
2257 d.addCallback(lambda res:
2258 self.HEAD(self.public_url + "/foo/new.txt",
2259 return_response=True))
2260 def _got_headers((res, status, headers)):
2261 self.failUnlessReallyEqual(res, "")
2262 self.failUnlessReallyEqual(headers["content-length"][0],
2263 str(len(NEW2_CONTENTS)))
2264 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2265 d.addCallback(_got_headers)
2267 # make sure that outdated size limits aren't enforced anymore.
2268 d.addCallback(lambda ignored:
2269 self.POST(self.public_url + "/foo", t="upload",
2272 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2273 d.addErrback(self.dump_error)
2276 def test_POST_upload_mutable_toobig(self):
2277 # SDMF had a size limti that was removed a while ago. MDMF has
2278 # never had a size limit. Test to make sure that we do not
2279 # encounter errors when trying to upload large mutable files,
2280 # since there should be no coded prohibitions regarding large
2282 d = self.POST(self.public_url + "/foo",
2283 t="upload", mutable="true",
2284 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2287 def dump_error(self, f):
2288 # if the web server returns an error code (like 400 Bad Request),
2289 # web.client.getPage puts the HTTP response body into the .response
2290 # attribute of the exception object that it gives back. It does not
2291 # appear in the Failure's repr(), so the ERROR that trial displays
2292 # will be rather terse and unhelpful. addErrback this method to the
2293 # end of your chain to get more information out of these errors.
2294 if f.check(error.Error):
2295 print "web.error.Error:"
2297 print f.value.response
2300 def test_POST_upload_replace(self):
2301 d = self.POST(self.public_url + "/foo", t="upload",
2302 file=("bar.txt", self.NEWFILE_CONTENTS))
2304 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2305 d.addCallback(lambda res:
2306 self.failUnlessChildContentsAre(fn, u"bar.txt",
2307 self.NEWFILE_CONTENTS))
2310 def test_POST_upload_no_replace_ok(self):
2311 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2312 file=("new.txt", self.NEWFILE_CONTENTS))
2313 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2314 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2315 self.NEWFILE_CONTENTS))
2318 def test_POST_upload_no_replace_queryarg(self):
2319 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2320 file=("bar.txt", self.NEWFILE_CONTENTS))
2321 d.addBoth(self.shouldFail, error.Error,
2322 "POST_upload_no_replace_queryarg",
2324 "There was already a child by that name, and you asked me "
2325 "to not replace it")
2326 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2327 d.addCallback(self.failUnlessIsBarDotTxt)
2330 def test_POST_upload_no_replace_field(self):
2331 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2332 file=("bar.txt", self.NEWFILE_CONTENTS))
2333 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2335 "There was already a child by that name, and you asked me "
2336 "to not replace it")
2337 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2338 d.addCallback(self.failUnlessIsBarDotTxt)
2341 def test_POST_upload_whendone(self):
2342 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2343 file=("new.txt", self.NEWFILE_CONTENTS))
2344 d.addBoth(self.shouldRedirect, "/THERE")
2346 d.addCallback(lambda res:
2347 self.failUnlessChildContentsAre(fn, u"new.txt",
2348 self.NEWFILE_CONTENTS))
2351 def test_POST_upload_named(self):
2353 d = self.POST(self.public_url + "/foo", t="upload",
2354 name="new.txt", file=self.NEWFILE_CONTENTS)
2355 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2356 d.addCallback(lambda res:
2357 self.failUnlessChildContentsAre(fn, u"new.txt",
2358 self.NEWFILE_CONTENTS))
2361 def test_POST_upload_named_badfilename(self):
2362 d = self.POST(self.public_url + "/foo", t="upload",
2363 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2364 d.addBoth(self.shouldFail, error.Error,
2365 "test_POST_upload_named_badfilename",
2367 "name= may not contain a slash",
2369 # make sure that nothing was added
2370 d.addCallback(lambda res:
2371 self.failUnlessNodeKeysAre(self._foo_node,
2372 [u"bar.txt", u"baz.txt", u"blockingfile",
2373 u"empty", u"n\u00fc.txt", u"quux.txt",
2377 def test_POST_FILEURL_check(self):
2378 bar_url = self.public_url + "/foo/bar.txt"
2379 d = self.POST(bar_url, t="check")
2381 self.failUnlessIn("Healthy :", res)
2382 d.addCallback(_check)
2383 redir_url = "http://allmydata.org/TARGET"
2384 def _check2(statuscode, target):
2385 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2386 self.failUnlessReallyEqual(target, redir_url)
2387 d.addCallback(lambda res:
2388 self.shouldRedirect2("test_POST_FILEURL_check",
2392 when_done=redir_url))
2393 d.addCallback(lambda res:
2394 self.POST(bar_url, t="check", return_to=redir_url))
2396 self.failUnlessIn("Healthy :", res)
2397 self.failUnlessIn("Return to file", res)
2398 self.failUnlessIn(redir_url, res)
2399 d.addCallback(_check3)
2401 d.addCallback(lambda res:
2402 self.POST(bar_url, t="check", output="JSON"))
2403 def _check_json(res):
2404 data = simplejson.loads(res)
2405 self.failUnlessIn("storage-index", data)
2406 self.failUnless(data["results"]["healthy"])
2407 d.addCallback(_check_json)
2411 def test_POST_FILEURL_check_and_repair(self):
2412 bar_url = self.public_url + "/foo/bar.txt"
2413 d = self.POST(bar_url, t="check", repair="true")
2415 self.failUnlessIn("Healthy :", res)
2416 d.addCallback(_check)
2417 redir_url = "http://allmydata.org/TARGET"
2418 def _check2(statuscode, target):
2419 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2420 self.failUnlessReallyEqual(target, redir_url)
2421 d.addCallback(lambda res:
2422 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2425 t="check", repair="true",
2426 when_done=redir_url))
2427 d.addCallback(lambda res:
2428 self.POST(bar_url, t="check", return_to=redir_url))
2430 self.failUnlessIn("Healthy :", res)
2431 self.failUnlessIn("Return to file", res)
2432 self.failUnlessIn(redir_url, res)
2433 d.addCallback(_check3)
2436 def test_POST_DIRURL_check(self):
2437 foo_url = self.public_url + "/foo/"
2438 d = self.POST(foo_url, t="check")
2440 self.failUnlessIn("Healthy :", res)
2441 d.addCallback(_check)
2442 redir_url = "http://allmydata.org/TARGET"
2443 def _check2(statuscode, target):
2444 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2445 self.failUnlessReallyEqual(target, redir_url)
2446 d.addCallback(lambda res:
2447 self.shouldRedirect2("test_POST_DIRURL_check",
2451 when_done=redir_url))
2452 d.addCallback(lambda res:
2453 self.POST(foo_url, t="check", return_to=redir_url))
2455 self.failUnlessIn("Healthy :", res)
2456 self.failUnlessIn("Return to file/directory", res)
2457 self.failUnlessIn(redir_url, res)
2458 d.addCallback(_check3)
2460 d.addCallback(lambda res:
2461 self.POST(foo_url, t="check", output="JSON"))
2462 def _check_json(res):
2463 data = simplejson.loads(res)
2464 self.failUnlessIn("storage-index", data)
2465 self.failUnless(data["results"]["healthy"])
2466 d.addCallback(_check_json)
2470 def test_POST_DIRURL_check_and_repair(self):
2471 foo_url = self.public_url + "/foo/"
2472 d = self.POST(foo_url, t="check", repair="true")
2474 self.failUnlessIn("Healthy :", res)
2475 d.addCallback(_check)
2476 redir_url = "http://allmydata.org/TARGET"
2477 def _check2(statuscode, target):
2478 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2479 self.failUnlessReallyEqual(target, redir_url)
2480 d.addCallback(lambda res:
2481 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2484 t="check", repair="true",
2485 when_done=redir_url))
2486 d.addCallback(lambda res:
2487 self.POST(foo_url, t="check", return_to=redir_url))
2489 self.failUnlessIn("Healthy :", res)
2490 self.failUnlessIn("Return to file/directory", res)
2491 self.failUnlessIn(redir_url, res)
2492 d.addCallback(_check3)
2495 def test_POST_FILEURL_mdmf_check(self):
2496 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2497 d = self.POST(quux_url, t="check")
2499 self.failUnlessIn("Healthy", res)
2500 d.addCallback(_check)
2501 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2502 d.addCallback(lambda ignored:
2503 self.POST(quux_extension_url, t="check"))
2504 d.addCallback(_check)
2507 def test_POST_FILEURL_mdmf_check_and_repair(self):
2508 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2509 d = self.POST(quux_url, t="check", repair="true")
2511 self.failUnlessIn("Healthy", res)
2512 d.addCallback(_check)
2513 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2514 d.addCallback(lambda ignored:
2515 self.POST(quux_extension_url, t="check", repair="true"))
2516 d.addCallback(_check)
2519 def wait_for_operation(self, ignored, ophandle):
2520 url = "/operations/" + ophandle
2521 url += "?t=status&output=JSON"
2524 data = simplejson.loads(res)
2525 if not data["finished"]:
2526 d = self.stall(delay=1.0)
2527 d.addCallback(self.wait_for_operation, ophandle)
2533 def get_operation_results(self, ignored, ophandle, output=None):
2534 url = "/operations/" + ophandle
2537 url += "&output=" + output
2540 if output and output.lower() == "json":
2541 return simplejson.loads(res)
2546 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2547 d = self.shouldFail2(error.Error,
2548 "test_POST_DIRURL_deepcheck_no_ophandle",
2550 "slow operation requires ophandle=",
2551 self.POST, self.public_url, t="start-deep-check")
2554 def test_POST_DIRURL_deepcheck(self):
2555 def _check_redirect(statuscode, target):
2556 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2557 self.failUnless(target.endswith("/operations/123"))
2558 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2559 self.POST, self.public_url,
2560 t="start-deep-check", ophandle="123")
2561 d.addCallback(self.wait_for_operation, "123")
2562 def _check_json(data):
2563 self.failUnlessReallyEqual(data["finished"], True)
2564 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2565 self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2566 d.addCallback(_check_json)
2567 d.addCallback(self.get_operation_results, "123", "html")
2568 def _check_html(res):
2569 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2570 self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2571 self.failUnlessIn(FAVICON_MARKUP, res)
2572 d.addCallback(_check_html)
2574 d.addCallback(lambda res:
2575 self.GET("/operations/123/"))
2576 d.addCallback(_check_html) # should be the same as without the slash
2578 d.addCallback(lambda res:
2579 self.shouldFail2(error.Error, "one", "404 Not Found",
2580 "No detailed results for SI bogus",
2581 self.GET, "/operations/123/bogus"))
2583 foo_si = self._foo_node.get_storage_index()
2584 foo_si_s = base32.b2a(foo_si)
2585 d.addCallback(lambda res:
2586 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2587 def _check_foo_json(res):
2588 data = simplejson.loads(res)
2589 self.failUnlessEqual(data["storage-index"], foo_si_s)
2590 self.failUnless(data["results"]["healthy"])
2591 d.addCallback(_check_foo_json)
2594 def test_POST_DIRURL_deepcheck_and_repair(self):
2595 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2596 ophandle="124", output="json", followRedirect=True)
2597 d.addCallback(self.wait_for_operation, "124")
2598 def _check_json(data):
2599 self.failUnlessReallyEqual(data["finished"], True)
2600 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2601 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2602 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2603 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2604 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2605 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2606 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2607 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2608 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2609 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2610 d.addCallback(_check_json)
2611 d.addCallback(self.get_operation_results, "124", "html")
2612 def _check_html(res):
2613 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2615 self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2616 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2617 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2619 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2620 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2621 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2623 self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2624 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2625 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2627 self.failUnlessIn(FAVICON_MARKUP, res)
2628 d.addCallback(_check_html)
2631 def test_POST_FILEURL_bad_t(self):
2632 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2633 "POST to file: bad t=bogus",
2634 self.POST, self.public_url + "/foo/bar.txt",
2638 def test_POST_mkdir(self): # return value?
2639 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2640 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2641 d.addCallback(self.failUnlessNodeKeysAre, [])
2644 def test_POST_mkdir_mdmf(self):
2645 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2646 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2647 d.addCallback(lambda node:
2648 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2651 def test_POST_mkdir_sdmf(self):
2652 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2653 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2654 d.addCallback(lambda node:
2655 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2658 def test_POST_mkdir_bad_format(self):
2659 return self.shouldHTTPError("POST_mkdir_bad_format",
2660 400, "Bad Request", "Unknown format: foo",
2661 self.POST, self.public_url +
2662 "/foo?t=mkdir&name=newdir&format=foo")
2664 def test_POST_mkdir_initial_children(self):
2665 (newkids, caps) = self._create_initial_children()
2666 d = self.POST2(self.public_url +
2667 "/foo?t=mkdir-with-children&name=newdir",
2668 simplejson.dumps(newkids))
2669 d.addCallback(lambda res:
2670 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2671 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2672 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2673 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2674 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2677 def test_POST_mkdir_initial_children_mdmf(self):
2678 (newkids, caps) = self._create_initial_children()
2679 d = self.POST2(self.public_url +
2680 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2681 simplejson.dumps(newkids))
2682 d.addCallback(lambda res:
2683 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2684 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2685 d.addCallback(lambda node:
2686 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2687 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2688 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2693 def test_POST_mkdir_initial_children_sdmf(self):
2694 (newkids, caps) = self._create_initial_children()
2695 d = self.POST2(self.public_url +
2696 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2697 simplejson.dumps(newkids))
2698 d.addCallback(lambda res:
2699 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2700 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2701 d.addCallback(lambda node:
2702 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2703 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2704 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2708 def test_POST_mkdir_initial_children_bad_format(self):
2709 (newkids, caps) = self._create_initial_children()
2710 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2711 400, "Bad Request", "Unknown format: foo",
2712 self.POST, self.public_url + \
2713 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2714 simplejson.dumps(newkids))
2716 def test_POST_mkdir_immutable(self):
2717 (newkids, caps) = self._create_immutable_children()
2718 d = self.POST2(self.public_url +
2719 "/foo?t=mkdir-immutable&name=newdir",
2720 simplejson.dumps(newkids))
2721 d.addCallback(lambda res:
2722 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2723 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2724 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2725 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2726 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2727 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2728 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2729 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2730 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2731 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2732 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2733 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2734 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2737 def test_POST_mkdir_immutable_bad(self):
2738 (newkids, caps) = self._create_initial_children()
2739 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2741 "needed to be immutable but was not",
2744 "/foo?t=mkdir-immutable&name=newdir",
2745 simplejson.dumps(newkids))
2748 def test_POST_mkdir_2(self):
2749 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2750 d.addCallback(lambda res:
2751 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753 d.addCallback(self.failUnlessNodeKeysAre, [])
2756 def test_POST_mkdirs_2(self):
2757 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2758 d.addCallback(lambda res:
2759 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2760 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2761 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2762 d.addCallback(self.failUnlessNodeKeysAre, [])
2765 def test_POST_mkdir_no_parentdir_noredirect(self):
2766 d = self.POST("/uri?t=mkdir")
2767 def _after_mkdir(res):
2768 uri.DirectoryURI.init_from_string(res)
2769 d.addCallback(_after_mkdir)
2772 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2773 d = self.POST("/uri?t=mkdir&format=mdmf")
2774 def _after_mkdir(res):
2775 u = uri.from_string(res)
2776 # Check that this is an MDMF writecap
2777 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2778 d.addCallback(_after_mkdir)
2781 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2782 d = self.POST("/uri?t=mkdir&format=sdmf")
2783 def _after_mkdir(res):
2784 u = uri.from_string(res)
2785 self.failUnlessIsInstance(u, uri.DirectoryURI)
2786 d.addCallback(_after_mkdir)
2789 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2790 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2791 400, "Bad Request", "Unknown format: foo",
2792 self.POST, self.public_url +
2793 "/uri?t=mkdir&format=foo")
2795 def test_POST_mkdir_no_parentdir_noredirect2(self):
2796 # make sure form-based arguments (as on the welcome page) still work
2797 d = self.POST("/uri", t="mkdir")
2798 def _after_mkdir(res):
2799 uri.DirectoryURI.init_from_string(res)
2800 d.addCallback(_after_mkdir)
2801 d.addErrback(self.explain_web_error)
2804 def test_POST_mkdir_no_parentdir_redirect(self):
2805 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2806 d.addBoth(self.shouldRedirect, None, statuscode='303')
2807 def _check_target(target):
2808 target = urllib.unquote(target)
2809 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2810 d.addCallback(_check_target)
2813 def test_POST_mkdir_no_parentdir_redirect2(self):
2814 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2815 d.addBoth(self.shouldRedirect, None, statuscode='303')
2816 def _check_target(target):
2817 target = urllib.unquote(target)
2818 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2819 d.addCallback(_check_target)
2820 d.addErrback(self.explain_web_error)
2823 def _make_readonly(self, u):
2824 ro_uri = uri.from_string(u).get_readonly()
2827 return ro_uri.to_string()
2829 def _create_initial_children(self):
2830 contents, n, filecap1 = self.makefile(12)
2831 md1 = {"metakey1": "metavalue1"}
2832 filecap2 = make_mutable_file_uri()
2833 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2834 filecap3 = node3.get_readonly_uri()
2835 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2836 dircap = DirectoryNode(node4, None, None).get_uri()
2837 mdmfcap = make_mutable_file_uri(mdmf=True)
2838 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2839 emptydircap = "URI:DIR2-LIT:"
2840 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
2841 "ro_uri": self._make_readonly(filecap1),
2842 "metadata": md1, }],
2843 u"child-mutable": ["filenode", {"rw_uri": filecap2,
2844 "ro_uri": self._make_readonly(filecap2)}],
2845 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2846 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
2847 "ro_uri": unknown_rocap}],
2848 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
2849 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2850 u"dirchild": ["dirnode", {"rw_uri": dircap,
2851 "ro_uri": self._make_readonly(dircap)}],
2852 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2853 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2854 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2855 "ro_uri": self._make_readonly(mdmfcap)}],
2857 return newkids, {'filecap1': filecap1,
2858 'filecap2': filecap2,
2859 'filecap3': filecap3,
2860 'unknown_rwcap': unknown_rwcap,
2861 'unknown_rocap': unknown_rocap,
2862 'unknown_immcap': unknown_immcap,
2864 'litdircap': litdircap,
2865 'emptydircap': emptydircap,
2868 def _create_immutable_children(self):
2869 contents, n, filecap1 = self.makefile(12)
2870 md1 = {"metakey1": "metavalue1"}
2871 tnode = create_chk_filenode("immutable directory contents\n"*10)
2872 dnode = DirectoryNode(tnode, None, None)
2873 assert not dnode.is_mutable()
2874 immdircap = dnode.get_uri()
2875 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2876 emptydircap = "URI:DIR2-LIT:"
2877 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2878 "metadata": md1, }],
2879 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2880 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2881 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2882 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2884 return newkids, {'filecap1': filecap1,
2885 'unknown_immcap': unknown_immcap,
2886 'immdircap': immdircap,
2887 'litdircap': litdircap,
2888 'emptydircap': emptydircap}
2890 def test_POST_mkdir_no_parentdir_initial_children(self):
2891 (newkids, caps) = self._create_initial_children()
2892 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2893 def _after_mkdir(res):
2894 self.failUnless(res.startswith("URI:DIR"), res)
2895 n = self.s.create_node_from_uri(res)
2896 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2897 d2.addCallback(lambda ign:
2898 self.failUnlessROChildURIIs(n, u"child-imm",
2900 d2.addCallback(lambda ign:
2901 self.failUnlessRWChildURIIs(n, u"child-mutable",
2903 d2.addCallback(lambda ign:
2904 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2906 d2.addCallback(lambda ign:
2907 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2908 caps['unknown_rwcap']))
2909 d2.addCallback(lambda ign:
2910 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2911 caps['unknown_rocap']))
2912 d2.addCallback(lambda ign:
2913 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2914 caps['unknown_immcap']))
2915 d2.addCallback(lambda ign:
2916 self.failUnlessRWChildURIIs(n, u"dirchild",
2919 d.addCallback(_after_mkdir)
2922 def test_POST_mkdir_no_parentdir_unexpected_children(self):
2923 # the regular /uri?t=mkdir operation is specified to ignore its body.
2924 # Only t=mkdir-with-children pays attention to it.
2925 (newkids, caps) = self._create_initial_children()
2926 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2928 "t=mkdir does not accept children=, "
2929 "try t=mkdir-with-children instead",
2930 self.POST2, "/uri?t=mkdir", # without children
2931 simplejson.dumps(newkids))
2934 def test_POST_noparent_bad(self):
2935 d = self.shouldHTTPError("POST_noparent_bad",
2937 "/uri accepts only PUT, PUT?t=mkdir, "
2938 "POST?t=upload, and POST?t=mkdir",
2939 self.POST, "/uri?t=bogus")
2942 def test_POST_mkdir_no_parentdir_immutable(self):
2943 (newkids, caps) = self._create_immutable_children()
2944 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2945 def _after_mkdir(res):
2946 self.failUnless(res.startswith("URI:DIR"), res)
2947 n = self.s.create_node_from_uri(res)
2948 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2949 d2.addCallback(lambda ign:
2950 self.failUnlessROChildURIIs(n, u"child-imm",
2952 d2.addCallback(lambda ign:
2953 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2954 caps['unknown_immcap']))
2955 d2.addCallback(lambda ign:
2956 self.failUnlessROChildURIIs(n, u"dirchild-imm",
2958 d2.addCallback(lambda ign:
2959 self.failUnlessROChildURIIs(n, u"dirchild-lit",
2961 d2.addCallback(lambda ign:
2962 self.failUnlessROChildURIIs(n, u"dirchild-empty",
2963 caps['emptydircap']))
2965 d.addCallback(_after_mkdir)
2968 def test_POST_mkdir_no_parentdir_immutable_bad(self):
2969 (newkids, caps) = self._create_initial_children()
2970 d = self.shouldFail2(error.Error,
2971 "test_POST_mkdir_no_parentdir_immutable_bad",
2973 "needed to be immutable but was not",
2975 "/uri?t=mkdir-immutable",
2976 simplejson.dumps(newkids))
2979 def test_welcome_page_mkdir_button(self):
2980 # Fetch the welcome page.
2982 def _after_get_welcome_page(res):
2983 MKDIR_BUTTON_RE = re.compile(
2984 '<form action="([^"]*)" method="post".*?'
2985 '<input type="hidden" name="t" value="([^"]*)" />'
2986 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2987 '<input type="submit" value="Create a directory" />',
2989 mo = MKDIR_BUTTON_RE.search(res)
2990 formaction = mo.group(1)
2992 formaname = mo.group(3)
2993 formavalue = mo.group(4)
2994 return (formaction, formt, formaname, formavalue)
2995 d.addCallback(_after_get_welcome_page)
2996 def _after_parse_form(res):
2997 (formaction, formt, formaname, formavalue) = res
2998 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2999 d.addCallback(_after_parse_form)
3000 d.addBoth(self.shouldRedirect, None, statuscode='303')
3003 def test_POST_mkdir_replace(self): # return value?
3004 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3005 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3006 d.addCallback(self.failUnlessNodeKeysAre, [])
3009 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3010 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3011 d.addBoth(self.shouldFail, error.Error,
3012 "POST_mkdir_no_replace_queryarg",
3014 "There was already a child by that name, and you asked me "
3015 "to not replace it")
3016 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3017 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3020 def test_POST_mkdir_no_replace_field(self): # return value?
3021 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3023 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3025 "There was already a child by that name, and you asked me "
3026 "to not replace it")
3027 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3028 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3031 def test_POST_mkdir_whendone_field(self):
3032 d = self.POST(self.public_url + "/foo",
3033 t="mkdir", name="newdir", when_done="/THERE")
3034 d.addBoth(self.shouldRedirect, "/THERE")
3035 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3036 d.addCallback(self.failUnlessNodeKeysAre, [])
3039 def test_POST_mkdir_whendone_queryarg(self):
3040 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3041 t="mkdir", name="newdir")
3042 d.addBoth(self.shouldRedirect, "/THERE")
3043 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3044 d.addCallback(self.failUnlessNodeKeysAre, [])
3047 def test_POST_bad_t(self):
3048 d = self.shouldFail2(error.Error, "POST_bad_t",
3050 "POST to a directory with bad t=BOGUS",
3051 self.POST, self.public_url + "/foo", t="BOGUS")
3054 def test_POST_set_children(self, command_name="set_children"):
3055 contents9, n9, newuri9 = self.makefile(9)
3056 contents10, n10, newuri10 = self.makefile(10)
3057 contents11, n11, newuri11 = self.makefile(11)
3060 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3063 "ctime": 1002777696.7564139,
3064 "mtime": 1002777696.7564139
3067 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3070 "ctime": 1002777696.7564139,
3071 "mtime": 1002777696.7564139
3074 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3077 "ctime": 1002777696.7564139,
3078 "mtime": 1002777696.7564139
3081 }""" % (newuri9, newuri10, newuri11)
3083 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3085 d = client.getPage(url, method="POST", postdata=reqbody)
3087 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3088 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3089 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3091 d.addCallback(_then)
3092 d.addErrback(self.dump_error)
3095 def test_POST_set_children_with_hyphen(self):
3096 return self.test_POST_set_children(command_name="set-children")
3098 def test_POST_link_uri(self):
3099 contents, n, newuri = self.makefile(8)
3100 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3101 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3102 d.addCallback(lambda res:
3103 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3107 def test_POST_link_uri_replace(self):
3108 contents, n, newuri = self.makefile(8)
3109 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3110 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3111 d.addCallback(lambda res:
3112 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3116 def test_POST_link_uri_unknown_bad(self):
3117 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3118 d.addBoth(self.shouldFail, error.Error,
3119 "POST_link_uri_unknown_bad",
3121 "unknown cap in a write slot")
3124 def test_POST_link_uri_unknown_ro_good(self):
3125 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3126 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3129 def test_POST_link_uri_unknown_imm_good(self):
3130 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3131 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3134 def test_POST_link_uri_no_replace_queryarg(self):
3135 contents, n, newuri = self.makefile(8)
3136 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3137 name="bar.txt", uri=newuri)
3138 d.addBoth(self.shouldFail, error.Error,
3139 "POST_link_uri_no_replace_queryarg",
3141 "There was already a child by that name, and you asked me "
3142 "to not replace it")
3143 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3144 d.addCallback(self.failUnlessIsBarDotTxt)
3147 def test_POST_link_uri_no_replace_field(self):
3148 contents, n, newuri = self.makefile(8)
3149 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3150 name="bar.txt", uri=newuri)
3151 d.addBoth(self.shouldFail, error.Error,
3152 "POST_link_uri_no_replace_field",
3154 "There was already a child by that name, and you asked me "
3155 "to not replace it")
3156 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3157 d.addCallback(self.failUnlessIsBarDotTxt)
3160 def test_POST_delete(self, command_name='delete'):
3161 d = self._foo_node.list()
3162 def _check_before(children):
3163 self.failUnlessIn(u"bar.txt", children)
3164 d.addCallback(_check_before)
3165 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3166 d.addCallback(lambda res: self._foo_node.list())
3167 def _check_after(children):
3168 self.failIfIn(u"bar.txt", children)
3169 d.addCallback(_check_after)
3172 def test_POST_unlink(self):
3173 return self.test_POST_delete(command_name='unlink')
3175 def test_POST_rename_file(self):
3176 d = self.POST(self.public_url + "/foo", t="rename",
3177 from_name="bar.txt", to_name='wibble.txt')
3178 d.addCallback(lambda res:
3179 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3180 d.addCallback(lambda res:
3181 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3182 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3183 d.addCallback(self.failUnlessIsBarDotTxt)
3184 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3185 d.addCallback(self.failUnlessIsBarJSON)
3188 def test_POST_rename_file_redundant(self):
3189 d = self.POST(self.public_url + "/foo", t="rename",
3190 from_name="bar.txt", to_name='bar.txt')
3191 d.addCallback(lambda res:
3192 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3193 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3194 d.addCallback(self.failUnlessIsBarDotTxt)
3195 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3196 d.addCallback(self.failUnlessIsBarJSON)
3199 def test_POST_rename_file_replace(self):
3200 # rename a file and replace a directory with it
3201 d = self.POST(self.public_url + "/foo", t="rename",
3202 from_name="bar.txt", to_name='empty')
3203 d.addCallback(lambda res:
3204 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3205 d.addCallback(lambda res:
3206 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3207 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3208 d.addCallback(self.failUnlessIsBarDotTxt)
3209 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3210 d.addCallback(self.failUnlessIsBarJSON)
3213 def test_POST_rename_file_no_replace_queryarg(self):
3214 # rename a file and replace a directory with it
3215 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3216 from_name="bar.txt", to_name='empty')
3217 d.addBoth(self.shouldFail, error.Error,
3218 "POST_rename_file_no_replace_queryarg",
3220 "There was already a child by that name, and you asked me "
3221 "to not replace it")
3222 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3223 d.addCallback(self.failUnlessIsEmptyJSON)
3226 def test_POST_rename_file_no_replace_field(self):
3227 # rename a file and replace a directory with it
3228 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3229 from_name="bar.txt", to_name='empty')
3230 d.addBoth(self.shouldFail, error.Error,
3231 "POST_rename_file_no_replace_field",
3233 "There was already a child by that name, and you asked me "
3234 "to not replace it")
3235 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3236 d.addCallback(self.failUnlessIsEmptyJSON)
3239 def failUnlessIsEmptyJSON(self, res):
3240 data = simplejson.loads(res)
3241 self.failUnlessEqual(data[0], "dirnode", data)
3242 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3244 def test_POST_rename_file_slash_fail(self):
3245 d = self.POST(self.public_url + "/foo", t="rename",
3246 from_name="bar.txt", to_name='kirk/spock.txt')
3247 d.addBoth(self.shouldFail, error.Error,
3248 "test_POST_rename_file_slash_fail",
3250 "to_name= may not contain a slash",
3252 d.addCallback(lambda res:
3253 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3256 def test_POST_rename_dir(self):
3257 d = self.POST(self.public_url, t="rename",
3258 from_name="foo", to_name='plunk')
3259 d.addCallback(lambda res:
3260 self.failIfNodeHasChild(self.public_root, u"foo"))
3261 d.addCallback(lambda res:
3262 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3263 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3264 d.addCallback(self.failUnlessIsFooJSON)
3267 def test_POST_move_file(self):
3268 d = self.POST(self.public_url + "/foo", t="move",
3269 from_name="bar.txt", to_dir="sub")
3270 d.addCallback(lambda res:
3271 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3272 d.addCallback(lambda res:
3273 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3274 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3275 d.addCallback(self.failUnlessIsBarDotTxt)
3276 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3277 d.addCallback(self.failUnlessIsBarJSON)
3280 def test_POST_move_file_new_name(self):
3281 d = self.POST(self.public_url + "/foo", t="move",
3282 from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3283 d.addCallback(lambda res:
3284 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3285 d.addCallback(lambda res:
3286 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3287 d.addCallback(lambda res:
3288 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3289 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3290 d.addCallback(self.failUnlessIsBarDotTxt)
3291 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3292 d.addCallback(self.failUnlessIsBarJSON)
3295 def test_POST_move_file_replace(self):
3296 d = self.POST(self.public_url + "/foo", t="move",
3297 from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3298 d.addCallback(lambda res:
3299 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3300 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3301 d.addCallback(self.failUnlessIsBarDotTxt)
3302 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3303 d.addCallback(self.failUnlessIsBarJSON)
3306 def test_POST_move_file_no_replace(self):
3307 d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3309 "There was already a child by that name, and you asked me to not replace it",
3310 self.POST, self.public_url + "/foo", t="move",
3311 replace="false", from_name="bar.txt",
3312 to_name="baz.txt", to_dir="sub")
3313 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3314 d.addCallback(self.failUnlessIsBarDotTxt)
3315 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3316 d.addCallback(self.failUnlessIsBarJSON)
3317 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3318 d.addCallback(self.failUnlessIsSubBazDotTxt)
3321 def test_POST_move_file_slash_fail(self):
3322 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3324 "to_name= may not contain a slash",
3325 self.POST, self.public_url + "/foo", t="move",
3326 from_name="bar.txt",
3327 to_name="slash/fail.txt", to_dir="sub")
3328 d.addCallback(lambda res:
3329 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3330 d.addCallback(lambda res:
3331 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3332 d.addCallback(lambda ign:
3333 self.shouldFail2(error.Error,
3334 "test_POST_rename_file_slash_fail2",
3336 "from_name= may not contain a slash",
3337 self.POST, self.public_url + "/foo",
3339 from_name="nope/bar.txt",
3340 to_name="fail.txt", to_dir="sub"))
3343 def test_POST_move_file_no_target(self):
3344 d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3346 "move requires from_name and to_dir",
3347 self.POST, self.public_url + "/foo", t="move",
3348 from_name="bar.txt", to_name="baz.txt")
3349 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3350 d.addCallback(self.failUnlessIsBarDotTxt)
3351 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3352 d.addCallback(self.failUnlessIsBarJSON)
3353 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3354 d.addCallback(self.failUnlessIsBazDotTxt)
3357 def test_POST_move_file_bad_target_type(self):
3358 d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3359 "400 Bad Request", "invalid target_type parameter",
3361 self.public_url + "/foo", t="move",
3362 target_type="*D", from_name="bar.txt",
3366 def test_POST_move_file_multi_level(self):
3367 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3368 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3369 from_name="bar.txt", to_dir="sub/level2"))
3370 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3371 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3372 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3373 d.addCallback(self.failUnlessIsBarDotTxt)
3374 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3375 d.addCallback(self.failUnlessIsBarJSON)
3378 def test_POST_move_file_to_uri(self):
3379 d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3380 from_name="bar.txt", to_dir=self._sub_uri)
3381 d.addCallback(lambda res:
3382 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3383 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3384 d.addCallback(self.failUnlessIsBarDotTxt)
3385 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3386 d.addCallback(self.failUnlessIsBarJSON)
3389 def test_POST_move_file_to_nonexist_dir(self):
3390 d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3391 "404 Not Found", "No such child: nopechucktesta",
3392 self.POST, self.public_url + "/foo", t="move",
3393 from_name="bar.txt", to_dir="nopechucktesta")
3396 def test_POST_move_file_into_file(self):
3397 d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3398 "400 Bad Request", "to_dir is not a directory",
3399 self.POST, self.public_url + "/foo", t="move",
3400 from_name="bar.txt", to_dir="baz.txt")
3401 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3402 d.addCallback(self.failUnlessIsBazDotTxt)
3403 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3404 d.addCallback(self.failUnlessIsBarDotTxt)
3405 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3406 d.addCallback(self.failUnlessIsBarJSON)
3409 def test_POST_move_file_to_bad_uri(self):
3410 d = self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3411 "400 Bad Request", "to_dir is not a directory",
3412 self.POST, self.public_url + "/foo", t="move",
3413 from_name="bar.txt", target_type="uri",
3414 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3415 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3416 d.addCallback(self.failUnlessIsBarDotTxt)
3417 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3418 d.addCallback(self.failUnlessIsBarJSON)
3421 def test_POST_move_dir(self):
3422 d = self.POST(self.public_url + "/foo", t="move",
3423 from_name="bar.txt", to_dir="empty")
3424 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3425 t="move", from_name="empty", to_dir="sub"))
3426 d.addCallback(lambda res:
3427 self.failIfNodeHasChild(self._foo_node, u"empty"))
3428 d.addCallback(lambda res:
3429 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3430 d.addCallback(lambda res:
3431 self._sub_node.get_child_at_path(u"empty"))
3432 d.addCallback(lambda node:
3433 self.failUnlessNodeHasChild(node, u"bar.txt"))
3434 d.addCallback(lambda res:
3435 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3436 d.addCallback(self.failUnlessIsBarDotTxt)
3439 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3440 """ If target is not None then the redirection has to go to target. If
3441 statuscode is not None then the redirection has to be accomplished with
3442 that HTTP status code."""
3443 if not isinstance(res, failure.Failure):
3444 to_where = (target is None) and "somewhere" or ("to " + target)
3445 self.fail("%s: we were expecting to get redirected %s, not get an"
3446 " actual page: %s" % (which, to_where, res))
3447 res.trap(error.PageRedirect)
3448 if statuscode is not None:
3449 self.failUnlessReallyEqual(res.value.status, statuscode,
3450 "%s: not a redirect" % which)
3451 if target is not None:
3452 # the PageRedirect does not seem to capture the uri= query arg
3453 # properly, so we can't check for it.
3454 realtarget = self.webish_url + target
3455 self.failUnlessReallyEqual(res.value.location, realtarget,
3456 "%s: wrong target" % which)
3457 return res.value.location
3459 def test_GET_URI_form(self):
3460 base = "/uri?uri=%s" % self._bar_txt_uri
3461 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3462 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3464 d.addBoth(self.shouldRedirect, targetbase)
3465 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3466 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3467 d.addCallback(lambda res: self.GET(base+"&t=json"))
3468 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3469 d.addCallback(self.log, "about to get file by uri")
3470 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3471 d.addCallback(self.failUnlessIsBarDotTxt)
3472 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3473 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3474 followRedirect=True))
3475 d.addCallback(self.failUnlessIsFooJSON)
3476 d.addCallback(self.log, "got dir by uri")
3480 def test_GET_URI_form_bad(self):
3481 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3482 "400 Bad Request", "GET /uri requires uri=",
3486 def test_GET_rename_form(self):
3487 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3488 followRedirect=True)
3490 self.failUnlessIn('name="when_done" value="."', res)
3491 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3492 self.failUnlessIn(FAVICON_MARKUP, res)
3493 d.addCallback(_check)
3496 def log(self, res, msg):
3497 #print "MSG: %s RES: %s" % (msg, res)
3501 def test_GET_URI_URL(self):
3502 base = "/uri/%s" % self._bar_txt_uri
3504 d.addCallback(self.failUnlessIsBarDotTxt)
3505 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3506 d.addCallback(self.failUnlessIsBarDotTxt)
3507 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3508 d.addCallback(self.failUnlessIsBarDotTxt)
3511 def test_GET_URI_URL_dir(self):
3512 base = "/uri/%s?t=json" % self._foo_uri
3514 d.addCallback(self.failUnlessIsFooJSON)
3517 def test_GET_URI_URL_missing(self):
3518 base = "/uri/%s" % self._bad_file_uri
3519 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3520 http.GONE, None, "NotEnoughSharesError",
3522 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3523 # here? we must arrange for a download to fail after target.open()
3524 # has been called, and then inspect the response to see that it is
3525 # shorter than we expected.
3528 def test_PUT_DIRURL_uri(self):
3529 d = self.s.create_dirnode()
3531 new_uri = dn.get_uri()
3532 # replace /foo with a new (empty) directory
3533 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3534 d.addCallback(lambda res:
3535 self.failUnlessReallyEqual(res.strip(), new_uri))
3536 d.addCallback(lambda res:
3537 self.failUnlessRWChildURIIs(self.public_root,
3541 d.addCallback(_made_dir)
3544 def test_PUT_DIRURL_uri_noreplace(self):
3545 d = self.s.create_dirnode()
3547 new_uri = dn.get_uri()
3548 # replace /foo with a new (empty) directory, but ask that
3549 # replace=false, so it should fail
3550 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3551 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3553 self.public_url + "/foo?t=uri&replace=false",
3555 d.addCallback(lambda res:
3556 self.failUnlessRWChildURIIs(self.public_root,
3560 d.addCallback(_made_dir)
3563 def test_PUT_DIRURL_bad_t(self):
3564 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3565 "400 Bad Request", "PUT to a directory",
3566 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3567 d.addCallback(lambda res:
3568 self.failUnlessRWChildURIIs(self.public_root,
3573 def test_PUT_NEWFILEURL_uri(self):
3574 contents, n, new_uri = self.makefile(8)
3575 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3576 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3577 d.addCallback(lambda res:
3578 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3582 def test_PUT_NEWFILEURL_mdmf(self):
3583 new_contents = self.NEWFILE_CONTENTS * 300000
3584 d = self.PUT(self.public_url + \
3585 "/foo/mdmf.txt?format=mdmf",
3587 d.addCallback(lambda ignored:
3588 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3589 def _got_json(json):
3590 data = simplejson.loads(json)
3592 self.failUnlessIn("format", data)
3593 self.failUnlessEqual(data["format"], "MDMF")
3594 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3595 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3596 d.addCallback(_got_json)
3599 def test_PUT_NEWFILEURL_sdmf(self):
3600 new_contents = self.NEWFILE_CONTENTS * 300000
3601 d = self.PUT(self.public_url + \
3602 "/foo/sdmf.txt?format=sdmf",
3604 d.addCallback(lambda ignored:
3605 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3606 def _got_json(json):
3607 data = simplejson.loads(json)
3609 self.failUnlessIn("format", data)
3610 self.failUnlessEqual(data["format"], "SDMF")
3611 d.addCallback(_got_json)
3614 def test_PUT_NEWFILEURL_bad_format(self):
3615 new_contents = self.NEWFILE_CONTENTS * 300000
3616 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3617 400, "Bad Request", "Unknown format: foo",
3618 self.PUT, self.public_url + \
3619 "/foo/foo.txt?format=foo",
3622 def test_PUT_NEWFILEURL_uri_replace(self):
3623 contents, n, new_uri = self.makefile(8)
3624 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3625 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3626 d.addCallback(lambda res:
3627 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3631 def test_PUT_NEWFILEURL_uri_no_replace(self):
3632 contents, n, new_uri = self.makefile(8)
3633 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3634 d.addBoth(self.shouldFail, error.Error,
3635 "PUT_NEWFILEURL_uri_no_replace",
3637 "There was already a child by that name, and you asked me "
3638 "to not replace it")
3641 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3642 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3643 d.addBoth(self.shouldFail, error.Error,
3644 "POST_put_uri_unknown_bad",
3646 "unknown cap in a write slot")
3649 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3650 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3651 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3652 u"put-future-ro.txt")
3655 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3656 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3657 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3658 u"put-future-imm.txt")
3661 def test_PUT_NEWFILE_URI(self):
3662 file_contents = "New file contents here\n"
3663 d = self.PUT("/uri", file_contents)
3665 assert isinstance(uri, str), uri
3666 self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3667 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3669 return self.GET("/uri/%s" % uri)
3670 d.addCallback(_check)
3672 self.failUnlessReallyEqual(res, file_contents)
3673 d.addCallback(_check2)
3676 def test_PUT_NEWFILE_URI_not_mutable(self):
3677 file_contents = "New file contents here\n"
3678 d = self.PUT("/uri?mutable=false", file_contents)
3680 assert isinstance(uri, str), uri
3681 self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3682 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3684 return self.GET("/uri/%s" % uri)
3685 d.addCallback(_check)
3687 self.failUnlessReallyEqual(res, file_contents)
3688 d.addCallback(_check2)
3691 def test_PUT_NEWFILE_URI_only_PUT(self):
3692 d = self.PUT("/uri?t=bogus", "")
3693 d.addBoth(self.shouldFail, error.Error,
3694 "PUT_NEWFILE_URI_only_PUT",
3696 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3699 def test_PUT_NEWFILE_URI_mutable(self):
3700 file_contents = "New file contents here\n"
3701 d = self.PUT("/uri?mutable=true", file_contents)
3702 def _check1(filecap):
3703 filecap = filecap.strip()
3704 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3705 self.filecap = filecap
3706 u = uri.WriteableSSKFileURI.init_from_string(filecap)
3707 self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3708 n = self.s.create_node_from_uri(filecap)
3709 return n.download_best_version()
3710 d.addCallback(_check1)
3712 self.failUnlessReallyEqual(data, file_contents)
3713 return self.GET("/uri/%s" % urllib.quote(self.filecap))
3714 d.addCallback(_check2)
3716 self.failUnlessReallyEqual(res, file_contents)
3717 d.addCallback(_check3)
3720 def test_PUT_mkdir(self):
3721 d = self.PUT("/uri?t=mkdir", "")
3723 n = self.s.create_node_from_uri(uri.strip())
3724 d2 = self.failUnlessNodeKeysAre(n, [])
3725 d2.addCallback(lambda res:
3726 self.GET("/uri/%s?t=json" % uri))
3728 d.addCallback(_check)
3729 d.addCallback(self.failUnlessIsEmptyJSON)
3732 def test_PUT_mkdir_mdmf(self):
3733 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3735 u = uri.from_string(res)
3736 # Check that this is an MDMF writecap
3737 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3741 def test_PUT_mkdir_sdmf(self):
3742 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3744 u = uri.from_string(res)
3745 self.failUnlessIsInstance(u, uri.DirectoryURI)
3749 def test_PUT_mkdir_bad_format(self):
3750 return self.shouldHTTPError("PUT_mkdir_bad_format",
3751 400, "Bad Request", "Unknown format: foo",
3752 self.PUT, "/uri?t=mkdir&format=foo",
3755 def test_POST_check(self):
3756 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3758 # this returns a string form of the results, which are probably
3759 # None since we're using fake filenodes.
3760 # TODO: verify that the check actually happened, by changing
3761 # FakeCHKFileNode to count how many times .check() has been
3764 d.addCallback(_done)
3768 def test_PUT_update_at_offset(self):
3769 file_contents = "test file" * 100000 # about 900 KiB
3770 d = self.PUT("/uri?mutable=true", file_contents)
3772 self.filecap = filecap
3773 new_data = file_contents[:100]
3774 new = "replaced and so on"
3776 new_data += file_contents[len(new_data):]
3777 assert len(new_data) == len(file_contents)
3778 self.new_data = new_data
3779 d.addCallback(_then)
3780 d.addCallback(lambda ignored:
3781 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3782 "replaced and so on"))
3783 def _get_data(filecap):
3784 n = self.s.create_node_from_uri(filecap)
3785 return n.download_best_version()
3786 d.addCallback(_get_data)
3787 d.addCallback(lambda results:
3788 self.failUnlessEqual(results, self.new_data))
3789 # Now try appending things to the file
3790 d.addCallback(lambda ignored:
3791 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3793 d.addCallback(_get_data)
3794 d.addCallback(lambda results:
3795 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3796 # and try replacing the beginning of the file
3797 d.addCallback(lambda ignored:
3798 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3799 d.addCallback(_get_data)
3800 d.addCallback(lambda results:
3801 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3804 def test_PUT_update_at_invalid_offset(self):
3805 file_contents = "test file" * 100000 # about 900 KiB
3806 d = self.PUT("/uri?mutable=true", file_contents)
3808 self.filecap = filecap
3809 d.addCallback(_then)
3810 # Negative offsets should cause an error.
3811 d.addCallback(lambda ignored:
3812 self.shouldHTTPError("PUT_update_at_invalid_offset",
3816 "/uri/%s?offset=-1" % self.filecap,
3820 def test_PUT_update_at_offset_immutable(self):
3821 file_contents = "Test file" * 100000
3822 d = self.PUT("/uri", file_contents)
3824 self.filecap = filecap
3825 d.addCallback(_then)
3826 d.addCallback(lambda ignored:
3827 self.shouldHTTPError("PUT_update_at_offset_immutable",
3831 "/uri/%s?offset=50" % self.filecap,
3836 def test_bad_method(self):
3837 url = self.webish_url + self.public_url + "/foo/bar.txt"
3838 d = self.shouldHTTPError("bad_method",
3839 501, "Not Implemented",
3840 "I don't know how to treat a BOGUS request.",
3841 client.getPage, url, method="BOGUS")
3844 def test_short_url(self):
3845 url = self.webish_url + "/uri"
3846 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3847 "I don't know how to treat a DELETE request.",
3848 client.getPage, url, method="DELETE")
3851 def test_ophandle_bad(self):
3852 url = self.webish_url + "/operations/bogus?t=status"
3853 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3854 "unknown/expired handle 'bogus'",
3855 client.getPage, url)
3858 def test_ophandle_cancel(self):
3859 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3860 followRedirect=True)
3861 d.addCallback(lambda ignored:
3862 self.GET("/operations/128?t=status&output=JSON"))
3864 data = simplejson.loads(res)
3865 self.failUnless("finished" in data, res)
3866 monitor = self.ws.root.child_operations.handles["128"][0]
3867 d = self.POST("/operations/128?t=cancel&output=JSON")
3869 data = simplejson.loads(res)
3870 self.failUnless("finished" in data, res)
3871 # t=cancel causes the handle to be forgotten
3872 self.failUnless(monitor.is_cancelled())
3873 d.addCallback(_check2)
3875 d.addCallback(_check1)
3876 d.addCallback(lambda ignored:
3877 self.shouldHTTPError("ophandle_cancel",
3878 404, "404 Not Found",
3879 "unknown/expired handle '128'",
3881 "/operations/128?t=status&output=JSON"))
3884 def test_ophandle_retainfor(self):
3885 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3886 followRedirect=True)
3887 d.addCallback(lambda ignored:
3888 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3890 data = simplejson.loads(res)
3891 self.failUnless("finished" in data, res)
3892 d.addCallback(_check1)
3893 # the retain-for=0 will cause the handle to be expired very soon
3894 d.addCallback(lambda ign:
3895 self.clock.advance(2.0))
3896 d.addCallback(lambda ignored:
3897 self.shouldHTTPError("ophandle_retainfor",
3898 404, "404 Not Found",
3899 "unknown/expired handle '129'",
3901 "/operations/129?t=status&output=JSON"))
3904 def test_ophandle_release_after_complete(self):
3905 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3906 followRedirect=True)
3907 d.addCallback(self.wait_for_operation, "130")
3908 d.addCallback(lambda ignored:
3909 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3910 # the release-after-complete=true will cause the handle to be expired
3911 d.addCallback(lambda ignored:
3912 self.shouldHTTPError("ophandle_release_after_complete",
3913 404, "404 Not Found",
3914 "unknown/expired handle '130'",
3916 "/operations/130?t=status&output=JSON"))
3919 def test_uncollected_ophandle_expiration(self):
3920 # uncollected ophandles should expire after 4 days
3921 def _make_uncollected_ophandle(ophandle):
3922 d = self.POST(self.public_url +
3923 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3924 followRedirect=False)
3925 # When we start the operation, the webapi server will want
3926 # to redirect us to the page for the ophandle, so we get
3927 # confirmation that the operation has started. If the
3928 # manifest operation has finished by the time we get there,
3929 # following that redirect (by setting followRedirect=True
3930 # above) has the side effect of collecting the ophandle that
3931 # we've just created, which means that we can't use the
3932 # ophandle to test the uncollected timeout anymore. So,
3933 # instead, catch the 302 here and don't follow it.
3934 d.addBoth(self.should302, "uncollected_ophandle_creation")
3936 # Create an ophandle, don't collect it, then advance the clock by
3937 # 4 days - 1 second and make sure that the ophandle is still there.
3938 d = _make_uncollected_ophandle(131)
3939 d.addCallback(lambda ign:
3940 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3941 d.addCallback(lambda ign:
3942 self.GET("/operations/131?t=status&output=JSON"))
3944 data = simplejson.loads(res)
3945 self.failUnless("finished" in data, res)
3946 d.addCallback(_check1)
3947 # Create an ophandle, don't collect it, then try to collect it
3948 # after 4 days. It should be gone.
3949 d.addCallback(lambda ign:
3950 _make_uncollected_ophandle(132))
3951 d.addCallback(lambda ign:
3952 self.clock.advance(96*60*60))
3953 d.addCallback(lambda ign:
3954 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3955 404, "404 Not Found",
3956 "unknown/expired handle '132'",
3958 "/operations/132?t=status&output=JSON"))
3961 def test_collected_ophandle_expiration(self):
3962 # collected ophandles should expire after 1 day
3963 def _make_collected_ophandle(ophandle):
3964 d = self.POST(self.public_url +
3965 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3966 followRedirect=True)
3967 # By following the initial redirect, we collect the ophandle
3968 # we've just created.
3970 # Create a collected ophandle, then collect it after 23 hours
3971 # and 59 seconds to make sure that it is still there.
3972 d = _make_collected_ophandle(133)
3973 d.addCallback(lambda ign:
3974 self.clock.advance((24*60*60) - 1))
3975 d.addCallback(lambda ign:
3976 self.GET("/operations/133?t=status&output=JSON"))
3978 data = simplejson.loads(res)
3979 self.failUnless("finished" in data, res)
3980 d.addCallback(_check1)
3981 # Create another uncollected ophandle, then try to collect it
3982 # after 24 hours to make sure that it is gone.
3983 d.addCallback(lambda ign:
3984 _make_collected_ophandle(134))
3985 d.addCallback(lambda ign:
3986 self.clock.advance(24*60*60))
3987 d.addCallback(lambda ign:
3988 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3989 404, "404 Not Found",
3990 "unknown/expired handle '134'",
3992 "/operations/134?t=status&output=JSON"))
3995 def test_incident(self):
3996 d = self.POST("/report_incident", details="eek")
3998 self.failIfIn("<html>", res)
3999 self.failUnlessIn("Thank you for your report!", res)
4000 d.addCallback(_done)
4003 def test_static(self):
4004 webdir = os.path.join(self.staticdir, "subdir")
4005 fileutil.make_dirs(webdir)
4006 f = open(os.path.join(webdir, "hello.txt"), "wb")
4010 d = self.GET("/static/subdir/hello.txt")
4012 self.failUnlessReallyEqual(res, "hello")
4013 d.addCallback(_check)
4017 class IntroducerWeb(unittest.TestCase):
4022 d = defer.succeed(None)
4024 d.addCallback(lambda ign: self.node.stopService())
4025 d.addCallback(flushEventualQueue)
4028 def test_welcome(self):
4029 basedir = "web.IntroducerWeb.test_welcome"
4031 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4032 self.node = IntroducerNode(basedir)
4033 self.ws = self.node.getServiceNamed("webish")
4035 d = fireEventually(None)
4036 d.addCallback(lambda ign: self.node.startService())
4037 d.addCallback(lambda ign: self.node.when_tub_ready())
4039 d.addCallback(lambda ign: self.GET("/"))
4041 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4042 self.failUnlessIn(FAVICON_MARKUP, res)
4043 d.addCallback(_check)
4046 def GET(self, urlpath, followRedirect=False, return_response=False,
4048 # if return_response=True, this fires with (data, statuscode,
4049 # respheaders) instead of just data.
4050 assert not isinstance(urlpath, unicode)
4051 url = self.ws.getURL().rstrip('/') + urlpath
4052 factory = HTTPClientGETFactory(url, method="GET",
4053 followRedirect=followRedirect, **kwargs)
4054 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4055 d = factory.deferred
4056 def _got_data(data):
4057 return (data, factory.status, factory.response_headers)
4059 d.addCallback(_got_data)
4060 return factory.deferred
4063 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4064 def test_load_file(self):
4065 # This will raise an exception unless a well-formed XML file is found under that name.
4066 common.getxmlfile('directory.xhtml').load()
4068 def test_parse_replace_arg(self):
4069 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4070 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4071 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4073 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4074 common.parse_replace_arg, "only_fles")
4076 def test_abbreviate_time(self):
4077 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4078 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4079 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4080 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4081 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4082 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4084 def test_compute_rate(self):
4085 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4086 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4087 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4088 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4089 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4090 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4091 self.shouldFail(AssertionError, "test_compute_rate", "",
4092 common.compute_rate, -100, 10)
4093 self.shouldFail(AssertionError, "test_compute_rate", "",
4094 common.compute_rate, 100, -10)
4097 rate = common.compute_rate(10*1000*1000, 1)
4098 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4100 def test_abbreviate_rate(self):
4101 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4102 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4103 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4104 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4106 def test_abbreviate_size(self):
4107 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4108 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4109 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4110 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4111 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4113 def test_plural(self):
4115 return "%d second%s" % (s, status.plural(s))
4116 self.failUnlessReallyEqual(convert(0), "0 seconds")
4117 self.failUnlessReallyEqual(convert(1), "1 second")
4118 self.failUnlessReallyEqual(convert(2), "2 seconds")
4120 return "has share%s: %s" % (status.plural(s), ",".join(s))
4121 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4122 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4123 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4126 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4128 def CHECK(self, ign, which, args, clientnum=0):
4129 fileurl = self.fileurls[which]
4130 url = fileurl + "?" + args
4131 return self.GET(url, method="POST", clientnum=clientnum)
4133 def test_filecheck(self):
4134 self.basedir = "web/Grid/filecheck"
4136 c0 = self.g.clients[0]
4139 d = c0.upload(upload.Data(DATA, convergence=""))
4140 def _stash_uri(ur, which):
4141 self.uris[which] = ur.uri
4142 d.addCallback(_stash_uri, "good")
4143 d.addCallback(lambda ign:
4144 c0.upload(upload.Data(DATA+"1", convergence="")))
4145 d.addCallback(_stash_uri, "sick")
4146 d.addCallback(lambda ign:
4147 c0.upload(upload.Data(DATA+"2", convergence="")))
4148 d.addCallback(_stash_uri, "dead")
4149 def _stash_mutable_uri(n, which):
4150 self.uris[which] = n.get_uri()
4151 assert isinstance(self.uris[which], str)
4152 d.addCallback(lambda ign:
4153 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4154 d.addCallback(_stash_mutable_uri, "corrupt")
4155 d.addCallback(lambda ign:
4156 c0.upload(upload.Data("literal", convergence="")))
4157 d.addCallback(_stash_uri, "small")
4158 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4159 d.addCallback(_stash_mutable_uri, "smalldir")
4161 def _compute_fileurls(ignored):
4163 for which in self.uris:
4164 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4165 d.addCallback(_compute_fileurls)
4167 def _clobber_shares(ignored):
4168 good_shares = self.find_uri_shares(self.uris["good"])
4169 self.failUnlessReallyEqual(len(good_shares), 10)
4170 sick_shares = self.find_uri_shares(self.uris["sick"])
4171 os.unlink(sick_shares[0][2])
4172 dead_shares = self.find_uri_shares(self.uris["dead"])
4173 for i in range(1, 10):
4174 os.unlink(dead_shares[i][2])
4175 c_shares = self.find_uri_shares(self.uris["corrupt"])
4176 cso = CorruptShareOptions()
4177 cso.stdout = StringIO()
4178 cso.parseOptions([c_shares[0][2]])
4180 d.addCallback(_clobber_shares)
4182 d.addCallback(self.CHECK, "good", "t=check")
4183 def _got_html_good(res):
4184 self.failUnlessIn("Healthy", res)
4185 self.failIfIn("Not Healthy", res)
4186 self.failUnlessIn(FAVICON_MARKUP, res)
4187 d.addCallback(_got_html_good)
4188 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4189 def _got_html_good_return_to(res):
4190 self.failUnlessIn("Healthy", res)
4191 self.failIfIn("Not Healthy", res)
4192 self.failUnlessIn('<a href="somewhere">Return to file', res)
4193 d.addCallback(_got_html_good_return_to)
4194 d.addCallback(self.CHECK, "good", "t=check&output=json")
4195 def _got_json_good(res):
4196 r = simplejson.loads(res)
4197 self.failUnlessEqual(r["summary"], "Healthy")
4198 self.failUnless(r["results"]["healthy"])
4199 self.failIf(r["results"]["needs-rebalancing"])
4200 self.failUnless(r["results"]["recoverable"])
4201 d.addCallback(_got_json_good)
4203 d.addCallback(self.CHECK, "small", "t=check")
4204 def _got_html_small(res):
4205 self.failUnlessIn("Literal files are always healthy", res)
4206 self.failIfIn("Not Healthy", res)
4207 d.addCallback(_got_html_small)
4208 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4209 def _got_html_small_return_to(res):
4210 self.failUnlessIn("Literal files are always healthy", res)
4211 self.failIfIn("Not Healthy", res)
4212 self.failUnlessIn('<a href="somewhere">Return to file', res)
4213 d.addCallback(_got_html_small_return_to)
4214 d.addCallback(self.CHECK, "small", "t=check&output=json")
4215 def _got_json_small(res):
4216 r = simplejson.loads(res)
4217 self.failUnlessEqual(r["storage-index"], "")
4218 self.failUnless(r["results"]["healthy"])
4219 d.addCallback(_got_json_small)
4221 d.addCallback(self.CHECK, "smalldir", "t=check")
4222 def _got_html_smalldir(res):
4223 self.failUnlessIn("Literal files are always healthy", res)
4224 self.failIfIn("Not Healthy", res)
4225 d.addCallback(_got_html_smalldir)
4226 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4227 def _got_json_smalldir(res):
4228 r = simplejson.loads(res)
4229 self.failUnlessEqual(r["storage-index"], "")
4230 self.failUnless(r["results"]["healthy"])
4231 d.addCallback(_got_json_smalldir)
4233 d.addCallback(self.CHECK, "sick", "t=check")
4234 def _got_html_sick(res):
4235 self.failUnlessIn("Not Healthy", res)
4236 d.addCallback(_got_html_sick)
4237 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4238 def _got_json_sick(res):
4239 r = simplejson.loads(res)
4240 self.failUnlessEqual(r["summary"],
4241 "Not Healthy: 9 shares (enc 3-of-10)")
4242 self.failIf(r["results"]["healthy"])
4243 self.failIf(r["results"]["needs-rebalancing"])
4244 self.failUnless(r["results"]["recoverable"])
4245 d.addCallback(_got_json_sick)
4247 d.addCallback(self.CHECK, "dead", "t=check")
4248 def _got_html_dead(res):
4249 self.failUnlessIn("Not Healthy", res)
4250 d.addCallback(_got_html_dead)
4251 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4252 def _got_json_dead(res):
4253 r = simplejson.loads(res)
4254 self.failUnlessEqual(r["summary"],
4255 "Not Healthy: 1 shares (enc 3-of-10)")
4256 self.failIf(r["results"]["healthy"])
4257 self.failIf(r["results"]["needs-rebalancing"])
4258 self.failIf(r["results"]["recoverable"])
4259 d.addCallback(_got_json_dead)
4261 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4262 def _got_html_corrupt(res):
4263 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4264 d.addCallback(_got_html_corrupt)
4265 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4266 def _got_json_corrupt(res):
4267 r = simplejson.loads(res)
4268 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4269 self.failIf(r["results"]["healthy"])
4270 self.failUnless(r["results"]["recoverable"])
4271 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4272 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4273 d.addCallback(_got_json_corrupt)
4275 d.addErrback(self.explain_web_error)
4278 def test_repair_html(self):
4279 self.basedir = "web/Grid/repair_html"
4281 c0 = self.g.clients[0]
4284 d = c0.upload(upload.Data(DATA, convergence=""))
4285 def _stash_uri(ur, which):
4286 self.uris[which] = ur.uri
4287 d.addCallback(_stash_uri, "good")
4288 d.addCallback(lambda ign:
4289 c0.upload(upload.Data(DATA+"1", convergence="")))
4290 d.addCallback(_stash_uri, "sick")
4291 d.addCallback(lambda ign:
4292 c0.upload(upload.Data(DATA+"2", convergence="")))
4293 d.addCallback(_stash_uri, "dead")
4294 def _stash_mutable_uri(n, which):
4295 self.uris[which] = n.get_uri()
4296 assert isinstance(self.uris[which], str)
4297 d.addCallback(lambda ign:
4298 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4299 d.addCallback(_stash_mutable_uri, "corrupt")
4301 def _compute_fileurls(ignored):
4303 for which in self.uris:
4304 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4305 d.addCallback(_compute_fileurls)
4307 def _clobber_shares(ignored):
4308 good_shares = self.find_uri_shares(self.uris["good"])
4309 self.failUnlessReallyEqual(len(good_shares), 10)
4310 sick_shares = self.find_uri_shares(self.uris["sick"])
4311 os.unlink(sick_shares[0][2])
4312 dead_shares = self.find_uri_shares(self.uris["dead"])
4313 for i in range(1, 10):
4314 os.unlink(dead_shares[i][2])
4315 c_shares = self.find_uri_shares(self.uris["corrupt"])
4316 cso = CorruptShareOptions()
4317 cso.stdout = StringIO()
4318 cso.parseOptions([c_shares[0][2]])
4320 d.addCallback(_clobber_shares)
4322 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4323 def _got_html_good(res):
4324 self.failUnlessIn("Healthy", res)
4325 self.failIfIn("Not Healthy", res)
4326 self.failUnlessIn("No repair necessary", res)
4327 self.failUnlessIn(FAVICON_MARKUP, res)
4328 d.addCallback(_got_html_good)
4330 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4331 def _got_html_sick(res):
4332 self.failUnlessIn("Healthy : healthy", res)
4333 self.failIfIn("Not Healthy", res)
4334 self.failUnlessIn("Repair successful", res)
4335 d.addCallback(_got_html_sick)
4337 # repair of a dead file will fail, of course, but it isn't yet
4338 # clear how this should be reported. Right now it shows up as
4341 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4342 #def _got_html_dead(res):
4344 # self.failUnlessIn("Healthy : healthy", res)
4345 # self.failIfIn("Not Healthy", res)
4346 # self.failUnlessIn("No repair necessary", res)
4347 #d.addCallback(_got_html_dead)
4349 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4350 def _got_html_corrupt(res):
4351 self.failUnlessIn("Healthy : Healthy", res)
4352 self.failIfIn("Not Healthy", res)
4353 self.failUnlessIn("Repair successful", res)
4354 d.addCallback(_got_html_corrupt)
4356 d.addErrback(self.explain_web_error)
4359 def test_repair_json(self):
4360 self.basedir = "web/Grid/repair_json"
4362 c0 = self.g.clients[0]
4365 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4366 def _stash_uri(ur, which):
4367 self.uris[which] = ur.uri
4368 d.addCallback(_stash_uri, "sick")
4370 def _compute_fileurls(ignored):
4372 for which in self.uris:
4373 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4374 d.addCallback(_compute_fileurls)
4376 def _clobber_shares(ignored):
4377 sick_shares = self.find_uri_shares(self.uris["sick"])
4378 os.unlink(sick_shares[0][2])
4379 d.addCallback(_clobber_shares)
4381 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4382 def _got_json_sick(res):
4383 r = simplejson.loads(res)
4384 self.failUnlessReallyEqual(r["repair-attempted"], True)
4385 self.failUnlessReallyEqual(r["repair-successful"], True)
4386 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4387 "Not Healthy: 9 shares (enc 3-of-10)")
4388 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4389 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4390 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4391 d.addCallback(_got_json_sick)
4393 d.addErrback(self.explain_web_error)
4396 def test_unknown(self, immutable=False):
4397 self.basedir = "web/Grid/unknown"
4399 self.basedir = "web/Grid/unknown-immutable"
4402 c0 = self.g.clients[0]
4406 # the future cap format may contain slashes, which must be tolerated
4407 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4411 name = u"future-imm"
4412 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4413 d = c0.create_immutable_dirnode({name: (future_node, {})})
4416 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4417 d = c0.create_dirnode()
4419 def _stash_root_and_create_file(n):
4421 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4422 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4424 return self.rootnode.set_node(name, future_node)
4425 d.addCallback(_stash_root_and_create_file)
4427 # make sure directory listing tolerates unknown nodes
4428 d.addCallback(lambda ign: self.GET(self.rooturl))
4429 def _check_directory_html(res, expected_type_suffix):
4430 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4431 '<td>%s</td>' % (expected_type_suffix, str(name)),
4433 self.failUnless(re.search(pattern, res), res)
4434 # find the More Info link for name, should be relative
4435 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4436 info_url = mo.group(1)
4437 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4439 d.addCallback(_check_directory_html, "-IMM")
4441 d.addCallback(_check_directory_html, "")
4443 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4444 def _check_directory_json(res, expect_rw_uri):
4445 data = simplejson.loads(res)
4446 self.failUnlessEqual(data[0], "dirnode")
4447 f = data[1]["children"][name]
4448 self.failUnlessEqual(f[0], "unknown")
4450 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4452 self.failIfIn("rw_uri", f[1])
4454 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4456 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4457 self.failUnlessIn("metadata", f[1])
4458 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4460 def _check_info(res, expect_rw_uri, expect_ro_uri):
4461 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4463 self.failUnlessIn(unknown_rwcap, res)
4466 self.failUnlessIn(unknown_immcap, res)
4468 self.failUnlessIn(unknown_rocap, res)
4470 self.failIfIn(unknown_rocap, res)
4471 self.failIfIn("Raw data as", res)
4472 self.failIfIn("Directory writecap", res)
4473 self.failIfIn("Checker Operations", res)
4474 self.failIfIn("Mutable File Operations", res)
4475 self.failIfIn("Directory Operations", res)
4477 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4478 # why they fail. Possibly related to ticket #922.
4480 d.addCallback(lambda ign: self.GET(expected_info_url))
4481 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4482 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4483 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4485 def _check_json(res, expect_rw_uri):
4486 data = simplejson.loads(res)
4487 self.failUnlessEqual(data[0], "unknown")
4489 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4491 self.failIfIn("rw_uri", data[1])
4494 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4495 self.failUnlessReallyEqual(data[1]["mutable"], False)
4497 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4498 self.failUnlessReallyEqual(data[1]["mutable"], True)
4500 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4501 self.failIfIn("mutable", data[1])
4503 # TODO: check metadata contents
4504 self.failUnlessIn("metadata", data[1])
4506 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4507 d.addCallback(_check_json, expect_rw_uri=not immutable)
4509 # and make sure that a read-only version of the directory can be
4510 # rendered too. This version will not have unknown_rwcap, whether
4511 # or not future_node was immutable.
4512 d.addCallback(lambda ign: self.GET(self.rourl))
4514 d.addCallback(_check_directory_html, "-IMM")
4516 d.addCallback(_check_directory_html, "-RO")
4518 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4519 d.addCallback(_check_directory_json, expect_rw_uri=False)
4521 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4522 d.addCallback(_check_json, expect_rw_uri=False)
4524 # TODO: check that getting t=info from the Info link in the ro directory
4525 # works, and does not include the writecap URI.
4528 def test_immutable_unknown(self):
4529 return self.test_unknown(immutable=True)
4531 def test_mutant_dirnodes_are_omitted(self):
4532 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4535 c = self.g.clients[0]
4540 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4541 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4542 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4544 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4545 # test the dirnode and web layers separately.
4547 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4548 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4549 # When the directory is read, the mutants should be silently disposed of, leaving
4550 # their lonely sibling.
4551 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4552 # because immutable directories don't have a writecap and therefore that field
4553 # isn't (and can't be) decrypted.
4554 # TODO: The field still exists in the netstring. Technically we should check what
4555 # happens if something is put there (_unpack_contents should raise ValueError),
4556 # but that can wait.
4558 lonely_child = nm.create_from_cap(lonely_uri)
4559 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4560 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4562 def _by_hook_or_by_crook():
4564 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4565 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4567 mutant_write_in_ro_child.get_write_uri = lambda: None
4568 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4570 kids = {u"lonely": (lonely_child, {}),
4571 u"ro": (mutant_ro_child, {}),
4572 u"write-in-ro": (mutant_write_in_ro_child, {}),
4574 d = c.create_immutable_dirnode(kids)
4577 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4578 self.failIf(dn.is_mutable())
4579 self.failUnless(dn.is_readonly())
4580 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4581 self.failIf(hasattr(dn._node, 'get_writekey'))
4583 self.failUnlessIn("RO-IMM", rep)
4585 self.failUnlessIn("CHK", cap.to_string())
4588 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4589 return download_to_data(dn._node)
4590 d.addCallback(_created)
4592 def _check_data(data):
4593 # Decode the netstring representation of the directory to check that all children
4594 # are present. This is a bit of an abstraction violation, but there's not really
4595 # any other way to do it given that the real DirectoryNode._unpack_contents would
4596 # strip the mutant children out (which is what we're trying to test, later).
4599 while position < len(data):
4600 entries, position = split_netstring(data, 1, position)
4602 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4603 name = name_utf8.decode("utf-8")
4604 self.failUnlessEqual(rwcapdata, "")
4605 self.failUnlessIn(name, kids)
4606 (expected_child, ign) = kids[name]
4607 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4610 self.failUnlessReallyEqual(numkids, 3)
4611 return self.rootnode.list()
4612 d.addCallback(_check_data)
4614 # Now when we use the real directory listing code, the mutants should be absent.
4615 def _check_kids(children):
4616 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4617 lonely_node, lonely_metadata = children[u"lonely"]
4619 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4620 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4621 d.addCallback(_check_kids)
4623 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4624 d.addCallback(lambda n: n.list())
4625 d.addCallback(_check_kids) # again with dirnode recreated from cap
4627 # Make sure the lonely child can be listed in HTML...
4628 d.addCallback(lambda ign: self.GET(self.rooturl))
4629 def _check_html(res):
4630 self.failIfIn("URI:SSK", res)
4631 get_lonely = "".join([r'<td>FILE</td>',
4633 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4635 r'\s+<td align="right">%d</td>' % len("one"),
4637 self.failUnless(re.search(get_lonely, res), res)
4639 # find the More Info link for name, should be relative
4640 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4641 info_url = mo.group(1)
4642 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4643 d.addCallback(_check_html)
4646 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4647 def _check_json(res):
4648 data = simplejson.loads(res)
4649 self.failUnlessEqual(data[0], "dirnode")
4650 listed_children = data[1]["children"]
4651 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4652 ll_type, ll_data = listed_children[u"lonely"]
4653 self.failUnlessEqual(ll_type, "filenode")
4654 self.failIfIn("rw_uri", ll_data)
4655 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4656 d.addCallback(_check_json)
4659 def test_deep_check(self):
4660 self.basedir = "web/Grid/deep_check"
4662 c0 = self.g.clients[0]
4666 d = c0.create_dirnode()
4667 def _stash_root_and_create_file(n):
4669 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4670 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4671 d.addCallback(_stash_root_and_create_file)
4672 def _stash_uri(fn, which):
4673 self.uris[which] = fn.get_uri()
4675 d.addCallback(_stash_uri, "good")
4676 d.addCallback(lambda ign:
4677 self.rootnode.add_file(u"small",
4678 upload.Data("literal",
4680 d.addCallback(_stash_uri, "small")
4681 d.addCallback(lambda ign:
4682 self.rootnode.add_file(u"sick",
4683 upload.Data(DATA+"1",
4685 d.addCallback(_stash_uri, "sick")
4687 # this tests that deep-check and stream-manifest will ignore
4688 # UnknownNode instances. Hopefully this will also cover deep-stats.
4689 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4690 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4692 def _clobber_shares(ignored):
4693 self.delete_shares_numbered(self.uris["sick"], [0,1])
4694 d.addCallback(_clobber_shares)
4702 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4705 units = [simplejson.loads(line)
4706 for line in res.splitlines()
4709 print "response is:", res
4710 print "undecodeable line was '%s'" % line
4712 self.failUnlessReallyEqual(len(units), 5+1)
4713 # should be parent-first
4715 self.failUnlessEqual(u0["path"], [])
4716 self.failUnlessEqual(u0["type"], "directory")
4717 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4718 u0cr = u0["check-results"]
4719 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4721 ugood = [u for u in units
4722 if u["type"] == "file" and u["path"] == [u"good"]][0]
4723 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4724 ugoodcr = ugood["check-results"]
4725 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4728 self.failUnlessEqual(stats["type"], "stats")
4730 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4731 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4732 self.failUnlessReallyEqual(s["count-directories"], 1)
4733 self.failUnlessReallyEqual(s["count-unknown"], 1)
4734 d.addCallback(_done)
4736 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4737 def _check_manifest(res):
4738 self.failUnless(res.endswith("\n"))
4739 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4740 self.failUnlessReallyEqual(len(units), 5+1)
4741 self.failUnlessEqual(units[-1]["type"], "stats")
4743 self.failUnlessEqual(first["path"], [])
4744 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4745 self.failUnlessEqual(first["type"], "directory")
4746 stats = units[-1]["stats"]
4747 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4748 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4749 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4750 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4751 self.failUnlessReallyEqual(stats["count-unknown"], 1)
4752 d.addCallback(_check_manifest)
4754 # now add root/subdir and root/subdir/grandchild, then make subdir
4755 # unrecoverable, then see what happens
4757 d.addCallback(lambda ign:
4758 self.rootnode.create_subdirectory(u"subdir"))
4759 d.addCallback(_stash_uri, "subdir")
4760 d.addCallback(lambda subdir_node:
4761 subdir_node.add_file(u"grandchild",
4762 upload.Data(DATA+"2",
4764 d.addCallback(_stash_uri, "grandchild")
4766 d.addCallback(lambda ign:
4767 self.delete_shares_numbered(self.uris["subdir"],
4775 # root/subdir [unrecoverable]
4776 # root/subdir/grandchild
4778 # how should a streaming-JSON API indicate fatal error?
4779 # answer: emit ERROR: instead of a JSON string
4781 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4782 def _check_broken_manifest(res):
4783 lines = res.splitlines()
4785 for (i,line) in enumerate(lines)
4786 if line.startswith("ERROR:")]
4788 self.fail("no ERROR: in output: %s" % (res,))
4789 first_error = error_lines[0]
4790 error_line = lines[first_error]
4791 error_msg = lines[first_error+1:]
4792 error_msg_s = "\n".join(error_msg) + "\n"
4793 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4795 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4796 units = [simplejson.loads(line) for line in lines[:first_error]]
4797 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4798 last_unit = units[-1]
4799 self.failUnlessEqual(last_unit["path"], ["subdir"])
4800 d.addCallback(_check_broken_manifest)
4802 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4803 def _check_broken_deepcheck(res):
4804 lines = res.splitlines()
4806 for (i,line) in enumerate(lines)
4807 if line.startswith("ERROR:")]
4809 self.fail("no ERROR: in output: %s" % (res,))
4810 first_error = error_lines[0]
4811 error_line = lines[first_error]
4812 error_msg = lines[first_error+1:]
4813 error_msg_s = "\n".join(error_msg) + "\n"
4814 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4816 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4817 units = [simplejson.loads(line) for line in lines[:first_error]]
4818 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4819 last_unit = units[-1]
4820 self.failUnlessEqual(last_unit["path"], ["subdir"])
4821 r = last_unit["check-results"]["results"]
4822 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4823 self.failUnlessReallyEqual(r["count-shares-good"], 1)
4824 self.failUnlessReallyEqual(r["recoverable"], False)
4825 d.addCallback(_check_broken_deepcheck)
4827 d.addErrback(self.explain_web_error)
4830 def test_deep_check_and_repair(self):
4831 self.basedir = "web/Grid/deep_check_and_repair"
4833 c0 = self.g.clients[0]
4837 d = c0.create_dirnode()
4838 def _stash_root_and_create_file(n):
4840 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4841 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4842 d.addCallback(_stash_root_and_create_file)
4843 def _stash_uri(fn, which):
4844 self.uris[which] = fn.get_uri()
4845 d.addCallback(_stash_uri, "good")
4846 d.addCallback(lambda ign:
4847 self.rootnode.add_file(u"small",
4848 upload.Data("literal",
4850 d.addCallback(_stash_uri, "small")
4851 d.addCallback(lambda ign:
4852 self.rootnode.add_file(u"sick",
4853 upload.Data(DATA+"1",
4855 d.addCallback(_stash_uri, "sick")
4856 #d.addCallback(lambda ign:
4857 # self.rootnode.add_file(u"dead",
4858 # upload.Data(DATA+"2",
4860 #d.addCallback(_stash_uri, "dead")
4862 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4863 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4864 #d.addCallback(_stash_uri, "corrupt")
4866 def _clobber_shares(ignored):
4867 good_shares = self.find_uri_shares(self.uris["good"])
4868 self.failUnlessReallyEqual(len(good_shares), 10)
4869 sick_shares = self.find_uri_shares(self.uris["sick"])
4870 os.unlink(sick_shares[0][2])
4871 #dead_shares = self.find_uri_shares(self.uris["dead"])
4872 #for i in range(1, 10):
4873 # os.unlink(dead_shares[i][2])
4875 #c_shares = self.find_uri_shares(self.uris["corrupt"])
4876 #cso = CorruptShareOptions()
4877 #cso.stdout = StringIO()
4878 #cso.parseOptions([c_shares[0][2]])
4880 d.addCallback(_clobber_shares)
4883 # root/good CHK, 10 shares
4885 # root/sick CHK, 9 shares
4887 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4889 units = [simplejson.loads(line)
4890 for line in res.splitlines()
4892 self.failUnlessReallyEqual(len(units), 4+1)
4893 # should be parent-first
4895 self.failUnlessEqual(u0["path"], [])
4896 self.failUnlessEqual(u0["type"], "directory")
4897 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4898 u0crr = u0["check-and-repair-results"]
4899 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4900 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4902 ugood = [u for u in units
4903 if u["type"] == "file" and u["path"] == [u"good"]][0]
4904 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4905 ugoodcrr = ugood["check-and-repair-results"]
4906 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4907 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4909 usick = [u for u in units
4910 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4911 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4912 usickcrr = usick["check-and-repair-results"]
4913 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4914 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4915 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4916 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4919 self.failUnlessEqual(stats["type"], "stats")
4921 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4922 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4923 self.failUnlessReallyEqual(s["count-directories"], 1)
4924 d.addCallback(_done)
4926 d.addErrback(self.explain_web_error)
4929 def _count_leases(self, ignored, which):
4930 u = self.uris[which]
4931 shares = self.find_uri_shares(u)
4933 for shnum, serverid, fn in shares:
4934 sf = get_share_file(fn)
4935 num_leases = len(list(sf.get_leases()))
4936 lease_counts.append( (fn, num_leases) )
4939 def _assert_leasecount(self, lease_counts, expected):
4940 for (fn, num_leases) in lease_counts:
4941 if num_leases != expected:
4942 self.fail("expected %d leases, have %d, on %s" %
4943 (expected, num_leases, fn))
4945 def test_add_lease(self):
4946 self.basedir = "web/Grid/add_lease"
4947 self.set_up_grid(num_clients=2)
4948 c0 = self.g.clients[0]
4951 d = c0.upload(upload.Data(DATA, convergence=""))
4952 def _stash_uri(ur, which):
4953 self.uris[which] = ur.uri
4954 d.addCallback(_stash_uri, "one")
4955 d.addCallback(lambda ign:
4956 c0.upload(upload.Data(DATA+"1", convergence="")))
4957 d.addCallback(_stash_uri, "two")
4958 def _stash_mutable_uri(n, which):
4959 self.uris[which] = n.get_uri()
4960 assert isinstance(self.uris[which], str)
4961 d.addCallback(lambda ign:
4962 c0.create_mutable_file(publish.MutableData(DATA+"2")))
4963 d.addCallback(_stash_mutable_uri, "mutable")
4965 def _compute_fileurls(ignored):
4967 for which in self.uris:
4968 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4969 d.addCallback(_compute_fileurls)
4971 d.addCallback(self._count_leases, "one")
4972 d.addCallback(self._assert_leasecount, 1)
4973 d.addCallback(self._count_leases, "two")
4974 d.addCallback(self._assert_leasecount, 1)
4975 d.addCallback(self._count_leases, "mutable")
4976 d.addCallback(self._assert_leasecount, 1)
4978 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4979 def _got_html_good(res):
4980 self.failUnlessIn("Healthy", res)
4981 self.failIfIn("Not Healthy", res)
4982 d.addCallback(_got_html_good)
4984 d.addCallback(self._count_leases, "one")
4985 d.addCallback(self._assert_leasecount, 1)
4986 d.addCallback(self._count_leases, "two")
4987 d.addCallback(self._assert_leasecount, 1)
4988 d.addCallback(self._count_leases, "mutable")
4989 d.addCallback(self._assert_leasecount, 1)
4991 # this CHECK uses the original client, which uses the same
4992 # lease-secrets, so it will just renew the original lease
4993 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4994 d.addCallback(_got_html_good)
4996 d.addCallback(self._count_leases, "one")
4997 d.addCallback(self._assert_leasecount, 1)
4998 d.addCallback(self._count_leases, "two")
4999 d.addCallback(self._assert_leasecount, 1)
5000 d.addCallback(self._count_leases, "mutable")
5001 d.addCallback(self._assert_leasecount, 1)
5003 # this CHECK uses an alternate client, which adds a second lease
5004 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5005 d.addCallback(_got_html_good)
5007 d.addCallback(self._count_leases, "one")
5008 d.addCallback(self._assert_leasecount, 2)
5009 d.addCallback(self._count_leases, "two")
5010 d.addCallback(self._assert_leasecount, 1)
5011 d.addCallback(self._count_leases, "mutable")
5012 d.addCallback(self._assert_leasecount, 1)
5014 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5015 d.addCallback(_got_html_good)
5017 d.addCallback(self._count_leases, "one")
5018 d.addCallback(self._assert_leasecount, 2)
5019 d.addCallback(self._count_leases, "two")
5020 d.addCallback(self._assert_leasecount, 1)
5021 d.addCallback(self._count_leases, "mutable")
5022 d.addCallback(self._assert_leasecount, 1)
5024 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5026 d.addCallback(_got_html_good)
5028 d.addCallback(self._count_leases, "one")
5029 d.addCallback(self._assert_leasecount, 2)
5030 d.addCallback(self._count_leases, "two")
5031 d.addCallback(self._assert_leasecount, 1)
5032 d.addCallback(self._count_leases, "mutable")
5033 d.addCallback(self._assert_leasecount, 2)
5035 d.addErrback(self.explain_web_error)
5038 def test_deep_add_lease(self):
5039 self.basedir = "web/Grid/deep_add_lease"
5040 self.set_up_grid(num_clients=2)
5041 c0 = self.g.clients[0]
5045 d = c0.create_dirnode()
5046 def _stash_root_and_create_file(n):
5048 self.uris["root"] = n.get_uri()
5049 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5050 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5051 d.addCallback(_stash_root_and_create_file)
5052 def _stash_uri(fn, which):
5053 self.uris[which] = fn.get_uri()
5054 d.addCallback(_stash_uri, "one")
5055 d.addCallback(lambda ign:
5056 self.rootnode.add_file(u"small",
5057 upload.Data("literal",
5059 d.addCallback(_stash_uri, "small")
5061 d.addCallback(lambda ign:
5062 c0.create_mutable_file(publish.MutableData("mutable")))
5063 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5064 d.addCallback(_stash_uri, "mutable")
5066 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5068 units = [simplejson.loads(line)
5069 for line in res.splitlines()
5071 # root, one, small, mutable, stats
5072 self.failUnlessReallyEqual(len(units), 4+1)
5073 d.addCallback(_done)
5075 d.addCallback(self._count_leases, "root")
5076 d.addCallback(self._assert_leasecount, 1)
5077 d.addCallback(self._count_leases, "one")
5078 d.addCallback(self._assert_leasecount, 1)
5079 d.addCallback(self._count_leases, "mutable")
5080 d.addCallback(self._assert_leasecount, 1)
5082 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5083 d.addCallback(_done)
5085 d.addCallback(self._count_leases, "root")
5086 d.addCallback(self._assert_leasecount, 1)
5087 d.addCallback(self._count_leases, "one")
5088 d.addCallback(self._assert_leasecount, 1)
5089 d.addCallback(self._count_leases, "mutable")
5090 d.addCallback(self._assert_leasecount, 1)
5092 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5094 d.addCallback(_done)
5096 d.addCallback(self._count_leases, "root")
5097 d.addCallback(self._assert_leasecount, 2)
5098 d.addCallback(self._count_leases, "one")
5099 d.addCallback(self._assert_leasecount, 2)
5100 d.addCallback(self._count_leases, "mutable")
5101 d.addCallback(self._assert_leasecount, 2)
5103 d.addErrback(self.explain_web_error)
5107 def test_exceptions(self):
5108 self.basedir = "web/Grid/exceptions"
5109 self.set_up_grid(num_clients=1, num_servers=2)
5110 c0 = self.g.clients[0]
5111 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5114 d = c0.create_dirnode()
5116 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5117 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5119 d.addCallback(_stash_root)
5120 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5122 self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5123 self.delete_shares_numbered(ur.uri, range(1,10))
5125 u = uri.from_string(ur.uri)
5126 u.key = testutil.flip_bit(u.key, 0)
5127 baduri = u.to_string()
5128 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5129 d.addCallback(_stash_bad)
5130 d.addCallback(lambda ign: c0.create_dirnode())
5131 def _mangle_dirnode_1share(n):
5133 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5134 self.fileurls["dir-1share-json"] = url + "?t=json"
5135 self.delete_shares_numbered(u, range(1,10))
5136 d.addCallback(_mangle_dirnode_1share)
5137 d.addCallback(lambda ign: c0.create_dirnode())
5138 def _mangle_dirnode_0share(n):
5140 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5141 self.fileurls["dir-0share-json"] = url + "?t=json"
5142 self.delete_shares_numbered(u, range(0,10))
5143 d.addCallback(_mangle_dirnode_0share)
5145 # NotEnoughSharesError should be reported sensibly, with a
5146 # text/plain explanation of the problem, and perhaps some
5147 # information on which shares *could* be found.
5149 d.addCallback(lambda ignored:
5150 self.shouldHTTPError("GET unrecoverable",
5151 410, "Gone", "NoSharesError",
5152 self.GET, self.fileurls["0shares"]))
5153 def _check_zero_shares(body):
5154 self.failIfIn("<html>", body)
5155 body = " ".join(body.strip().split())
5156 exp = ("NoSharesError: no shares could be found. "
5157 "Zero shares usually indicates a corrupt URI, or that "
5158 "no servers were connected, but it might also indicate "
5159 "severe corruption. You should perform a filecheck on "
5160 "this object to learn more. The full error message is: "
5161 "no shares (need 3). Last failure: None")
5162 self.failUnlessReallyEqual(exp, body)
5163 d.addCallback(_check_zero_shares)
5166 d.addCallback(lambda ignored:
5167 self.shouldHTTPError("GET 1share",
5168 410, "Gone", "NotEnoughSharesError",
5169 self.GET, self.fileurls["1share"]))
5170 def _check_one_share(body):
5171 self.failIfIn("<html>", body)
5172 body = " ".join(body.strip().split())
5173 msgbase = ("NotEnoughSharesError: This indicates that some "
5174 "servers were unavailable, or that shares have been "
5175 "lost to server departure, hard drive failure, or disk "
5176 "corruption. You should perform a filecheck on "
5177 "this object to learn more. The full error message is:"
5179 msg1 = msgbase + (" ran out of shares:"
5182 " overdue= unused= need 3. Last failure: None")
5183 msg2 = msgbase + (" ran out of shares:"
5185 " pending=Share(sh0-on-xgru5)"
5186 " overdue= unused= need 3. Last failure: None")
5187 self.failUnless(body == msg1 or body == msg2, body)
5188 d.addCallback(_check_one_share)
5190 d.addCallback(lambda ignored:
5191 self.shouldHTTPError("GET imaginary",
5192 404, "Not Found", None,
5193 self.GET, self.fileurls["imaginary"]))
5194 def _missing_child(body):
5195 self.failUnlessIn("No such child: imaginary", body)
5196 d.addCallback(_missing_child)
5198 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5199 def _check_0shares_dir_html(body):
5200 self.failUnlessIn("<html>", body)
5201 # we should see the regular page, but without the child table or
5203 body = " ".join(body.strip().split())
5204 self.failUnlessIn('href="?t=info">More info on this directory',
5206 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5207 "could not be retrieved, because there were insufficient "
5208 "good shares. This might indicate that no servers were "
5209 "connected, insufficient servers were connected, the URI "
5210 "was corrupt, or that shares have been lost due to server "
5211 "departure, hard drive failure, or disk corruption. You "
5212 "should perform a filecheck on this object to learn more.")
5213 self.failUnlessIn(exp, body)
5214 self.failUnlessIn("No upload forms: directory is unreadable", body)
5215 d.addCallback(_check_0shares_dir_html)
5217 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5218 def _check_1shares_dir_html(body):
5219 # at some point, we'll split UnrecoverableFileError into 0-shares
5220 # and some-shares like we did for immutable files (since there
5221 # are different sorts of advice to offer in each case). For now,
5222 # they present the same way.
5223 self.failUnlessIn("<html>", body)
5224 body = " ".join(body.strip().split())
5225 self.failUnlessIn('href="?t=info">More info on this directory',
5227 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5228 "could not be retrieved, because there were insufficient "
5229 "good shares. This might indicate that no servers were "
5230 "connected, insufficient servers were connected, the URI "
5231 "was corrupt, or that shares have been lost due to server "
5232 "departure, hard drive failure, or disk corruption. You "
5233 "should perform a filecheck on this object to learn more.")
5234 self.failUnlessIn(exp, body)
5235 self.failUnlessIn("No upload forms: directory is unreadable", body)
5236 d.addCallback(_check_1shares_dir_html)
5238 d.addCallback(lambda ignored:
5239 self.shouldHTTPError("GET dir-0share-json",
5240 410, "Gone", "UnrecoverableFileError",
5242 self.fileurls["dir-0share-json"]))
5243 def _check_unrecoverable_file(body):
5244 self.failIfIn("<html>", body)
5245 body = " ".join(body.strip().split())
5246 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5247 "could not be retrieved, because there were insufficient "
5248 "good shares. This might indicate that no servers were "
5249 "connected, insufficient servers were connected, the URI "
5250 "was corrupt, or that shares have been lost due to server "
5251 "departure, hard drive failure, or disk corruption. You "
5252 "should perform a filecheck on this object to learn more.")
5253 self.failUnlessReallyEqual(exp, body)
5254 d.addCallback(_check_unrecoverable_file)
5256 d.addCallback(lambda ignored:
5257 self.shouldHTTPError("GET dir-1share-json",
5258 410, "Gone", "UnrecoverableFileError",
5260 self.fileurls["dir-1share-json"]))
5261 d.addCallback(_check_unrecoverable_file)
5263 d.addCallback(lambda ignored:
5264 self.shouldHTTPError("GET imaginary",
5265 404, "Not Found", None,
5266 self.GET, self.fileurls["imaginary"]))
5268 # attach a webapi child that throws a random error, to test how it
5270 w = c0.getServiceNamed("webish")
5271 w.root.putChild("ERRORBOOM", ErrorBoom())
5273 # "Accept: */*" : should get a text/html stack trace
5274 # "Accept: text/plain" : should get a text/plain stack trace
5275 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5276 # no Accept header: should get a text/html stack trace
5278 d.addCallback(lambda ignored:
5279 self.shouldHTTPError("GET errorboom_html",
5280 500, "Internal Server Error", None,
5281 self.GET, "ERRORBOOM",
5282 headers={"accept": ["*/*"]}))
5283 def _internal_error_html1(body):
5284 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5285 d.addCallback(_internal_error_html1)
5287 d.addCallback(lambda ignored:
5288 self.shouldHTTPError("GET errorboom_text",
5289 500, "Internal Server Error", None,
5290 self.GET, "ERRORBOOM",
5291 headers={"accept": ["text/plain"]}))
5292 def _internal_error_text2(body):
5293 self.failIfIn("<html>", body)
5294 self.failUnless(body.startswith("Traceback "), body)
5295 d.addCallback(_internal_error_text2)
5297 CLI_accepts = "text/plain, application/octet-stream"
5298 d.addCallback(lambda ignored:
5299 self.shouldHTTPError("GET errorboom_text",
5300 500, "Internal Server Error", None,
5301 self.GET, "ERRORBOOM",
5302 headers={"accept": [CLI_accepts]}))
5303 def _internal_error_text3(body):
5304 self.failIfIn("<html>", body)
5305 self.failUnless(body.startswith("Traceback "), body)
5306 d.addCallback(_internal_error_text3)
5308 d.addCallback(lambda ignored:
5309 self.shouldHTTPError("GET errorboom_text",
5310 500, "Internal Server Error", None,
5311 self.GET, "ERRORBOOM"))
5312 def _internal_error_html4(body):
5313 self.failUnlessIn("<html>", body)
5314 d.addCallback(_internal_error_html4)
5316 def _flush_errors(res):
5317 # Trial: please ignore the CompletelyUnhandledError in the logs
5318 self.flushLoggedErrors(CompletelyUnhandledError)
5320 d.addBoth(_flush_errors)
5324 def test_blacklist(self):
5325 # download from a blacklisted URI, get an error
5326 self.basedir = "web/Grid/blacklist"
5328 c0 = self.g.clients[0]
5329 c0_basedir = c0.basedir
5330 fn = os.path.join(c0_basedir, "access.blacklist")
5332 DATA = "off-limits " * 50
5334 d = c0.upload(upload.Data(DATA, convergence=""))
5335 def _stash_uri_and_create_dir(ur):
5337 self.url = "uri/"+self.uri
5338 u = uri.from_string_filenode(self.uri)
5339 self.si = u.get_storage_index()
5340 childnode = c0.create_node_from_uri(self.uri, None)
5341 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5342 d.addCallback(_stash_uri_and_create_dir)
5343 def _stash_dir(node):
5344 self.dir_node = node
5345 self.dir_uri = node.get_uri()
5346 self.dir_url = "uri/"+self.dir_uri
5347 d.addCallback(_stash_dir)
5348 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5349 def _check_dir_html(body):
5350 self.failUnlessIn("<html>", body)
5351 self.failUnlessIn("blacklisted.txt</a>", body)
5352 d.addCallback(_check_dir_html)
5353 d.addCallback(lambda ign: self.GET(self.url))
5354 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5356 def _blacklist(ign):
5358 f.write(" # this is a comment\n")
5360 f.write("\n") # also exercise blank lines
5361 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5363 # clients should be checking the blacklist each time, so we don't
5364 # need to restart the client
5365 d.addCallback(_blacklist)
5366 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5368 "Access Prohibited: off-limits",
5369 self.GET, self.url))
5371 # We should still be able to list the parent directory, in HTML...
5372 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5373 def _check_dir_html2(body):
5374 self.failUnlessIn("<html>", body)
5375 self.failUnlessIn("blacklisted.txt</strike>", body)
5376 d.addCallback(_check_dir_html2)
5378 # ... and in JSON (used by CLI).
5379 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5380 def _check_dir_json(res):
5381 data = simplejson.loads(res)
5382 self.failUnless(isinstance(data, list), data)
5383 self.failUnlessEqual(data[0], "dirnode")
5384 self.failUnless(isinstance(data[1], dict), data)
5385 self.failUnlessIn("children", data[1])
5386 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5387 childdata = data[1]["children"]["blacklisted.txt"]
5388 self.failUnless(isinstance(childdata, list), data)
5389 self.failUnlessEqual(childdata[0], "filenode")
5390 self.failUnless(isinstance(childdata[1], dict), data)
5391 d.addCallback(_check_dir_json)
5393 def _unblacklist(ign):
5394 open(fn, "w").close()
5395 # the Blacklist object watches mtime to tell when the file has
5396 # changed, but on windows this test will run faster than the
5397 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5398 # to force a reload.
5399 self.g.clients[0].blacklist.last_mtime -= 2.0
5400 d.addCallback(_unblacklist)
5402 # now a read should work
5403 d.addCallback(lambda ign: self.GET(self.url))
5404 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5406 # read again to exercise the blacklist-is-unchanged logic
5407 d.addCallback(lambda ign: self.GET(self.url))
5408 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5410 # now add a blacklisted directory, and make sure files under it are
5413 childnode = c0.create_node_from_uri(self.uri, None)
5414 return c0.create_dirnode({u"child": (childnode,{}) })
5415 d.addCallback(_add_dir)
5416 def _get_dircap(dn):
5417 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5418 self.dir_url_base = "uri/"+dn.get_write_uri()
5419 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5420 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5421 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5422 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5423 d.addCallback(_get_dircap)
5424 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5425 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5426 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5427 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5428 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5429 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5430 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5431 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5432 d.addCallback(lambda ign: self.GET(self.child_url))
5433 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5435 def _block_dir(ign):
5437 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5439 self.g.clients[0].blacklist.last_mtime -= 2.0
5440 d.addCallback(_block_dir)
5441 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5443 "Access Prohibited: dir-off-limits",
5444 self.GET, self.dir_url_base))
5445 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5447 "Access Prohibited: dir-off-limits",
5448 self.GET, self.dir_url_json1))
5449 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5451 "Access Prohibited: dir-off-limits",
5452 self.GET, self.dir_url_json2))
5453 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5455 "Access Prohibited: dir-off-limits",
5456 self.GET, self.dir_url_json_ro))
5457 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5459 "Access Prohibited: dir-off-limits",
5460 self.GET, self.child_url))
5464 class CompletelyUnhandledError(Exception):
5466 class ErrorBoom(rend.Page):
5467 def beforeRender(self, ctx):
5468 raise CompletelyUnhandledError("whoops")