1 import os.path, re, urllib, time
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow import rend
16 from allmydata import interfaces, uri, webish, dirnode
17 from allmydata.storage.shares import get_share_file
18 from allmydata.storage_client import StorageFarmBroker
19 from allmydata.immutable import upload
20 from allmydata.immutable.downloader.status import DownloadStatus
21 from allmydata.dirnode import DirectoryNode
22 from allmydata.nodemaker import NodeMaker
23 from allmydata.unknown import UnknownNode
24 from allmydata.web import status, common
25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
26 from allmydata.util import fileutil, base32, hashutil
27 from allmydata.util.consumer import download_to_data
28 from allmydata.util.netstring import split_netstring
29 from allmydata.util.encodingutil import to_str
30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
31 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
32 make_mutable_file_uri, create_mutable_filenode
33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
34 from allmydata.mutable import servermap, publish, retrieve
35 import allmydata.test.common_util as testutil
36 from allmydata.test.no_network import GridTestMixin
37 from allmydata.test.common_web import HTTPClientGETFactory, \
39 from allmydata.client import Client, SecretHolder
40 from allmydata.introducer import IntroducerNode
42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
43 # create a webserver that works against them
45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
54 class FakeStatsProvider:
56 stats = {'stats': {}, 'counters': {}}
59 class FakeNodeMaker(NodeMaker):
64 'max_segment_size':128*1024 # 1024=KiB
66 def _create_lit(self, cap):
67 return FakeCHKFileNode(cap)
68 def _create_immutable(self, cap):
69 return FakeCHKFileNode(cap)
70 def _create_mutable(self, cap):
71 return FakeMutableFileNode(None,
73 self.encoding_params, None).init_from_cap(cap)
74 def create_mutable_file(self, contents="", keysize=None,
75 version=SDMF_VERSION):
76 n = FakeMutableFileNode(None, None, self.encoding_params, None)
77 return n.create(contents, version=version)
79 class FakeUploader(service.Service):
81 def upload(self, uploadable):
82 d = uploadable.get_size()
83 d.addCallback(lambda size: uploadable.read(size))
86 n = create_chk_filenode(data)
87 results = upload.UploadResults()
88 results.uri = n.get_uri()
90 d.addCallback(_got_data)
92 def get_helper_info(self):
96 def __init__(self, binaryserverid):
97 self.binaryserverid = binaryserverid
98 def get_name(self): return "short"
99 def get_longname(self): return "long"
100 def get_serverid(self): return self.binaryserverid
103 ds = DownloadStatus("storage_index", 1234)
106 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
107 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
108 storage_index = hashutil.storage_index_hash("SI")
109 e0 = ds.add_segment_request(0, now)
111 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
112 e1 = ds.add_segment_request(1, now+2)
114 # two outstanding requests
115 e2 = ds.add_segment_request(2, now+4)
116 e3 = ds.add_segment_request(3, now+5)
117 del e2,e3 # hush pyflakes
119 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
120 e = ds.add_segment_request(4, now)
122 e.deliver(now, 0, 140, 0.5)
124 e = ds.add_dyhb_request(serverA, now)
125 e.finished([1,2], now+1)
126 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
128 e = ds.add_read_event(0, 120, now)
129 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
131 e = ds.add_read_event(120, 30, now+2) # left unfinished
133 e = ds.add_block_request(serverA, 1, 100, 20, now)
134 e.finished(20, now+1)
135 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
137 # make sure that add_read_event() can come first too
138 ds1 = DownloadStatus(storage_index, 1234)
139 e = ds1.add_read_event(0, 120, now)
140 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
146 _all_upload_status = [upload.UploadStatus()]
147 _all_download_status = [build_one_ds()]
148 _all_mapupdate_statuses = [servermap.UpdateStatus()]
149 _all_publish_statuses = [publish.PublishStatus()]
150 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
152 def list_all_upload_statuses(self):
153 return self._all_upload_status
154 def list_all_download_statuses(self):
155 return self._all_download_status
156 def list_all_mapupdate_statuses(self):
157 return self._all_mapupdate_statuses
158 def list_all_publish_statuses(self):
159 return self._all_publish_statuses
160 def list_all_retrieve_statuses(self):
161 return self._all_retrieve_statuses
162 def list_all_helper_statuses(self):
165 class FakeClient(Client):
167 # don't upcall to Client.__init__, since we only want to initialize a
169 service.MultiService.__init__(self)
170 self.nodeid = "fake_nodeid"
171 self.nickname = "fake_nickname"
172 self.introducer_furl = "None"
173 self.stats_provider = FakeStatsProvider()
174 self._secret_holder = SecretHolder("lease secret", "convergence secret")
176 self.convergence = "some random string"
177 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
178 self.introducer_client = None
179 self.history = FakeHistory()
180 self.uploader = FakeUploader()
181 self.uploader.setServiceParent(self)
182 self.blacklist = None
183 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
186 self.mutable_file_default = SDMF_VERSION
188 def startService(self):
189 return service.MultiService.startService(self)
190 def stopService(self):
191 return service.MultiService.stopService(self)
193 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
195 class WebMixin(object):
197 self.s = FakeClient()
198 self.s.startService()
199 self.staticdir = self.mktemp()
201 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
203 self.ws.setServiceParent(self.s)
204 self.webish_port = self.ws.getPortnum()
205 self.webish_url = self.ws.getURL()
206 assert self.webish_url.endswith("/")
207 self.webish_url = self.webish_url[:-1] # these tests add their own /
209 l = [ self.s.create_dirnode() for x in range(6) ]
210 d = defer.DeferredList(l)
212 self.public_root = res[0][1]
213 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
214 self.public_url = "/uri/" + self.public_root.get_uri()
215 self.private_root = res[1][1]
219 self._foo_uri = foo.get_uri()
220 self._foo_readonly_uri = foo.get_readonly_uri()
221 self._foo_verifycap = foo.get_verify_cap().to_string()
222 # NOTE: we ignore the deferred on all set_uri() calls, because we
223 # know the fake nodes do these synchronously
224 self.public_root.set_uri(u"foo", foo.get_uri(),
225 foo.get_readonly_uri())
227 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
228 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
229 self._bar_txt_verifycap = n.get_verify_cap().to_string()
232 # XXX: Do we ever use this?
233 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
235 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
238 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
239 assert self._quux_txt_uri.startswith("URI:MDMF")
240 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
242 foo.set_uri(u"empty", res[3][1].get_uri(),
243 res[3][1].get_readonly_uri())
244 sub_uri = res[4][1].get_uri()
245 self._sub_uri = sub_uri
246 foo.set_uri(u"sub", sub_uri, sub_uri)
247 sub = self.s.create_node_from_uri(sub_uri)
250 _ign, n, blocking_uri = self.makefile(1)
251 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
253 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
254 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
255 # still think of it as an umlaut
256 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
258 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
259 self._baz_file_uri = baz_file
260 sub.set_uri(u"baz.txt", baz_file, baz_file)
262 _ign, n, self._bad_file_uri = self.makefile(3)
263 # this uri should not be downloadable
264 del FakeCHKFileNode.all_contents[self._bad_file_uri]
267 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
268 rodir.get_readonly_uri())
269 rodir.set_uri(u"nor", baz_file, baz_file)
275 # public/foo/quux.txt
276 # public/foo/blockingfile
279 # public/foo/sub/baz.txt
281 # public/reedownlee/nor
282 self.NEWFILE_CONTENTS = "newfile contents\n"
284 return foo.get_metadata_for(u"bar.txt")
286 def _got_metadata(metadata):
287 self._bar_txt_metadata = metadata
288 d.addCallback(_got_metadata)
291 def makefile(self, number):
292 contents = "contents of file %s\n" % number
293 n = create_chk_filenode(contents)
294 return contents, n, n.get_uri()
296 def makefile_mutable(self, number, mdmf=False):
297 contents = "contents of mutable file %s\n" % number
298 n = create_mutable_filenode(contents, mdmf)
299 return contents, n, n.get_uri(), n.get_readonly_uri()
302 return self.s.stopService()
304 def failUnlessIsBarDotTxt(self, res):
305 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
307 def failUnlessIsQuuxDotTxt(self, res):
308 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
310 def failUnlessIsBazDotTxt(self, res):
311 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
313 def failUnlessIsSubBazDotTxt(self, res):
314 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
316 def failUnlessIsBarJSON(self, res):
317 data = simplejson.loads(res)
318 self.failUnless(isinstance(data, list))
319 self.failUnlessEqual(data[0], "filenode")
320 self.failUnless(isinstance(data[1], dict))
321 self.failIf(data[1]["mutable"])
322 self.failIfIn("rw_uri", data[1]) # immutable
323 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
324 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
325 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
327 def failUnlessIsQuuxJSON(self, res, readonly=False):
328 data = simplejson.loads(res)
329 self.failUnless(isinstance(data, list))
330 self.failUnlessEqual(data[0], "filenode")
331 self.failUnless(isinstance(data[1], dict))
333 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
335 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
336 self.failUnless(metadata['mutable'])
338 self.failIfIn("rw_uri", metadata)
340 self.failUnlessIn("rw_uri", metadata)
341 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
342 self.failUnlessIn("ro_uri", metadata)
343 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
344 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
346 def failUnlessIsFooJSON(self, res):
347 data = simplejson.loads(res)
348 self.failUnless(isinstance(data, list))
349 self.failUnlessEqual(data[0], "dirnode", res)
350 self.failUnless(isinstance(data[1], dict))
351 self.failUnless(data[1]["mutable"])
352 self.failUnlessIn("rw_uri", data[1]) # mutable
353 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
354 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
355 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
357 kidnames = sorted([unicode(n) for n in data[1]["children"]])
358 self.failUnlessEqual(kidnames,
359 [u"bar.txt", u"baz.txt", u"blockingfile",
360 u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
361 kids = dict( [(unicode(name),value)
363 in data[1]["children"].iteritems()] )
364 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
365 self.failUnlessIn("metadata", kids[u"sub"][1])
366 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
367 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
368 self.failUnlessIn("linkcrtime", tahoe_md)
369 self.failUnlessIn("linkmotime", tahoe_md)
370 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
371 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
372 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
373 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
374 self._bar_txt_verifycap)
375 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
376 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
377 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
378 self._bar_txt_metadata["tahoe"]["linkcrtime"])
379 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
381 self.failUnlessIn("quux.txt", kids)
382 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
384 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
385 self._quux_txt_readonly_uri)
387 def GET(self, urlpath, followRedirect=False, return_response=False,
389 # if return_response=True, this fires with (data, statuscode,
390 # respheaders) instead of just data.
391 assert not isinstance(urlpath, unicode)
392 url = self.webish_url + urlpath
393 factory = HTTPClientGETFactory(url, method="GET",
394 followRedirect=followRedirect, **kwargs)
395 reactor.connectTCP("localhost", self.webish_port, factory)
398 return (data, factory.status, factory.response_headers)
400 d.addCallback(_got_data)
401 return factory.deferred
403 def HEAD(self, urlpath, return_response=False, **kwargs):
404 # this requires some surgery, because twisted.web.client doesn't want
405 # to give us back the response headers.
406 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
407 reactor.connectTCP("localhost", self.webish_port, factory)
410 return (data, factory.status, factory.response_headers)
412 d.addCallback(_got_data)
413 return factory.deferred
415 def PUT(self, urlpath, data, **kwargs):
416 url = self.webish_url + urlpath
417 return client.getPage(url, method="PUT", postdata=data, **kwargs)
419 def DELETE(self, urlpath):
420 url = self.webish_url + urlpath
421 return client.getPage(url, method="DELETE")
423 def POST(self, urlpath, followRedirect=False, **fields):
424 sepbase = "boogabooga"
428 form.append('Content-Disposition: form-data; name="_charset"')
432 for name, value in fields.iteritems():
433 if isinstance(value, tuple):
434 filename, value = value
435 form.append('Content-Disposition: form-data; name="%s"; '
436 'filename="%s"' % (name, filename.encode("utf-8")))
438 form.append('Content-Disposition: form-data; name="%s"' % name)
440 if isinstance(value, unicode):
441 value = value.encode("utf-8")
444 assert isinstance(value, str)
451 body = "\r\n".join(form) + "\r\n"
452 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
453 return self.POST2(urlpath, body, headers, followRedirect)
455 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
456 url = self.webish_url + urlpath
457 return client.getPage(url, method="POST", postdata=body,
458 headers=headers, followRedirect=followRedirect)
460 def shouldFail(self, res, expected_failure, which,
461 substring=None, response_substring=None):
462 if isinstance(res, failure.Failure):
463 res.trap(expected_failure)
465 self.failUnlessIn(substring, str(res), which)
466 if response_substring:
467 self.failUnlessIn(response_substring, res.value.response, which)
469 self.fail("%s was supposed to raise %s, not get '%s'" %
470 (which, expected_failure, res))
472 def shouldFail2(self, expected_failure, which, substring,
474 callable, *args, **kwargs):
475 assert substring is None or isinstance(substring, str)
476 assert response_substring is None or isinstance(response_substring, str)
477 d = defer.maybeDeferred(callable, *args, **kwargs)
479 if isinstance(res, failure.Failure):
480 res.trap(expected_failure)
482 self.failUnlessIn(substring, str(res),
483 "'%s' not in '%s' for test '%s'" % \
484 (substring, str(res), which))
485 if response_substring:
486 self.failUnlessIn(response_substring, res.value.response,
487 "'%s' not in '%s' for test '%s'" % \
488 (response_substring, res.value.response,
491 self.fail("%s was supposed to raise %s, not get '%s'" %
492 (which, expected_failure, res))
496 def should404(self, res, which):
497 if isinstance(res, failure.Failure):
498 res.trap(error.Error)
499 self.failUnlessReallyEqual(res.value.status, "404")
501 self.fail("%s was supposed to Error(404), not get '%s'" %
504 def should302(self, res, which):
505 if isinstance(res, failure.Failure):
506 res.trap(error.Error)
507 self.failUnlessReallyEqual(res.value.status, "302")
509 self.fail("%s was supposed to Error(302), not get '%s'" %
513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
514 def test_create(self):
517 def test_welcome(self):
520 self.failUnlessIn('Welcome to Tahoe-LAFS', res)
521 self.failUnlessIn(FAVICON_MARKUP, res)
522 self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
524 self.s.basedir = 'web/test_welcome'
525 fileutil.make_dirs("web/test_welcome")
526 fileutil.make_dirs("web/test_welcome/private")
528 d.addCallback(_check)
531 def test_status(self):
532 h = self.s.get_history()
533 dl_num = h.list_all_download_statuses()[0].get_counter()
534 ul_num = h.list_all_upload_statuses()[0].get_counter()
535 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
536 pub_num = h.list_all_publish_statuses()[0].get_counter()
537 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
538 d = self.GET("/status", followRedirect=True)
540 self.failUnlessIn('Upload and Download Status', res)
541 self.failUnlessIn('"down-%d"' % dl_num, res)
542 self.failUnlessIn('"up-%d"' % ul_num, res)
543 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
544 self.failUnlessIn('"publish-%d"' % pub_num, res)
545 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
546 d.addCallback(_check)
547 d.addCallback(lambda res: self.GET("/status/?t=json"))
548 def _check_json(res):
549 data = simplejson.loads(res)
550 self.failUnless(isinstance(data, dict))
551 #active = data["active"]
552 # TODO: test more. We need a way to fake an active operation
554 d.addCallback(_check_json)
556 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
558 self.failUnlessIn("File Download Status", res)
559 d.addCallback(_check_dl)
560 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
561 def _check_dl_json(res):
562 data = simplejson.loads(res)
563 self.failUnless(isinstance(data, dict))
564 self.failUnlessIn("read", data)
565 self.failUnlessEqual(data["read"][0]["length"], 120)
566 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
567 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
568 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
569 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
570 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
571 # serverids[] keys are strings, since that's what JSON does, but
572 # we'd really like them to be ints
573 self.failUnlessEqual(data["serverids"]["0"], "phwr")
574 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
575 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
576 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
577 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
578 self.failUnlessIn("dyhb", data)
579 self.failUnlessIn("misc", data)
580 d.addCallback(_check_dl_json)
581 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
583 self.failUnlessIn("File Upload Status", res)
584 d.addCallback(_check_ul)
585 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
586 def _check_mapupdate(res):
587 self.failUnlessIn("Mutable File Servermap Update Status", res)
588 d.addCallback(_check_mapupdate)
589 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
590 def _check_publish(res):
591 self.failUnlessIn("Mutable File Publish Status", res)
592 d.addCallback(_check_publish)
593 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
594 def _check_retrieve(res):
595 self.failUnlessIn("Mutable File Retrieve Status", res)
596 d.addCallback(_check_retrieve)
600 def test_status_numbers(self):
601 drrm = status.DownloadResultsRendererMixin()
602 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
603 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
604 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
605 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
606 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
607 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
608 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
609 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
610 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
612 urrm = status.UploadResultsRendererMixin()
613 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
614 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
615 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
616 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
617 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
618 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
619 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
620 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
621 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
623 def test_GET_FILEURL(self):
624 d = self.GET(self.public_url + "/foo/bar.txt")
625 d.addCallback(self.failUnlessIsBarDotTxt)
628 def test_GET_FILEURL_range(self):
629 headers = {"range": "bytes=1-10"}
630 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
631 return_response=True)
632 def _got((res, status, headers)):
633 self.failUnlessReallyEqual(int(status), 206)
634 self.failUnless(headers.has_key("content-range"))
635 self.failUnlessReallyEqual(headers["content-range"][0],
636 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
637 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
641 def test_GET_FILEURL_partial_range(self):
642 headers = {"range": "bytes=5-"}
643 length = len(self.BAR_CONTENTS)
644 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
645 return_response=True)
646 def _got((res, status, headers)):
647 self.failUnlessReallyEqual(int(status), 206)
648 self.failUnless(headers.has_key("content-range"))
649 self.failUnlessReallyEqual(headers["content-range"][0],
650 "bytes 5-%d/%d" % (length-1, length))
651 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
655 def test_GET_FILEURL_partial_end_range(self):
656 headers = {"range": "bytes=-5"}
657 length = len(self.BAR_CONTENTS)
658 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
659 return_response=True)
660 def _got((res, status, headers)):
661 self.failUnlessReallyEqual(int(status), 206)
662 self.failUnless(headers.has_key("content-range"))
663 self.failUnlessReallyEqual(headers["content-range"][0],
664 "bytes %d-%d/%d" % (length-5, length-1, length))
665 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
669 def test_GET_FILEURL_partial_range_overrun(self):
670 headers = {"range": "bytes=100-200"}
671 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
672 "416 Requested Range not satisfiable",
673 "First beyond end of file",
674 self.GET, self.public_url + "/foo/bar.txt",
678 def test_HEAD_FILEURL_range(self):
679 headers = {"range": "bytes=1-10"}
680 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
681 return_response=True)
682 def _got((res, status, headers)):
683 self.failUnlessReallyEqual(res, "")
684 self.failUnlessReallyEqual(int(status), 206)
685 self.failUnless(headers.has_key("content-range"))
686 self.failUnlessReallyEqual(headers["content-range"][0],
687 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
691 def test_HEAD_FILEURL_partial_range(self):
692 headers = {"range": "bytes=5-"}
693 length = len(self.BAR_CONTENTS)
694 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
695 return_response=True)
696 def _got((res, status, headers)):
697 self.failUnlessReallyEqual(int(status), 206)
698 self.failUnless(headers.has_key("content-range"))
699 self.failUnlessReallyEqual(headers["content-range"][0],
700 "bytes 5-%d/%d" % (length-1, length))
704 def test_HEAD_FILEURL_partial_end_range(self):
705 headers = {"range": "bytes=-5"}
706 length = len(self.BAR_CONTENTS)
707 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
708 return_response=True)
709 def _got((res, status, headers)):
710 self.failUnlessReallyEqual(int(status), 206)
711 self.failUnless(headers.has_key("content-range"))
712 self.failUnlessReallyEqual(headers["content-range"][0],
713 "bytes %d-%d/%d" % (length-5, length-1, length))
717 def test_HEAD_FILEURL_partial_range_overrun(self):
718 headers = {"range": "bytes=100-200"}
719 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
720 "416 Requested Range not satisfiable",
722 self.HEAD, self.public_url + "/foo/bar.txt",
726 def test_GET_FILEURL_range_bad(self):
727 headers = {"range": "BOGUS=fizbop-quarnak"}
728 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
729 return_response=True)
730 def _got((res, status, headers)):
731 self.failUnlessReallyEqual(int(status), 200)
732 self.failUnless(not headers.has_key("content-range"))
733 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
737 def test_HEAD_FILEURL(self):
738 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
739 def _got((res, status, headers)):
740 self.failUnlessReallyEqual(res, "")
741 self.failUnlessReallyEqual(headers["content-length"][0],
742 str(len(self.BAR_CONTENTS)))
743 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
747 def test_GET_FILEURL_named(self):
748 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
749 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
750 d = self.GET(base + "/@@name=/blah.txt")
751 d.addCallback(self.failUnlessIsBarDotTxt)
752 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
753 d.addCallback(self.failUnlessIsBarDotTxt)
754 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
755 d.addCallback(self.failUnlessIsBarDotTxt)
756 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
757 d.addCallback(self.failUnlessIsBarDotTxt)
758 save_url = base + "?save=true&filename=blah.txt"
759 d.addCallback(lambda res: self.GET(save_url))
760 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
761 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
762 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
763 u_url = base + "?save=true&filename=" + u_fn_e
764 d.addCallback(lambda res: self.GET(u_url))
765 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
768 def test_PUT_FILEURL_named_bad(self):
769 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
770 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
772 "/file can only be used with GET or HEAD",
773 self.PUT, base + "/@@name=/blah.txt", "")
777 def test_GET_DIRURL_named_bad(self):
778 base = "/file/%s" % urllib.quote(self._foo_uri)
779 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
782 self.GET, base + "/@@name=/blah.txt")
785 def test_GET_slash_file_bad(self):
786 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
788 "/file must be followed by a file-cap and a name",
792 def test_GET_unhandled_URI_named(self):
793 contents, n, newuri = self.makefile(12)
794 verifier_cap = n.get_verify_cap().to_string()
795 base = "/file/%s" % urllib.quote(verifier_cap)
796 # client.create_node_from_uri() can't handle verify-caps
797 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
798 "400 Bad Request", "is not a file-cap",
802 def test_GET_unhandled_URI(self):
803 contents, n, newuri = self.makefile(12)
804 verifier_cap = n.get_verify_cap().to_string()
805 base = "/uri/%s" % urllib.quote(verifier_cap)
806 # client.create_node_from_uri() can't handle verify-caps
807 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
809 "GET unknown URI type: can only do t=info",
813 def test_GET_FILE_URI(self):
814 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
816 d.addCallback(self.failUnlessIsBarDotTxt)
819 def test_GET_FILE_URI_mdmf(self):
820 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
822 d.addCallback(self.failUnlessIsQuuxDotTxt)
825 def test_GET_FILE_URI_mdmf_extensions(self):
826 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
828 d.addCallback(self.failUnlessIsQuuxDotTxt)
831 def test_GET_FILE_URI_mdmf_readonly(self):
832 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
834 d.addCallback(self.failUnlessIsQuuxDotTxt)
837 def test_GET_FILE_URI_badchild(self):
838 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
839 errmsg = "Files have no children, certainly not named 'boguschild'"
840 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
841 "400 Bad Request", errmsg,
845 def test_PUT_FILE_URI_badchild(self):
846 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
847 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
848 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
849 "400 Bad Request", errmsg,
853 def test_PUT_FILE_URI_mdmf(self):
854 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
855 self._quux_new_contents = "new_contents"
857 d.addCallback(lambda res:
858 self.failUnlessIsQuuxDotTxt(res))
859 d.addCallback(lambda ignored:
860 self.PUT(base, self._quux_new_contents))
861 d.addCallback(lambda ignored:
863 d.addCallback(lambda res:
864 self.failUnlessReallyEqual(res, self._quux_new_contents))
867 def test_PUT_FILE_URI_mdmf_extensions(self):
868 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
869 self._quux_new_contents = "new_contents"
871 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
872 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
873 d.addCallback(lambda ignored: self.GET(base))
874 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
878 def test_PUT_FILE_URI_mdmf_readonly(self):
879 # We're not allowed to PUT things to a readonly cap.
880 base = "/uri/%s" % self._quux_txt_readonly_uri
882 d.addCallback(lambda res:
883 self.failUnlessIsQuuxDotTxt(res))
884 # What should we get here? We get a 500 error now; that's not right.
885 d.addCallback(lambda ignored:
886 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
887 "400 Bad Request", "read-only cap",
888 self.PUT, base, "new data"))
891 def test_PUT_FILE_URI_sdmf_readonly(self):
892 # We're not allowed to put things to a readonly cap.
893 base = "/uri/%s" % self._baz_txt_readonly_uri
895 d.addCallback(lambda res:
896 self.failUnlessIsBazDotTxt(res))
897 d.addCallback(lambda ignored:
898 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
899 "400 Bad Request", "read-only cap",
900 self.PUT, base, "new_data"))
903 def test_GET_etags(self):
905 def _check_etags(uri):
907 d2 = _get_etag(uri, 'json')
908 d = defer.DeferredList([d1, d2], consumeErrors=True)
910 # All deferred must succeed
911 self.failUnless(all([r[0] for r in results]))
912 # the etag for the t=json form should be just like the etag
913 # fo the default t='' form, but with a 'json' suffix
914 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
915 d.addCallback(_check)
918 def _get_etag(uri, t=''):
919 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
920 d = self.GET(targetbase, return_response=True, followRedirect=True)
921 def _just_the_etag(result):
922 data, response, headers = result
923 etag = headers['etag'][0]
924 if uri.startswith('URI:DIR'):
925 self.failUnless(etag.startswith('DIR:'), etag)
927 return d.addCallback(_just_the_etag)
929 # Check that etags work with immutable directories
930 (newkids, caps) = self._create_immutable_children()
931 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
932 simplejson.dumps(newkids))
933 def _stash_immdir_uri(uri):
934 self._immdir_uri = uri
936 d.addCallback(_stash_immdir_uri)
937 d.addCallback(_check_etags)
939 # Check that etags work with immutable files
940 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
942 # use the ETag on GET
943 def _check_match(ign):
944 uri = "/uri/%s" % self._bar_txt_uri
945 d = self.GET(uri, return_response=True)
947 d.addCallback(lambda (data, code, headers):
949 # do a GET that's supposed to match the ETag
950 d.addCallback(lambda etag:
951 self.GET(uri, return_response=True,
952 headers={"If-None-Match": etag}))
953 # make sure it short-circuited (304 instead of 200)
954 d.addCallback(lambda (data, code, headers):
955 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
957 d.addCallback(_check_match)
959 def _no_etag(uri, t):
960 target = "/uri/%s?t=%s" % (uri, t)
961 d = self.GET(target, return_response=True, followRedirect=True)
962 d.addCallback(lambda (data, code, headers):
963 self.failIf("etag" in headers, target))
965 def _yes_etag(uri, t):
966 target = "/uri/%s?t=%s" % (uri, t)
967 d = self.GET(target, return_response=True, followRedirect=True)
968 d.addCallback(lambda (data, code, headers):
969 self.failUnless("etag" in headers, target))
972 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
973 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
974 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
975 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
976 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
978 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
979 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
980 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
981 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
982 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
983 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
987 # TODO: version of this with a Unicode filename
988 def test_GET_FILEURL_save(self):
989 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
990 return_response=True)
991 def _got((res, statuscode, headers)):
992 content_disposition = headers["content-disposition"][0]
993 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
994 self.failUnlessIsBarDotTxt(res)
998 def test_GET_FILEURL_missing(self):
999 d = self.GET(self.public_url + "/foo/missing")
1000 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1003 def test_GET_FILEURL_info_mdmf(self):
1004 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1006 self.failUnlessIn("mutable file (mdmf)", res)
1007 self.failUnlessIn(self._quux_txt_uri, res)
1008 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1012 def test_GET_FILEURL_info_mdmf_readonly(self):
1013 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1015 self.failUnlessIn("mutable file (mdmf)", res)
1016 self.failIfIn(self._quux_txt_uri, res)
1017 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1021 def test_GET_FILEURL_info_sdmf(self):
1022 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1024 self.failUnlessIn("mutable file (sdmf)", res)
1025 self.failUnlessIn(self._baz_txt_uri, res)
1029 def test_GET_FILEURL_info_mdmf_extensions(self):
1030 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1032 self.failUnlessIn("mutable file (mdmf)", res)
1033 self.failUnlessIn(self._quux_txt_uri, res)
1034 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1038 def test_PUT_overwrite_only_files(self):
1039 # create a directory, put a file in that directory.
1040 contents, n, filecap = self.makefile(8)
1041 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1042 d.addCallback(lambda res:
1043 self.PUT(self.public_url + "/foo/dir/file1.txt",
1044 self.NEWFILE_CONTENTS))
1045 # try to overwrite the file with replace=only-files
1046 # (this should work)
1047 d.addCallback(lambda res:
1048 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1050 d.addCallback(lambda res:
1051 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1052 "There was already a child by that name, and you asked me "
1053 "to not replace it",
1054 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1058 def test_PUT_NEWFILEURL(self):
1059 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1060 # TODO: we lose the response code, so we can't check this
1061 #self.failUnlessReallyEqual(responsecode, 201)
1062 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1063 d.addCallback(lambda res:
1064 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1065 self.NEWFILE_CONTENTS))
1068 def test_PUT_NEWFILEURL_not_mutable(self):
1069 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1070 self.NEWFILE_CONTENTS)
1071 # TODO: we lose the response code, so we can't check this
1072 #self.failUnlessReallyEqual(responsecode, 201)
1073 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1074 d.addCallback(lambda res:
1075 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1076 self.NEWFILE_CONTENTS))
1079 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1080 # this should get us a few segments of an MDMF mutable file,
1081 # which we can then test for.
1082 contents = self.NEWFILE_CONTENTS * 300000
1083 d = self.PUT("/uri?format=mdmf",
1085 def _got_filecap(filecap):
1086 self.failUnless(filecap.startswith("URI:MDMF"))
1088 d.addCallback(_got_filecap)
1089 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1090 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1093 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1094 contents = self.NEWFILE_CONTENTS * 300000
1095 d = self.PUT("/uri?format=sdmf",
1097 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1098 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1101 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1102 contents = self.NEWFILE_CONTENTS * 300000
1103 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1104 400, "Bad Request", "Unknown format: foo",
1105 self.PUT, "/uri?format=foo",
1108 def test_PUT_NEWFILEURL_range_bad(self):
1109 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1110 target = self.public_url + "/foo/new.txt"
1111 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1112 "501 Not Implemented",
1113 "Content-Range in PUT not yet supported",
1114 # (and certainly not for immutable files)
1115 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1117 d.addCallback(lambda res:
1118 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1121 def test_PUT_NEWFILEURL_mutable(self):
1122 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1123 self.NEWFILE_CONTENTS)
1124 # TODO: we lose the response code, so we can't check this
1125 #self.failUnlessReallyEqual(responsecode, 201)
1126 def _check_uri(res):
1127 u = uri.from_string_mutable_filenode(res)
1128 self.failUnless(u.is_mutable())
1129 self.failIf(u.is_readonly())
1131 d.addCallback(_check_uri)
1132 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1133 d.addCallback(lambda res:
1134 self.failUnlessMutableChildContentsAre(self._foo_node,
1136 self.NEWFILE_CONTENTS))
1139 def test_PUT_NEWFILEURL_mutable_toobig(self):
1140 # It is okay to upload large mutable files, so we should be able
1142 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1143 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1146 def test_PUT_NEWFILEURL_replace(self):
1147 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1148 # TODO: we lose the response code, so we can't check this
1149 #self.failUnlessReallyEqual(responsecode, 200)
1150 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1151 d.addCallback(lambda res:
1152 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1153 self.NEWFILE_CONTENTS))
1156 def test_PUT_NEWFILEURL_bad_t(self):
1157 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1158 "PUT to a file: bad t=bogus",
1159 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1163 def test_PUT_NEWFILEURL_no_replace(self):
1164 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1165 self.NEWFILE_CONTENTS)
1166 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1168 "There was already a child by that name, and you asked me "
1169 "to not replace it")
1172 def test_PUT_NEWFILEURL_mkdirs(self):
1173 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1175 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1176 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1177 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1178 d.addCallback(lambda res:
1179 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1180 self.NEWFILE_CONTENTS))
1183 def test_PUT_NEWFILEURL_blocked(self):
1184 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1185 self.NEWFILE_CONTENTS)
1186 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1188 "Unable to create directory 'blockingfile': a file was in the way")
1191 def test_PUT_NEWFILEURL_emptyname(self):
1192 # an empty pathname component (i.e. a double-slash) is disallowed
1193 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1195 "The webapi does not allow empty pathname components",
1196 self.PUT, self.public_url + "/foo//new.txt", "")
1199 def test_DELETE_FILEURL(self):
1200 d = self.DELETE(self.public_url + "/foo/bar.txt")
1201 d.addCallback(lambda res:
1202 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1205 def test_DELETE_FILEURL_missing(self):
1206 d = self.DELETE(self.public_url + "/foo/missing")
1207 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1210 def test_DELETE_FILEURL_missing2(self):
1211 d = self.DELETE(self.public_url + "/missing/missing")
1212 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1215 def failUnlessHasBarDotTxtMetadata(self, res):
1216 data = simplejson.loads(res)
1217 self.failUnless(isinstance(data, list))
1218 self.failUnlessIn("metadata", data[1])
1219 self.failUnlessIn("tahoe", data[1]["metadata"])
1220 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1221 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1222 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1223 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1225 def test_GET_FILEURL_json(self):
1226 # twisted.web.http.parse_qs ignores any query args without an '=', so
1227 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1228 # instead. This may make it tricky to emulate the S3 interface
1230 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1232 self.failUnlessIsBarJSON(data)
1233 self.failUnlessHasBarDotTxtMetadata(data)
1235 d.addCallback(_check1)
1238 def test_GET_FILEURL_json_mutable_type(self):
1239 # The JSON should include format, which says whether the
1240 # file is SDMF or MDMF
1241 d = self.PUT("/uri?format=mdmf",
1242 self.NEWFILE_CONTENTS * 300000)
1243 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1244 def _got_json(json, version):
1245 data = simplejson.loads(json)
1246 assert "filenode" == data[0]
1248 assert isinstance(data, dict)
1250 self.failUnlessIn("format", data)
1251 self.failUnlessEqual(data["format"], version)
1253 d.addCallback(_got_json, "MDMF")
1254 # Now make an SDMF file and check that it is reported correctly.
1255 d.addCallback(lambda ignored:
1256 self.PUT("/uri?format=sdmf",
1257 self.NEWFILE_CONTENTS * 300000))
1258 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1259 d.addCallback(_got_json, "SDMF")
1262 def test_GET_FILEURL_json_mdmf(self):
1263 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1264 d.addCallback(self.failUnlessIsQuuxJSON)
1267 def test_GET_FILEURL_json_missing(self):
1268 d = self.GET(self.public_url + "/foo/missing?json")
1269 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1272 def test_GET_FILEURL_uri(self):
1273 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1275 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1276 d.addCallback(_check)
1277 d.addCallback(lambda res:
1278 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1280 # for now, for files, uris and readonly-uris are the same
1281 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1282 d.addCallback(_check2)
1285 def test_GET_FILEURL_badtype(self):
1286 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1289 self.public_url + "/foo/bar.txt?t=bogus")
1292 def test_CSS_FILE(self):
1293 d = self.GET("/tahoe.css", followRedirect=True)
1295 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1296 self.failUnless(CSS_STYLE.search(res), res)
1297 d.addCallback(_check)
1300 def test_GET_FILEURL_uri_missing(self):
1301 d = self.GET(self.public_url + "/foo/missing?t=uri")
1302 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1305 def _check_upload_and_mkdir_forms(self, html):
1306 # We should have a form to create a file, with radio buttons that allow
1307 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1308 self.failUnlessIn('name="t" value="upload"', html)
1309 self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1310 self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1311 self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1313 # We should also have the ability to create a mutable directory, with
1314 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1315 # or MDMF directory.
1316 self.failUnlessIn('name="t" value="mkdir"', html)
1317 self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1318 self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1320 self.failUnlessIn(FAVICON_MARKUP, html)
1322 def test_GET_DIRECTORY_html(self):
1323 d = self.GET(self.public_url + "/foo", followRedirect=True)
1325 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1326 self._check_upload_and_mkdir_forms(html)
1327 self.failUnlessIn("quux", html)
1328 d.addCallback(_check)
1331 def test_GET_root_html(self):
1333 d.addCallback(self._check_upload_and_mkdir_forms)
1336 def test_GET_DIRURL(self):
1337 # the addSlash means we get a redirect here
1338 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1340 d = self.GET(self.public_url + "/foo", followRedirect=True)
1342 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1344 # the FILE reference points to a URI, but it should end in bar.txt
1345 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1346 (ROOT, urllib.quote(self._bar_txt_uri)))
1347 get_bar = "".join([r'<td>FILE</td>',
1349 r'<a href="%s">bar.txt</a>' % bar_url,
1351 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1353 self.failUnless(re.search(get_bar, res), res)
1354 for label in ['unlink', 'rename/move']:
1355 for line in res.split("\n"):
1356 # find the line that contains the relevant button for bar.txt
1357 if ("form action" in line and
1358 ('value="%s"' % (label,)) in line and
1359 'value="bar.txt"' in line):
1360 # the form target should use a relative URL
1361 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1362 self.failUnlessIn('action="%s"' % foo_url, line)
1363 # and the when_done= should too
1364 #done_url = urllib.quote(???)
1365 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1367 # 'unlink' needs to use POST because it directly has a side effect
1368 if label == 'unlink':
1369 self.failUnlessIn('method="post"', line)
1372 self.fail("unable to find '%s bar.txt' line" % (label,))
1374 # the DIR reference just points to a URI
1375 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1376 get_sub = ((r'<td>DIR</td>')
1377 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1378 self.failUnless(re.search(get_sub, res), res)
1379 d.addCallback(_check)
1381 # look at a readonly directory
1382 d.addCallback(lambda res:
1383 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1385 self.failUnlessIn("(read-only)", res)
1386 self.failIfIn("Upload a file", res)
1387 d.addCallback(_check2)
1389 # and at a directory that contains a readonly directory
1390 d.addCallback(lambda res:
1391 self.GET(self.public_url, followRedirect=True))
1393 self.failUnless(re.search('<td>DIR-RO</td>'
1394 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1395 d.addCallback(_check3)
1397 # and an empty directory
1398 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1400 self.failUnlessIn("directory is empty", res)
1401 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1402 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1403 d.addCallback(_check4)
1405 # and at a literal directory
1406 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1407 d.addCallback(lambda res:
1408 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1410 self.failUnlessIn('(immutable)', res)
1411 self.failUnless(re.search('<td>FILE</td>'
1412 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1413 d.addCallback(_check5)
1416 def test_GET_DIRURL_badtype(self):
1417 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1421 self.public_url + "/foo?t=bogus")
1424 def test_GET_DIRURL_json(self):
1425 d = self.GET(self.public_url + "/foo?t=json")
1426 d.addCallback(self.failUnlessIsFooJSON)
1429 def test_GET_DIRURL_json_format(self):
1430 d = self.PUT(self.public_url + \
1431 "/foo/sdmf.txt?format=sdmf",
1432 self.NEWFILE_CONTENTS * 300000)
1433 d.addCallback(lambda ignored:
1434 self.PUT(self.public_url + \
1435 "/foo/mdmf.txt?format=mdmf",
1436 self.NEWFILE_CONTENTS * 300000))
1437 # Now we have an MDMF and SDMF file in the directory. If we GET
1438 # its JSON, we should see their encodings.
1439 d.addCallback(lambda ignored:
1440 self.GET(self.public_url + "/foo?t=json"))
1441 def _got_json(json):
1442 data = simplejson.loads(json)
1443 assert data[0] == "dirnode"
1446 kids = data['children']
1448 mdmf_data = kids['mdmf.txt'][1]
1449 self.failUnlessIn("format", mdmf_data)
1450 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1452 sdmf_data = kids['sdmf.txt'][1]
1453 self.failUnlessIn("format", sdmf_data)
1454 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1455 d.addCallback(_got_json)
1459 def test_POST_DIRURL_manifest_no_ophandle(self):
1460 d = self.shouldFail2(error.Error,
1461 "test_POST_DIRURL_manifest_no_ophandle",
1463 "slow operation requires ophandle=",
1464 self.POST, self.public_url, t="start-manifest")
1467 def test_POST_DIRURL_manifest(self):
1468 d = defer.succeed(None)
1469 def getman(ignored, output):
1470 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1471 followRedirect=True)
1472 d.addCallback(self.wait_for_operation, "125")
1473 d.addCallback(self.get_operation_results, "125", output)
1475 d.addCallback(getman, None)
1476 def _got_html(manifest):
1477 self.failUnlessIn("Manifest of SI=", manifest)
1478 self.failUnlessIn("<td>sub</td>", manifest)
1479 self.failUnlessIn(self._sub_uri, manifest)
1480 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1481 self.failUnlessIn(FAVICON_MARKUP, manifest)
1482 d.addCallback(_got_html)
1484 # both t=status and unadorned GET should be identical
1485 d.addCallback(lambda res: self.GET("/operations/125"))
1486 d.addCallback(_got_html)
1488 d.addCallback(getman, "html")
1489 d.addCallback(_got_html)
1490 d.addCallback(getman, "text")
1491 def _got_text(manifest):
1492 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1493 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1494 d.addCallback(_got_text)
1495 d.addCallback(getman, "JSON")
1497 data = res["manifest"]
1499 for (path_list, cap) in data:
1500 got[tuple(path_list)] = cap
1501 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1502 self.failUnlessIn((u"sub", u"baz.txt"), got)
1503 self.failUnlessIn("finished", res)
1504 self.failUnlessIn("origin", res)
1505 self.failUnlessIn("storage-index", res)
1506 self.failUnlessIn("verifycaps", res)
1507 self.failUnlessIn("stats", res)
1508 d.addCallback(_got_json)
1511 def test_POST_DIRURL_deepsize_no_ophandle(self):
1512 d = self.shouldFail2(error.Error,
1513 "test_POST_DIRURL_deepsize_no_ophandle",
1515 "slow operation requires ophandle=",
1516 self.POST, self.public_url, t="start-deep-size")
1519 def test_POST_DIRURL_deepsize(self):
1520 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1521 followRedirect=True)
1522 d.addCallback(self.wait_for_operation, "126")
1523 d.addCallback(self.get_operation_results, "126", "json")
1524 def _got_json(data):
1525 self.failUnlessReallyEqual(data["finished"], True)
1527 self.failUnless(size > 1000)
1528 d.addCallback(_got_json)
1529 d.addCallback(self.get_operation_results, "126", "text")
1531 mo = re.search(r'^size: (\d+)$', res, re.M)
1532 self.failUnless(mo, res)
1533 size = int(mo.group(1))
1534 # with directories, the size varies.
1535 self.failUnless(size > 1000)
1536 d.addCallback(_got_text)
1539 def test_POST_DIRURL_deepstats_no_ophandle(self):
1540 d = self.shouldFail2(error.Error,
1541 "test_POST_DIRURL_deepstats_no_ophandle",
1543 "slow operation requires ophandle=",
1544 self.POST, self.public_url, t="start-deep-stats")
1547 def test_POST_DIRURL_deepstats(self):
1548 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1549 followRedirect=True)
1550 d.addCallback(self.wait_for_operation, "127")
1551 d.addCallback(self.get_operation_results, "127", "json")
1552 def _got_json(stats):
1553 expected = {"count-immutable-files": 3,
1554 "count-mutable-files": 2,
1555 "count-literal-files": 0,
1557 "count-directories": 3,
1558 "size-immutable-files": 57,
1559 "size-literal-files": 0,
1560 #"size-directories": 1912, # varies
1561 #"largest-directory": 1590,
1562 "largest-directory-children": 7,
1563 "largest-immutable-file": 19,
1565 for k,v in expected.iteritems():
1566 self.failUnlessReallyEqual(stats[k], v,
1567 "stats[%s] was %s, not %s" %
1569 self.failUnlessReallyEqual(stats["size-files-histogram"],
1571 d.addCallback(_got_json)
1574 def test_POST_DIRURL_stream_manifest(self):
1575 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1577 self.failUnless(res.endswith("\n"))
1578 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1579 self.failUnlessReallyEqual(len(units), 9)
1580 self.failUnlessEqual(units[-1]["type"], "stats")
1582 self.failUnlessEqual(first["path"], [])
1583 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1584 self.failUnlessEqual(first["type"], "directory")
1585 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1586 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1587 self.failIfEqual(baz["storage-index"], None)
1588 self.failIfEqual(baz["verifycap"], None)
1589 self.failIfEqual(baz["repaircap"], None)
1590 # XXX: Add quux and baz to this test.
1592 d.addCallback(_check)
1595 def test_GET_DIRURL_uri(self):
1596 d = self.GET(self.public_url + "/foo?t=uri")
1598 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1599 d.addCallback(_check)
1602 def test_GET_DIRURL_readonly_uri(self):
1603 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1605 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1606 d.addCallback(_check)
1609 def test_PUT_NEWDIRURL(self):
1610 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1611 d.addCallback(lambda res:
1612 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1613 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1614 d.addCallback(self.failUnlessNodeKeysAre, [])
1617 def test_PUT_NEWDIRURL_mdmf(self):
1618 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1619 d.addCallback(lambda res:
1620 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1621 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1622 d.addCallback(lambda node:
1623 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1626 def test_PUT_NEWDIRURL_sdmf(self):
1627 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1629 d.addCallback(lambda res:
1630 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1631 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1632 d.addCallback(lambda node:
1633 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1636 def test_PUT_NEWDIRURL_bad_format(self):
1637 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1638 400, "Bad Request", "Unknown format: foo",
1639 self.PUT, self.public_url +
1640 "/foo/newdir=?t=mkdir&format=foo", "")
1642 def test_POST_NEWDIRURL(self):
1643 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1644 d.addCallback(lambda res:
1645 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1646 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1647 d.addCallback(self.failUnlessNodeKeysAre, [])
1650 def test_POST_NEWDIRURL_mdmf(self):
1651 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1652 d.addCallback(lambda res:
1653 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1654 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1655 d.addCallback(lambda node:
1656 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1659 def test_POST_NEWDIRURL_sdmf(self):
1660 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1661 d.addCallback(lambda res:
1662 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1663 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1664 d.addCallback(lambda node:
1665 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1668 def test_POST_NEWDIRURL_bad_format(self):
1669 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1670 400, "Bad Request", "Unknown format: foo",
1671 self.POST2, self.public_url + \
1672 "/foo/newdir?t=mkdir&format=foo", "")
1674 def test_POST_NEWDIRURL_emptyname(self):
1675 # an empty pathname component (i.e. a double-slash) is disallowed
1676 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1678 "The webapi does not allow empty pathname components, i.e. a double slash",
1679 self.POST, self.public_url + "//?t=mkdir")
1682 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1683 (newkids, caps) = self._create_initial_children()
1684 query = "/foo/newdir?t=mkdir-with-children"
1685 if version == MDMF_VERSION:
1686 query += "&format=mdmf"
1687 elif version == SDMF_VERSION:
1688 query += "&format=sdmf"
1690 version = SDMF_VERSION # for later
1691 d = self.POST2(self.public_url + query,
1692 simplejson.dumps(newkids))
1694 n = self.s.create_node_from_uri(uri.strip())
1695 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1696 self.failUnlessEqual(n._node.get_version(), version)
1697 d2.addCallback(lambda ign:
1698 self.failUnlessROChildURIIs(n, u"child-imm",
1700 d2.addCallback(lambda ign:
1701 self.failUnlessRWChildURIIs(n, u"child-mutable",
1703 d2.addCallback(lambda ign:
1704 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1706 d2.addCallback(lambda ign:
1707 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1708 caps['unknown_rocap']))
1709 d2.addCallback(lambda ign:
1710 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1711 caps['unknown_rwcap']))
1712 d2.addCallback(lambda ign:
1713 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1714 caps['unknown_immcap']))
1715 d2.addCallback(lambda ign:
1716 self.failUnlessRWChildURIIs(n, u"dirchild",
1718 d2.addCallback(lambda ign:
1719 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1721 d2.addCallback(lambda ign:
1722 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1723 caps['emptydircap']))
1725 d.addCallback(_check)
1726 d.addCallback(lambda res:
1727 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1728 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1729 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1730 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1731 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1734 def test_POST_NEWDIRURL_initial_children(self):
1735 return self._do_POST_NEWDIRURL_initial_children_test()
1737 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1738 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1740 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1741 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1743 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1744 (newkids, caps) = self._create_initial_children()
1745 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1746 400, "Bad Request", "Unknown format: foo",
1747 self.POST2, self.public_url + \
1748 "/foo/newdir?t=mkdir-with-children&format=foo",
1749 simplejson.dumps(newkids))
1751 def test_POST_NEWDIRURL_immutable(self):
1752 (newkids, caps) = self._create_immutable_children()
1753 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1754 simplejson.dumps(newkids))
1756 n = self.s.create_node_from_uri(uri.strip())
1757 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1758 d2.addCallback(lambda ign:
1759 self.failUnlessROChildURIIs(n, u"child-imm",
1761 d2.addCallback(lambda ign:
1762 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1763 caps['unknown_immcap']))
1764 d2.addCallback(lambda ign:
1765 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1767 d2.addCallback(lambda ign:
1768 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1770 d2.addCallback(lambda ign:
1771 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1772 caps['emptydircap']))
1774 d.addCallback(_check)
1775 d.addCallback(lambda res:
1776 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1777 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1778 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1779 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1780 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1781 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1782 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1783 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1784 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1785 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1786 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1787 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1788 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1789 d.addErrback(self.explain_web_error)
1792 def test_POST_NEWDIRURL_immutable_bad(self):
1793 (newkids, caps) = self._create_initial_children()
1794 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1796 "needed to be immutable but was not",
1798 self.public_url + "/foo/newdir?t=mkdir-immutable",
1799 simplejson.dumps(newkids))
1802 def test_PUT_NEWDIRURL_exists(self):
1803 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1804 d.addCallback(lambda res:
1805 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1806 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1807 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1810 def test_PUT_NEWDIRURL_blocked(self):
1811 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1812 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1814 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1815 d.addCallback(lambda res:
1816 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1817 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1818 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1821 def test_PUT_NEWDIRURL_mkdirs(self):
1822 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1823 d.addCallback(lambda res:
1824 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1825 d.addCallback(lambda res:
1826 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1827 d.addCallback(lambda res:
1828 self._foo_node.get_child_at_path(u"subdir/newdir"))
1829 d.addCallback(self.failUnlessNodeKeysAre, [])
1832 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1833 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1834 d.addCallback(lambda ignored:
1835 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1836 d.addCallback(lambda ignored:
1837 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1838 d.addCallback(lambda ignored:
1839 self._foo_node.get_child_at_path(u"subdir"))
1840 def _got_subdir(subdir):
1841 # XXX: What we want?
1842 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1843 self.failUnlessNodeHasChild(subdir, u"newdir")
1844 return subdir.get_child_at_path(u"newdir")
1845 d.addCallback(_got_subdir)
1846 d.addCallback(lambda newdir:
1847 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1850 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1851 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1852 d.addCallback(lambda ignored:
1853 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1854 d.addCallback(lambda ignored:
1855 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1856 d.addCallback(lambda ignored:
1857 self._foo_node.get_child_at_path(u"subdir"))
1858 def _got_subdir(subdir):
1859 # XXX: What we want?
1860 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1861 self.failUnlessNodeHasChild(subdir, u"newdir")
1862 return subdir.get_child_at_path(u"newdir")
1863 d.addCallback(_got_subdir)
1864 d.addCallback(lambda newdir:
1865 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1868 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1869 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1870 400, "Bad Request", "Unknown format: foo",
1871 self.PUT, self.public_url + \
1872 "/foo/subdir/newdir?t=mkdir&format=foo",
1875 def test_DELETE_DIRURL(self):
1876 d = self.DELETE(self.public_url + "/foo")
1877 d.addCallback(lambda res:
1878 self.failIfNodeHasChild(self.public_root, u"foo"))
1881 def test_DELETE_DIRURL_missing(self):
1882 d = self.DELETE(self.public_url + "/foo/missing")
1883 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1884 d.addCallback(lambda res:
1885 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1888 def test_DELETE_DIRURL_missing2(self):
1889 d = self.DELETE(self.public_url + "/missing")
1890 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1893 def dump_root(self):
1895 w = webish.DirnodeWalkerMixin()
1896 def visitor(childpath, childnode, metadata):
1898 d = w.walk(self.public_root, visitor)
1901 def failUnlessNodeKeysAre(self, node, expected_keys):
1902 for k in expected_keys:
1903 assert isinstance(k, unicode)
1905 def _check(children):
1906 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1907 d.addCallback(_check)
1909 def failUnlessNodeHasChild(self, node, name):
1910 assert isinstance(name, unicode)
1912 def _check(children):
1913 self.failUnlessIn(name, children)
1914 d.addCallback(_check)
1916 def failIfNodeHasChild(self, node, name):
1917 assert isinstance(name, unicode)
1919 def _check(children):
1920 self.failIfIn(name, children)
1921 d.addCallback(_check)
1924 def failUnlessChildContentsAre(self, node, name, expected_contents):
1925 assert isinstance(name, unicode)
1926 d = node.get_child_at_path(name)
1927 d.addCallback(lambda node: download_to_data(node))
1928 def _check(contents):
1929 self.failUnlessReallyEqual(contents, expected_contents)
1930 d.addCallback(_check)
1933 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1934 assert isinstance(name, unicode)
1935 d = node.get_child_at_path(name)
1936 d.addCallback(lambda node: node.download_best_version())
1937 def _check(contents):
1938 self.failUnlessReallyEqual(contents, expected_contents)
1939 d.addCallback(_check)
1942 def failUnlessRWChildURIIs(self, node, name, expected_uri):
1943 assert isinstance(name, unicode)
1944 d = node.get_child_at_path(name)
1946 self.failUnless(child.is_unknown() or not child.is_readonly())
1947 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1948 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1949 expected_ro_uri = self._make_readonly(expected_uri)
1951 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1952 d.addCallback(_check)
1955 def failUnlessROChildURIIs(self, node, name, expected_uri):
1956 assert isinstance(name, unicode)
1957 d = node.get_child_at_path(name)
1959 self.failUnless(child.is_unknown() or child.is_readonly())
1960 self.failUnlessReallyEqual(child.get_write_uri(), None)
1961 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1962 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1963 d.addCallback(_check)
1966 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1967 assert isinstance(name, unicode)
1968 d = node.get_child_at_path(name)
1970 self.failUnless(child.is_unknown() or not child.is_readonly())
1971 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1972 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1973 expected_ro_uri = self._make_readonly(got_uri)
1975 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1976 d.addCallback(_check)
1979 def failUnlessURIMatchesROChild(self, got_uri, node, name):
1980 assert isinstance(name, unicode)
1981 d = node.get_child_at_path(name)
1983 self.failUnless(child.is_unknown() or child.is_readonly())
1984 self.failUnlessReallyEqual(child.get_write_uri(), None)
1985 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1986 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1987 d.addCallback(_check)
1990 def failUnlessCHKURIHasContents(self, got_uri, contents):
1991 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1993 def test_POST_upload(self):
1994 d = self.POST(self.public_url + "/foo", t="upload",
1995 file=("new.txt", self.NEWFILE_CONTENTS))
1997 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1998 d.addCallback(lambda res:
1999 self.failUnlessChildContentsAre(fn, u"new.txt",
2000 self.NEWFILE_CONTENTS))
2003 def test_POST_upload_unicode(self):
2004 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2005 d = self.POST(self.public_url + "/foo", t="upload",
2006 file=(filename, self.NEWFILE_CONTENTS))
2008 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2009 d.addCallback(lambda res:
2010 self.failUnlessChildContentsAre(fn, filename,
2011 self.NEWFILE_CONTENTS))
2012 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2013 d.addCallback(lambda res: self.GET(target_url))
2014 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2015 self.NEWFILE_CONTENTS,
2019 def test_POST_upload_unicode_named(self):
2020 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2021 d = self.POST(self.public_url + "/foo", t="upload",
2023 file=("overridden", self.NEWFILE_CONTENTS))
2025 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2026 d.addCallback(lambda res:
2027 self.failUnlessChildContentsAre(fn, filename,
2028 self.NEWFILE_CONTENTS))
2029 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2030 d.addCallback(lambda res: self.GET(target_url))
2031 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2032 self.NEWFILE_CONTENTS,
2036 def test_POST_upload_no_link(self):
2037 d = self.POST("/uri", t="upload",
2038 file=("new.txt", self.NEWFILE_CONTENTS))
2039 def _check_upload_results(page):
2040 # this should be a page which describes the results of the upload
2041 # that just finished.
2042 self.failUnlessIn("Upload Results:", page)
2043 self.failUnlessIn("URI:", page)
2044 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2045 mo = uri_re.search(page)
2046 self.failUnless(mo, page)
2047 new_uri = mo.group(1)
2049 d.addCallback(_check_upload_results)
2050 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2053 def test_POST_upload_no_link_whendone(self):
2054 d = self.POST("/uri", t="upload", when_done="/",
2055 file=("new.txt", self.NEWFILE_CONTENTS))
2056 d.addBoth(self.shouldRedirect, "/")
2059 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2060 d = defer.maybeDeferred(callable, *args, **kwargs)
2062 if isinstance(res, failure.Failure):
2063 res.trap(error.PageRedirect)
2064 statuscode = res.value.status
2065 target = res.value.location
2066 return checker(statuscode, target)
2067 self.fail("%s: callable was supposed to redirect, not return '%s'"
2072 def test_POST_upload_no_link_whendone_results(self):
2073 def check(statuscode, target):
2074 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2075 self.failUnless(target.startswith(self.webish_url), target)
2076 return client.getPage(target, method="GET")
2077 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2079 self.POST, "/uri", t="upload",
2080 when_done="/uri/%(uri)s",
2081 file=("new.txt", self.NEWFILE_CONTENTS))
2082 d.addCallback(lambda res:
2083 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2086 def test_POST_upload_no_link_mutable(self):
2087 d = self.POST("/uri", t="upload", mutable="true",
2088 file=("new.txt", self.NEWFILE_CONTENTS))
2089 def _check(filecap):
2090 filecap = filecap.strip()
2091 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2092 self.filecap = filecap
2093 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2094 self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2095 n = self.s.create_node_from_uri(filecap)
2096 return n.download_best_version()
2097 d.addCallback(_check)
2099 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2100 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2101 d.addCallback(_check2)
2103 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2104 return self.GET("/file/%s" % urllib.quote(self.filecap))
2105 d.addCallback(_check3)
2107 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2108 d.addCallback(_check4)
2111 def test_POST_upload_no_link_mutable_toobig(self):
2112 # The SDMF size limit is no longer in place, so we should be
2113 # able to upload mutable files that are as large as we want them
2115 d = self.POST("/uri", t="upload", mutable="true",
2116 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2120 def test_POST_upload_format_unlinked(self):
2121 def _check_upload_unlinked(ign, format, uri_prefix):
2122 filename = format + ".txt"
2123 d = self.POST("/uri?t=upload&format=" + format,
2124 file=(filename, self.NEWFILE_CONTENTS * 300000))
2125 def _got_results(results):
2126 if format.upper() in ("SDMF", "MDMF"):
2127 # webapi.rst says this returns a filecap
2130 # for immutable, it returns an "upload results page", and
2131 # the filecap is buried inside
2132 line = [l for l in results.split("\n") if "URI: " in l][0]
2133 mo = re.search(r'<span>([^<]+)</span>', line)
2134 filecap = mo.group(1)
2135 self.failUnless(filecap.startswith(uri_prefix),
2136 (uri_prefix, filecap))
2137 return self.GET("/uri/%s?t=json" % filecap)
2138 d.addCallback(_got_results)
2139 def _got_json(json):
2140 data = simplejson.loads(json)
2142 self.failUnlessIn("format", data)
2143 self.failUnlessEqual(data["format"], format.upper())
2144 d.addCallback(_got_json)
2146 d = defer.succeed(None)
2147 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2148 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2149 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2150 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2153 def test_POST_upload_bad_format_unlinked(self):
2154 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2155 400, "Bad Request", "Unknown format: foo",
2157 "/uri?t=upload&format=foo",
2158 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2160 def test_POST_upload_format(self):
2161 def _check_upload(ign, format, uri_prefix, fn=None):
2162 filename = format + ".txt"
2163 d = self.POST(self.public_url +
2164 "/foo?t=upload&format=" + format,
2165 file=(filename, self.NEWFILE_CONTENTS * 300000))
2166 def _got_filecap(filecap):
2168 filenameu = unicode(filename)
2169 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2170 self.failUnless(filecap.startswith(uri_prefix))
2171 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2172 d.addCallback(_got_filecap)
2173 def _got_json(json):
2174 data = simplejson.loads(json)
2176 self.failUnlessIn("format", data)
2177 self.failUnlessEqual(data["format"], format.upper())
2178 d.addCallback(_got_json)
2181 d = defer.succeed(None)
2182 d.addCallback(_check_upload, "chk", "URI:CHK")
2183 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2184 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2185 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2188 def test_POST_upload_bad_format(self):
2189 return self.shouldHTTPError("POST_upload_bad_format",
2190 400, "Bad Request", "Unknown format: foo",
2191 self.POST, self.public_url + \
2192 "/foo?t=upload&format=foo",
2193 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2195 def test_POST_upload_mutable(self):
2196 # this creates a mutable file
2197 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2198 file=("new.txt", self.NEWFILE_CONTENTS))
2200 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2201 d.addCallback(lambda res:
2202 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2203 self.NEWFILE_CONTENTS))
2204 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2206 self.failUnless(IMutableFileNode.providedBy(newnode))
2207 self.failUnless(newnode.is_mutable())
2208 self.failIf(newnode.is_readonly())
2209 self._mutable_node = newnode
2210 self._mutable_uri = newnode.get_uri()
2213 # now upload it again and make sure that the URI doesn't change
2214 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2215 d.addCallback(lambda res:
2216 self.POST(self.public_url + "/foo", t="upload",
2218 file=("new.txt", NEWER_CONTENTS)))
2219 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2220 d.addCallback(lambda res:
2221 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2223 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2225 self.failUnless(IMutableFileNode.providedBy(newnode))
2226 self.failUnless(newnode.is_mutable())
2227 self.failIf(newnode.is_readonly())
2228 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2229 d.addCallback(_got2)
2231 # upload a second time, using PUT instead of POST
2232 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2233 d.addCallback(lambda res:
2234 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2235 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2236 d.addCallback(lambda res:
2237 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2240 # finally list the directory, since mutable files are displayed
2241 # slightly differently
2243 d.addCallback(lambda res:
2244 self.GET(self.public_url + "/foo/",
2245 followRedirect=True))
2246 def _check_page(res):
2247 # TODO: assert more about the contents
2248 self.failUnlessIn("SSK", res)
2250 d.addCallback(_check_page)
2252 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2254 self.failUnless(IMutableFileNode.providedBy(newnode))
2255 self.failUnless(newnode.is_mutable())
2256 self.failIf(newnode.is_readonly())
2257 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2258 d.addCallback(_got3)
2260 # look at the JSON form of the enclosing directory
2261 d.addCallback(lambda res:
2262 self.GET(self.public_url + "/foo/?t=json",
2263 followRedirect=True))
2264 def _check_page_json(res):
2265 parsed = simplejson.loads(res)
2266 self.failUnlessEqual(parsed[0], "dirnode")
2267 children = dict( [(unicode(name),value)
2269 in parsed[1]["children"].iteritems()] )
2270 self.failUnlessIn(u"new.txt", children)
2271 new_json = children[u"new.txt"]
2272 self.failUnlessEqual(new_json[0], "filenode")
2273 self.failUnless(new_json[1]["mutable"])
2274 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2275 ro_uri = self._mutable_node.get_readonly().to_string()
2276 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2277 d.addCallback(_check_page_json)
2279 # and the JSON form of the file
2280 d.addCallback(lambda res:
2281 self.GET(self.public_url + "/foo/new.txt?t=json"))
2282 def _check_file_json(res):
2283 parsed = simplejson.loads(res)
2284 self.failUnlessEqual(parsed[0], "filenode")
2285 self.failUnless(parsed[1]["mutable"])
2286 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2287 ro_uri = self._mutable_node.get_readonly().to_string()
2288 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2289 d.addCallback(_check_file_json)
2291 # and look at t=uri and t=readonly-uri
2292 d.addCallback(lambda res:
2293 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2294 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2295 d.addCallback(lambda res:
2296 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2297 def _check_ro_uri(res):
2298 ro_uri = self._mutable_node.get_readonly().to_string()
2299 self.failUnlessReallyEqual(res, ro_uri)
2300 d.addCallback(_check_ro_uri)
2302 # make sure we can get to it from /uri/URI
2303 d.addCallback(lambda res:
2304 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2305 d.addCallback(lambda res:
2306 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2308 # and that HEAD computes the size correctly
2309 d.addCallback(lambda res:
2310 self.HEAD(self.public_url + "/foo/new.txt",
2311 return_response=True))
2312 def _got_headers((res, status, headers)):
2313 self.failUnlessReallyEqual(res, "")
2314 self.failUnlessReallyEqual(headers["content-length"][0],
2315 str(len(NEW2_CONTENTS)))
2316 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2317 d.addCallback(_got_headers)
2319 # make sure that outdated size limits aren't enforced anymore.
2320 d.addCallback(lambda ignored:
2321 self.POST(self.public_url + "/foo", t="upload",
2324 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2325 d.addErrback(self.dump_error)
2328 def test_POST_upload_mutable_toobig(self):
2329 # SDMF had a size limti that was removed a while ago. MDMF has
2330 # never had a size limit. Test to make sure that we do not
2331 # encounter errors when trying to upload large mutable files,
2332 # since there should be no coded prohibitions regarding large
2334 d = self.POST(self.public_url + "/foo",
2335 t="upload", mutable="true",
2336 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2339 def dump_error(self, f):
2340 # if the web server returns an error code (like 400 Bad Request),
2341 # web.client.getPage puts the HTTP response body into the .response
2342 # attribute of the exception object that it gives back. It does not
2343 # appear in the Failure's repr(), so the ERROR that trial displays
2344 # will be rather terse and unhelpful. addErrback this method to the
2345 # end of your chain to get more information out of these errors.
2346 if f.check(error.Error):
2347 print "web.error.Error:"
2349 print f.value.response
2352 def test_POST_upload_replace(self):
2353 d = self.POST(self.public_url + "/foo", t="upload",
2354 file=("bar.txt", self.NEWFILE_CONTENTS))
2356 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2357 d.addCallback(lambda res:
2358 self.failUnlessChildContentsAre(fn, u"bar.txt",
2359 self.NEWFILE_CONTENTS))
2362 def test_POST_upload_no_replace_ok(self):
2363 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2364 file=("new.txt", self.NEWFILE_CONTENTS))
2365 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2366 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2367 self.NEWFILE_CONTENTS))
2370 def test_POST_upload_no_replace_queryarg(self):
2371 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2372 file=("bar.txt", self.NEWFILE_CONTENTS))
2373 d.addBoth(self.shouldFail, error.Error,
2374 "POST_upload_no_replace_queryarg",
2376 "There was already a child by that name, and you asked me "
2377 "to not replace it")
2378 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2379 d.addCallback(self.failUnlessIsBarDotTxt)
2382 def test_POST_upload_no_replace_field(self):
2383 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2384 file=("bar.txt", self.NEWFILE_CONTENTS))
2385 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2387 "There was already a child by that name, and you asked me "
2388 "to not replace it")
2389 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2390 d.addCallback(self.failUnlessIsBarDotTxt)
2393 def test_POST_upload_whendone(self):
2394 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2395 file=("new.txt", self.NEWFILE_CONTENTS))
2396 d.addBoth(self.shouldRedirect, "/THERE")
2398 d.addCallback(lambda res:
2399 self.failUnlessChildContentsAre(fn, u"new.txt",
2400 self.NEWFILE_CONTENTS))
2403 def test_POST_upload_named(self):
2405 d = self.POST(self.public_url + "/foo", t="upload",
2406 name="new.txt", file=self.NEWFILE_CONTENTS)
2407 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2408 d.addCallback(lambda res:
2409 self.failUnlessChildContentsAre(fn, u"new.txt",
2410 self.NEWFILE_CONTENTS))
2413 def test_POST_upload_named_badfilename(self):
2414 d = self.POST(self.public_url + "/foo", t="upload",
2415 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2416 d.addBoth(self.shouldFail, error.Error,
2417 "test_POST_upload_named_badfilename",
2419 "name= may not contain a slash",
2421 # make sure that nothing was added
2422 d.addCallback(lambda res:
2423 self.failUnlessNodeKeysAre(self._foo_node,
2424 [u"bar.txt", u"baz.txt", u"blockingfile",
2425 u"empty", u"n\u00fc.txt", u"quux.txt",
2429 def test_POST_FILEURL_check(self):
2430 bar_url = self.public_url + "/foo/bar.txt"
2431 d = self.POST(bar_url, t="check")
2433 self.failUnlessIn("Healthy :", res)
2434 d.addCallback(_check)
2435 redir_url = "http://allmydata.org/TARGET"
2436 def _check2(statuscode, target):
2437 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2438 self.failUnlessReallyEqual(target, redir_url)
2439 d.addCallback(lambda res:
2440 self.shouldRedirect2("test_POST_FILEURL_check",
2444 when_done=redir_url))
2445 d.addCallback(lambda res:
2446 self.POST(bar_url, t="check", return_to=redir_url))
2448 self.failUnlessIn("Healthy :", res)
2449 self.failUnlessIn("Return to file", res)
2450 self.failUnlessIn(redir_url, res)
2451 d.addCallback(_check3)
2453 d.addCallback(lambda res:
2454 self.POST(bar_url, t="check", output="JSON"))
2455 def _check_json(res):
2456 data = simplejson.loads(res)
2457 self.failUnlessIn("storage-index", data)
2458 self.failUnless(data["results"]["healthy"])
2459 d.addCallback(_check_json)
2463 def test_POST_FILEURL_check_and_repair(self):
2464 bar_url = self.public_url + "/foo/bar.txt"
2465 d = self.POST(bar_url, t="check", repair="true")
2467 self.failUnlessIn("Healthy :", res)
2468 d.addCallback(_check)
2469 redir_url = "http://allmydata.org/TARGET"
2470 def _check2(statuscode, target):
2471 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2472 self.failUnlessReallyEqual(target, redir_url)
2473 d.addCallback(lambda res:
2474 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2477 t="check", repair="true",
2478 when_done=redir_url))
2479 d.addCallback(lambda res:
2480 self.POST(bar_url, t="check", return_to=redir_url))
2482 self.failUnlessIn("Healthy :", res)
2483 self.failUnlessIn("Return to file", res)
2484 self.failUnlessIn(redir_url, res)
2485 d.addCallback(_check3)
2488 def test_POST_DIRURL_check(self):
2489 foo_url = self.public_url + "/foo/"
2490 d = self.POST(foo_url, t="check")
2492 self.failUnlessIn("Healthy :", res)
2493 d.addCallback(_check)
2494 redir_url = "http://allmydata.org/TARGET"
2495 def _check2(statuscode, target):
2496 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2497 self.failUnlessReallyEqual(target, redir_url)
2498 d.addCallback(lambda res:
2499 self.shouldRedirect2("test_POST_DIRURL_check",
2503 when_done=redir_url))
2504 d.addCallback(lambda res:
2505 self.POST(foo_url, t="check", return_to=redir_url))
2507 self.failUnlessIn("Healthy :", res)
2508 self.failUnlessIn("Return to file/directory", res)
2509 self.failUnlessIn(redir_url, res)
2510 d.addCallback(_check3)
2512 d.addCallback(lambda res:
2513 self.POST(foo_url, t="check", output="JSON"))
2514 def _check_json(res):
2515 data = simplejson.loads(res)
2516 self.failUnlessIn("storage-index", data)
2517 self.failUnless(data["results"]["healthy"])
2518 d.addCallback(_check_json)
2522 def test_POST_DIRURL_check_and_repair(self):
2523 foo_url = self.public_url + "/foo/"
2524 d = self.POST(foo_url, t="check", repair="true")
2526 self.failUnlessIn("Healthy :", res)
2527 d.addCallback(_check)
2528 redir_url = "http://allmydata.org/TARGET"
2529 def _check2(statuscode, target):
2530 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2531 self.failUnlessReallyEqual(target, redir_url)
2532 d.addCallback(lambda res:
2533 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2536 t="check", repair="true",
2537 when_done=redir_url))
2538 d.addCallback(lambda res:
2539 self.POST(foo_url, t="check", return_to=redir_url))
2541 self.failUnlessIn("Healthy :", res)
2542 self.failUnlessIn("Return to file/directory", res)
2543 self.failUnlessIn(redir_url, res)
2544 d.addCallback(_check3)
2547 def test_POST_FILEURL_mdmf_check(self):
2548 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2549 d = self.POST(quux_url, t="check")
2551 self.failUnlessIn("Healthy", res)
2552 d.addCallback(_check)
2553 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2554 d.addCallback(lambda ignored:
2555 self.POST(quux_extension_url, t="check"))
2556 d.addCallback(_check)
2559 def test_POST_FILEURL_mdmf_check_and_repair(self):
2560 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2561 d = self.POST(quux_url, t="check", repair="true")
2563 self.failUnlessIn("Healthy", res)
2564 d.addCallback(_check)
2565 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2566 d.addCallback(lambda ignored:
2567 self.POST(quux_extension_url, t="check", repair="true"))
2568 d.addCallback(_check)
2571 def wait_for_operation(self, ignored, ophandle):
2572 url = "/operations/" + ophandle
2573 url += "?t=status&output=JSON"
2576 data = simplejson.loads(res)
2577 if not data["finished"]:
2578 d = self.stall(delay=1.0)
2579 d.addCallback(self.wait_for_operation, ophandle)
2585 def get_operation_results(self, ignored, ophandle, output=None):
2586 url = "/operations/" + ophandle
2589 url += "&output=" + output
2592 if output and output.lower() == "json":
2593 return simplejson.loads(res)
2598 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2599 d = self.shouldFail2(error.Error,
2600 "test_POST_DIRURL_deepcheck_no_ophandle",
2602 "slow operation requires ophandle=",
2603 self.POST, self.public_url, t="start-deep-check")
2606 def test_POST_DIRURL_deepcheck(self):
2607 def _check_redirect(statuscode, target):
2608 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2609 self.failUnless(target.endswith("/operations/123"))
2610 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2611 self.POST, self.public_url,
2612 t="start-deep-check", ophandle="123")
2613 d.addCallback(self.wait_for_operation, "123")
2614 def _check_json(data):
2615 self.failUnlessReallyEqual(data["finished"], True)
2616 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2617 self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2618 d.addCallback(_check_json)
2619 d.addCallback(self.get_operation_results, "123", "html")
2620 def _check_html(res):
2621 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2622 self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2623 self.failUnlessIn(FAVICON_MARKUP, res)
2624 d.addCallback(_check_html)
2626 d.addCallback(lambda res:
2627 self.GET("/operations/123/"))
2628 d.addCallback(_check_html) # should be the same as without the slash
2630 d.addCallback(lambda res:
2631 self.shouldFail2(error.Error, "one", "404 Not Found",
2632 "No detailed results for SI bogus",
2633 self.GET, "/operations/123/bogus"))
2635 foo_si = self._foo_node.get_storage_index()
2636 foo_si_s = base32.b2a(foo_si)
2637 d.addCallback(lambda res:
2638 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2639 def _check_foo_json(res):
2640 data = simplejson.loads(res)
2641 self.failUnlessEqual(data["storage-index"], foo_si_s)
2642 self.failUnless(data["results"]["healthy"])
2643 d.addCallback(_check_foo_json)
2646 def test_POST_DIRURL_deepcheck_and_repair(self):
2647 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2648 ophandle="124", output="json", followRedirect=True)
2649 d.addCallback(self.wait_for_operation, "124")
2650 def _check_json(data):
2651 self.failUnlessReallyEqual(data["finished"], True)
2652 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2653 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2654 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2655 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2656 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2657 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2658 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2659 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2660 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2661 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2662 d.addCallback(_check_json)
2663 d.addCallback(self.get_operation_results, "124", "html")
2664 def _check_html(res):
2665 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2667 self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2668 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2669 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2671 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2672 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2673 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2675 self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2676 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2677 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2679 self.failUnlessIn(FAVICON_MARKUP, res)
2680 d.addCallback(_check_html)
2683 def test_POST_FILEURL_bad_t(self):
2684 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2685 "POST to file: bad t=bogus",
2686 self.POST, self.public_url + "/foo/bar.txt",
2690 def test_POST_mkdir(self): # return value?
2691 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2692 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2693 d.addCallback(self.failUnlessNodeKeysAre, [])
2696 def test_POST_mkdir_mdmf(self):
2697 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2698 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2699 d.addCallback(lambda node:
2700 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2703 def test_POST_mkdir_sdmf(self):
2704 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2705 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706 d.addCallback(lambda node:
2707 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2710 def test_POST_mkdir_bad_format(self):
2711 return self.shouldHTTPError("POST_mkdir_bad_format",
2712 400, "Bad Request", "Unknown format: foo",
2713 self.POST, self.public_url +
2714 "/foo?t=mkdir&name=newdir&format=foo")
2716 def test_POST_mkdir_initial_children(self):
2717 (newkids, caps) = self._create_initial_children()
2718 d = self.POST2(self.public_url +
2719 "/foo?t=mkdir-with-children&name=newdir",
2720 simplejson.dumps(newkids))
2721 d.addCallback(lambda res:
2722 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2723 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2724 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2725 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2726 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2729 def test_POST_mkdir_initial_children_mdmf(self):
2730 (newkids, caps) = self._create_initial_children()
2731 d = self.POST2(self.public_url +
2732 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2733 simplejson.dumps(newkids))
2734 d.addCallback(lambda res:
2735 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2736 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2737 d.addCallback(lambda node:
2738 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2739 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2745 def test_POST_mkdir_initial_children_sdmf(self):
2746 (newkids, caps) = self._create_initial_children()
2747 d = self.POST2(self.public_url +
2748 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2749 simplejson.dumps(newkids))
2750 d.addCallback(lambda res:
2751 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753 d.addCallback(lambda node:
2754 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2755 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2756 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2760 def test_POST_mkdir_initial_children_bad_format(self):
2761 (newkids, caps) = self._create_initial_children()
2762 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2763 400, "Bad Request", "Unknown format: foo",
2764 self.POST, self.public_url + \
2765 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2766 simplejson.dumps(newkids))
2768 def test_POST_mkdir_immutable(self):
2769 (newkids, caps) = self._create_immutable_children()
2770 d = self.POST2(self.public_url +
2771 "/foo?t=mkdir-immutable&name=newdir",
2772 simplejson.dumps(newkids))
2773 d.addCallback(lambda res:
2774 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2775 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2776 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2777 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2778 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2779 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2780 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2781 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2782 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2783 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2784 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2785 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2786 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2789 def test_POST_mkdir_immutable_bad(self):
2790 (newkids, caps) = self._create_initial_children()
2791 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2793 "needed to be immutable but was not",
2796 "/foo?t=mkdir-immutable&name=newdir",
2797 simplejson.dumps(newkids))
2800 def test_POST_mkdir_2(self):
2801 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2802 d.addCallback(lambda res:
2803 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2804 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2805 d.addCallback(self.failUnlessNodeKeysAre, [])
2808 def test_POST_mkdirs_2(self):
2809 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2810 d.addCallback(lambda res:
2811 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2812 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2813 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2814 d.addCallback(self.failUnlessNodeKeysAre, [])
2817 def test_POST_mkdir_no_parentdir_noredirect(self):
2818 d = self.POST("/uri?t=mkdir")
2819 def _after_mkdir(res):
2820 uri.DirectoryURI.init_from_string(res)
2821 d.addCallback(_after_mkdir)
2824 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2825 d = self.POST("/uri?t=mkdir&format=mdmf")
2826 def _after_mkdir(res):
2827 u = uri.from_string(res)
2828 # Check that this is an MDMF writecap
2829 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2830 d.addCallback(_after_mkdir)
2833 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2834 d = self.POST("/uri?t=mkdir&format=sdmf")
2835 def _after_mkdir(res):
2836 u = uri.from_string(res)
2837 self.failUnlessIsInstance(u, uri.DirectoryURI)
2838 d.addCallback(_after_mkdir)
2841 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2842 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2843 400, "Bad Request", "Unknown format: foo",
2844 self.POST, self.public_url +
2845 "/uri?t=mkdir&format=foo")
2847 def test_POST_mkdir_no_parentdir_noredirect2(self):
2848 # make sure form-based arguments (as on the welcome page) still work
2849 d = self.POST("/uri", t="mkdir")
2850 def _after_mkdir(res):
2851 uri.DirectoryURI.init_from_string(res)
2852 d.addCallback(_after_mkdir)
2853 d.addErrback(self.explain_web_error)
2856 def test_POST_mkdir_no_parentdir_redirect(self):
2857 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2858 d.addBoth(self.shouldRedirect, None, statuscode='303')
2859 def _check_target(target):
2860 target = urllib.unquote(target)
2861 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2862 d.addCallback(_check_target)
2865 def test_POST_mkdir_no_parentdir_redirect2(self):
2866 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2867 d.addBoth(self.shouldRedirect, None, statuscode='303')
2868 def _check_target(target):
2869 target = urllib.unquote(target)
2870 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2871 d.addCallback(_check_target)
2872 d.addErrback(self.explain_web_error)
2875 def _make_readonly(self, u):
2876 ro_uri = uri.from_string(u).get_readonly()
2879 return ro_uri.to_string()
2881 def _create_initial_children(self):
2882 contents, n, filecap1 = self.makefile(12)
2883 md1 = {"metakey1": "metavalue1"}
2884 filecap2 = make_mutable_file_uri()
2885 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2886 filecap3 = node3.get_readonly_uri()
2887 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2888 dircap = DirectoryNode(node4, None, None).get_uri()
2889 mdmfcap = make_mutable_file_uri(mdmf=True)
2890 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2891 emptydircap = "URI:DIR2-LIT:"
2892 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
2893 "ro_uri": self._make_readonly(filecap1),
2894 "metadata": md1, }],
2895 u"child-mutable": ["filenode", {"rw_uri": filecap2,
2896 "ro_uri": self._make_readonly(filecap2)}],
2897 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2898 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
2899 "ro_uri": unknown_rocap}],
2900 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
2901 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2902 u"dirchild": ["dirnode", {"rw_uri": dircap,
2903 "ro_uri": self._make_readonly(dircap)}],
2904 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2905 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2906 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2907 "ro_uri": self._make_readonly(mdmfcap)}],
2909 return newkids, {'filecap1': filecap1,
2910 'filecap2': filecap2,
2911 'filecap3': filecap3,
2912 'unknown_rwcap': unknown_rwcap,
2913 'unknown_rocap': unknown_rocap,
2914 'unknown_immcap': unknown_immcap,
2916 'litdircap': litdircap,
2917 'emptydircap': emptydircap,
2920 def _create_immutable_children(self):
2921 contents, n, filecap1 = self.makefile(12)
2922 md1 = {"metakey1": "metavalue1"}
2923 tnode = create_chk_filenode("immutable directory contents\n"*10)
2924 dnode = DirectoryNode(tnode, None, None)
2925 assert not dnode.is_mutable()
2926 immdircap = dnode.get_uri()
2927 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2928 emptydircap = "URI:DIR2-LIT:"
2929 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2930 "metadata": md1, }],
2931 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2932 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2933 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2934 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2936 return newkids, {'filecap1': filecap1,
2937 'unknown_immcap': unknown_immcap,
2938 'immdircap': immdircap,
2939 'litdircap': litdircap,
2940 'emptydircap': emptydircap}
2942 def test_POST_mkdir_no_parentdir_initial_children(self):
2943 (newkids, caps) = self._create_initial_children()
2944 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2945 def _after_mkdir(res):
2946 self.failUnless(res.startswith("URI:DIR"), res)
2947 n = self.s.create_node_from_uri(res)
2948 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2949 d2.addCallback(lambda ign:
2950 self.failUnlessROChildURIIs(n, u"child-imm",
2952 d2.addCallback(lambda ign:
2953 self.failUnlessRWChildURIIs(n, u"child-mutable",
2955 d2.addCallback(lambda ign:
2956 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2958 d2.addCallback(lambda ign:
2959 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2960 caps['unknown_rwcap']))
2961 d2.addCallback(lambda ign:
2962 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2963 caps['unknown_rocap']))
2964 d2.addCallback(lambda ign:
2965 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2966 caps['unknown_immcap']))
2967 d2.addCallback(lambda ign:
2968 self.failUnlessRWChildURIIs(n, u"dirchild",
2971 d.addCallback(_after_mkdir)
2974 def test_POST_mkdir_no_parentdir_unexpected_children(self):
2975 # the regular /uri?t=mkdir operation is specified to ignore its body.
2976 # Only t=mkdir-with-children pays attention to it.
2977 (newkids, caps) = self._create_initial_children()
2978 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2980 "t=mkdir does not accept children=, "
2981 "try t=mkdir-with-children instead",
2982 self.POST2, "/uri?t=mkdir", # without children
2983 simplejson.dumps(newkids))
2986 def test_POST_noparent_bad(self):
2987 d = self.shouldHTTPError("POST_noparent_bad",
2989 "/uri accepts only PUT, PUT?t=mkdir, "
2990 "POST?t=upload, and POST?t=mkdir",
2991 self.POST, "/uri?t=bogus")
2994 def test_POST_mkdir_no_parentdir_immutable(self):
2995 (newkids, caps) = self._create_immutable_children()
2996 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2997 def _after_mkdir(res):
2998 self.failUnless(res.startswith("URI:DIR"), res)
2999 n = self.s.create_node_from_uri(res)
3000 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3001 d2.addCallback(lambda ign:
3002 self.failUnlessROChildURIIs(n, u"child-imm",
3004 d2.addCallback(lambda ign:
3005 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3006 caps['unknown_immcap']))
3007 d2.addCallback(lambda ign:
3008 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3010 d2.addCallback(lambda ign:
3011 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3013 d2.addCallback(lambda ign:
3014 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3015 caps['emptydircap']))
3017 d.addCallback(_after_mkdir)
3020 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3021 (newkids, caps) = self._create_initial_children()
3022 d = self.shouldFail2(error.Error,
3023 "test_POST_mkdir_no_parentdir_immutable_bad",
3025 "needed to be immutable but was not",
3027 "/uri?t=mkdir-immutable",
3028 simplejson.dumps(newkids))
3031 def test_welcome_page_mkdir_button(self):
3032 # Fetch the welcome page.
3034 def _after_get_welcome_page(res):
3035 MKDIR_BUTTON_RE = re.compile(
3036 '<form action="([^"]*)" method="post".*?'
3037 '<input type="hidden" name="t" value="([^"]*)" />'
3038 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3039 '<input type="submit" value="Create a directory" />',
3041 mo = MKDIR_BUTTON_RE.search(res)
3042 formaction = mo.group(1)
3044 formaname = mo.group(3)
3045 formavalue = mo.group(4)
3046 return (formaction, formt, formaname, formavalue)
3047 d.addCallback(_after_get_welcome_page)
3048 def _after_parse_form(res):
3049 (formaction, formt, formaname, formavalue) = res
3050 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3051 d.addCallback(_after_parse_form)
3052 d.addBoth(self.shouldRedirect, None, statuscode='303')
3055 def test_POST_mkdir_replace(self): # return value?
3056 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3057 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3058 d.addCallback(self.failUnlessNodeKeysAre, [])
3061 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3062 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3063 d.addBoth(self.shouldFail, error.Error,
3064 "POST_mkdir_no_replace_queryarg",
3066 "There was already a child by that name, and you asked me "
3067 "to not replace it")
3068 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3069 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3072 def test_POST_mkdir_no_replace_field(self): # return value?
3073 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3075 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3077 "There was already a child by that name, and you asked me "
3078 "to not replace it")
3079 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3080 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3083 def test_POST_mkdir_whendone_field(self):
3084 d = self.POST(self.public_url + "/foo",
3085 t="mkdir", name="newdir", when_done="/THERE")
3086 d.addBoth(self.shouldRedirect, "/THERE")
3087 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3088 d.addCallback(self.failUnlessNodeKeysAre, [])
3091 def test_POST_mkdir_whendone_queryarg(self):
3092 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3093 t="mkdir", name="newdir")
3094 d.addBoth(self.shouldRedirect, "/THERE")
3095 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3096 d.addCallback(self.failUnlessNodeKeysAre, [])
3099 def test_POST_bad_t(self):
3100 d = self.shouldFail2(error.Error, "POST_bad_t",
3102 "POST to a directory with bad t=BOGUS",
3103 self.POST, self.public_url + "/foo", t="BOGUS")
3106 def test_POST_set_children(self, command_name="set_children"):
3107 contents9, n9, newuri9 = self.makefile(9)
3108 contents10, n10, newuri10 = self.makefile(10)
3109 contents11, n11, newuri11 = self.makefile(11)
3112 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3115 "ctime": 1002777696.7564139,
3116 "mtime": 1002777696.7564139
3119 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3122 "ctime": 1002777696.7564139,
3123 "mtime": 1002777696.7564139
3126 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3129 "ctime": 1002777696.7564139,
3130 "mtime": 1002777696.7564139
3133 }""" % (newuri9, newuri10, newuri11)
3135 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3137 d = client.getPage(url, method="POST", postdata=reqbody)
3139 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3140 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3141 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3143 d.addCallback(_then)
3144 d.addErrback(self.dump_error)
3147 def test_POST_set_children_with_hyphen(self):
3148 return self.test_POST_set_children(command_name="set-children")
3150 def test_POST_link_uri(self):
3151 contents, n, newuri = self.makefile(8)
3152 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3153 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3154 d.addCallback(lambda res:
3155 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3159 def test_POST_link_uri_replace(self):
3160 contents, n, newuri = self.makefile(8)
3161 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3162 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3163 d.addCallback(lambda res:
3164 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3168 def test_POST_link_uri_unknown_bad(self):
3169 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3170 d.addBoth(self.shouldFail, error.Error,
3171 "POST_link_uri_unknown_bad",
3173 "unknown cap in a write slot")
3176 def test_POST_link_uri_unknown_ro_good(self):
3177 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3178 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3181 def test_POST_link_uri_unknown_imm_good(self):
3182 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3183 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3186 def test_POST_link_uri_no_replace_queryarg(self):
3187 contents, n, newuri = self.makefile(8)
3188 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3189 name="bar.txt", uri=newuri)
3190 d.addBoth(self.shouldFail, error.Error,
3191 "POST_link_uri_no_replace_queryarg",
3193 "There was already a child by that name, and you asked me "
3194 "to not replace it")
3195 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3196 d.addCallback(self.failUnlessIsBarDotTxt)
3199 def test_POST_link_uri_no_replace_field(self):
3200 contents, n, newuri = self.makefile(8)
3201 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3202 name="bar.txt", uri=newuri)
3203 d.addBoth(self.shouldFail, error.Error,
3204 "POST_link_uri_no_replace_field",
3206 "There was already a child by that name, and you asked me "
3207 "to not replace it")
3208 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3209 d.addCallback(self.failUnlessIsBarDotTxt)
3212 def test_POST_delete(self, command_name='delete'):
3213 d = self._foo_node.list()
3214 def _check_before(children):
3215 self.failUnlessIn(u"bar.txt", children)
3216 d.addCallback(_check_before)
3217 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3218 d.addCallback(lambda res: self._foo_node.list())
3219 def _check_after(children):
3220 self.failIfIn(u"bar.txt", children)
3221 d.addCallback(_check_after)
3224 def test_POST_unlink(self):
3225 return self.test_POST_delete(command_name='unlink')
3227 def test_POST_rename_file(self):
3228 d = self.POST(self.public_url + "/foo", t="rename",
3229 from_name="bar.txt", to_name='wibble.txt')
3230 d.addCallback(lambda res:
3231 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3232 d.addCallback(lambda res:
3233 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3234 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3235 d.addCallback(self.failUnlessIsBarDotTxt)
3236 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3237 d.addCallback(self.failUnlessIsBarJSON)
3240 def test_POST_rename_file_redundant(self):
3241 d = self.POST(self.public_url + "/foo", t="rename",
3242 from_name="bar.txt", to_name='bar.txt')
3243 d.addCallback(lambda res:
3244 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3245 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3246 d.addCallback(self.failUnlessIsBarDotTxt)
3247 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3248 d.addCallback(self.failUnlessIsBarJSON)
3251 def test_POST_rename_file_replace(self):
3252 # rename a file and replace a directory with it
3253 d = self.POST(self.public_url + "/foo", t="rename",
3254 from_name="bar.txt", to_name='empty')
3255 d.addCallback(lambda res:
3256 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3257 d.addCallback(lambda res:
3258 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3259 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3260 d.addCallback(self.failUnlessIsBarDotTxt)
3261 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3262 d.addCallback(self.failUnlessIsBarJSON)
3265 def test_POST_rename_file_no_replace_queryarg(self):
3266 # rename a file and replace a directory with it
3267 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3268 from_name="bar.txt", to_name='empty')
3269 d.addBoth(self.shouldFail, error.Error,
3270 "POST_rename_file_no_replace_queryarg",
3272 "There was already a child by that name, and you asked me "
3273 "to not replace it")
3274 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3275 d.addCallback(self.failUnlessIsEmptyJSON)
3278 def test_POST_rename_file_no_replace_field(self):
3279 # rename a file and replace a directory with it
3280 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3281 from_name="bar.txt", to_name='empty')
3282 d.addBoth(self.shouldFail, error.Error,
3283 "POST_rename_file_no_replace_field",
3285 "There was already a child by that name, and you asked me "
3286 "to not replace it")
3287 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3288 d.addCallback(self.failUnlessIsEmptyJSON)
3291 def failUnlessIsEmptyJSON(self, res):
3292 data = simplejson.loads(res)
3293 self.failUnlessEqual(data[0], "dirnode", data)
3294 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3296 def test_POST_rename_file_slash_fail(self):
3297 d = self.POST(self.public_url + "/foo", t="rename",
3298 from_name="bar.txt", to_name='kirk/spock.txt')
3299 d.addBoth(self.shouldFail, error.Error,
3300 "test_POST_rename_file_slash_fail",
3302 "to_name= may not contain a slash",
3304 d.addCallback(lambda res:
3305 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3308 def test_POST_rename_dir(self):
3309 d = self.POST(self.public_url, t="rename",
3310 from_name="foo", to_name='plunk')
3311 d.addCallback(lambda res:
3312 self.failIfNodeHasChild(self.public_root, u"foo"))
3313 d.addCallback(lambda res:
3314 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3315 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3316 d.addCallback(self.failUnlessIsFooJSON)
3319 def test_POST_move_file(self):
3320 d = self.POST(self.public_url + "/foo", t="move",
3321 from_name="bar.txt", to_dir="sub")
3322 d.addCallback(lambda res:
3323 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3324 d.addCallback(lambda res:
3325 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3326 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3327 d.addCallback(self.failUnlessIsBarDotTxt)
3328 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3329 d.addCallback(self.failUnlessIsBarJSON)
3332 def test_POST_move_file_new_name(self):
3333 d = self.POST(self.public_url + "/foo", t="move",
3334 from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3335 d.addCallback(lambda res:
3336 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3337 d.addCallback(lambda res:
3338 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3339 d.addCallback(lambda res:
3340 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3341 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3342 d.addCallback(self.failUnlessIsBarDotTxt)
3343 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3344 d.addCallback(self.failUnlessIsBarJSON)
3347 def test_POST_move_file_replace(self):
3348 d = self.POST(self.public_url + "/foo", t="move",
3349 from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3350 d.addCallback(lambda res:
3351 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3352 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3353 d.addCallback(self.failUnlessIsBarDotTxt)
3354 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3355 d.addCallback(self.failUnlessIsBarJSON)
3358 def test_POST_move_file_no_replace(self):
3359 d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3361 "There was already a child by that name, and you asked me to not replace it",
3362 self.POST, self.public_url + "/foo", t="move",
3363 replace="false", from_name="bar.txt",
3364 to_name="baz.txt", to_dir="sub")
3365 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3366 d.addCallback(self.failUnlessIsBarDotTxt)
3367 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3368 d.addCallback(self.failUnlessIsBarJSON)
3369 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3370 d.addCallback(self.failUnlessIsSubBazDotTxt)
3373 def test_POST_move_file_slash_fail(self):
3374 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3376 "to_name= may not contain a slash",
3377 self.POST, self.public_url + "/foo", t="move",
3378 from_name="bar.txt",
3379 to_name="slash/fail.txt", to_dir="sub")
3380 d.addCallback(lambda res:
3381 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3382 d.addCallback(lambda res:
3383 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3384 d.addCallback(lambda ign:
3385 self.shouldFail2(error.Error,
3386 "test_POST_rename_file_slash_fail2",
3388 "from_name= may not contain a slash",
3389 self.POST, self.public_url + "/foo",
3391 from_name="nope/bar.txt",
3392 to_name="fail.txt", to_dir="sub"))
3395 def test_POST_move_file_no_target(self):
3396 d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3398 "move requires from_name and to_dir",
3399 self.POST, self.public_url + "/foo", t="move",
3400 from_name="bar.txt", to_name="baz.txt")
3401 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3402 d.addCallback(self.failUnlessIsBarDotTxt)
3403 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3404 d.addCallback(self.failUnlessIsBarJSON)
3405 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3406 d.addCallback(self.failUnlessIsBazDotTxt)
3409 def test_POST_move_file_bad_target_type(self):
3410 d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3411 "400 Bad Request", "invalid target_type parameter",
3413 self.public_url + "/foo", t="move",
3414 target_type="*D", from_name="bar.txt",
3418 def test_POST_move_file_multi_level(self):
3419 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3420 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3421 from_name="bar.txt", to_dir="sub/level2"))
3422 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3423 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3424 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3425 d.addCallback(self.failUnlessIsBarDotTxt)
3426 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3427 d.addCallback(self.failUnlessIsBarJSON)
3430 def test_POST_move_file_to_uri(self):
3431 d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3432 from_name="bar.txt", to_dir=self._sub_uri)
3433 d.addCallback(lambda res:
3434 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3435 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3436 d.addCallback(self.failUnlessIsBarDotTxt)
3437 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3438 d.addCallback(self.failUnlessIsBarJSON)
3441 def test_POST_move_file_to_nonexist_dir(self):
3442 d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3443 "404 Not Found", "No such child: nopechucktesta",
3444 self.POST, self.public_url + "/foo", t="move",
3445 from_name="bar.txt", to_dir="nopechucktesta")
3448 def test_POST_move_file_into_file(self):
3449 d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3450 "400 Bad Request", "to_dir is not a directory",
3451 self.POST, self.public_url + "/foo", t="move",
3452 from_name="bar.txt", to_dir="baz.txt")
3453 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3454 d.addCallback(self.failUnlessIsBazDotTxt)
3455 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3456 d.addCallback(self.failUnlessIsBarDotTxt)
3457 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3458 d.addCallback(self.failUnlessIsBarJSON)
3461 def test_POST_move_file_to_bad_uri(self):
3462 d = self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3463 "400 Bad Request", "to_dir is not a directory",
3464 self.POST, self.public_url + "/foo", t="move",
3465 from_name="bar.txt", target_type="uri",
3466 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3467 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3468 d.addCallback(self.failUnlessIsBarDotTxt)
3469 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3470 d.addCallback(self.failUnlessIsBarJSON)
3473 def test_POST_move_dir(self):
3474 d = self.POST(self.public_url + "/foo", t="move",
3475 from_name="bar.txt", to_dir="empty")
3476 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3477 t="move", from_name="empty", to_dir="sub"))
3478 d.addCallback(lambda res:
3479 self.failIfNodeHasChild(self._foo_node, u"empty"))
3480 d.addCallback(lambda res:
3481 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3482 d.addCallback(lambda res:
3483 self._sub_node.get_child_at_path(u"empty"))
3484 d.addCallback(lambda node:
3485 self.failUnlessNodeHasChild(node, u"bar.txt"))
3486 d.addCallback(lambda res:
3487 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3488 d.addCallback(self.failUnlessIsBarDotTxt)
3491 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3492 """ If target is not None then the redirection has to go to target. If
3493 statuscode is not None then the redirection has to be accomplished with
3494 that HTTP status code."""
3495 if not isinstance(res, failure.Failure):
3496 to_where = (target is None) and "somewhere" or ("to " + target)
3497 self.fail("%s: we were expecting to get redirected %s, not get an"
3498 " actual page: %s" % (which, to_where, res))
3499 res.trap(error.PageRedirect)
3500 if statuscode is not None:
3501 self.failUnlessReallyEqual(res.value.status, statuscode,
3502 "%s: not a redirect" % which)
3503 if target is not None:
3504 # the PageRedirect does not seem to capture the uri= query arg
3505 # properly, so we can't check for it.
3506 realtarget = self.webish_url + target
3507 self.failUnlessReallyEqual(res.value.location, realtarget,
3508 "%s: wrong target" % which)
3509 return res.value.location
3511 def test_GET_URI_form(self):
3512 base = "/uri?uri=%s" % self._bar_txt_uri
3513 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3514 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3516 d.addBoth(self.shouldRedirect, targetbase)
3517 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3518 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3519 d.addCallback(lambda res: self.GET(base+"&t=json"))
3520 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3521 d.addCallback(self.log, "about to get file by uri")
3522 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3523 d.addCallback(self.failUnlessIsBarDotTxt)
3524 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3525 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3526 followRedirect=True))
3527 d.addCallback(self.failUnlessIsFooJSON)
3528 d.addCallback(self.log, "got dir by uri")
3532 def test_GET_URI_form_bad(self):
3533 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3534 "400 Bad Request", "GET /uri requires uri=",
3538 def test_GET_rename_form(self):
3539 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3540 followRedirect=True)
3542 self.failUnlessIn('name="when_done" value="."', res)
3543 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3544 self.failUnlessIn(FAVICON_MARKUP, res)
3545 d.addCallback(_check)
3548 def log(self, res, msg):
3549 #print "MSG: %s RES: %s" % (msg, res)
3553 def test_GET_URI_URL(self):
3554 base = "/uri/%s" % self._bar_txt_uri
3556 d.addCallback(self.failUnlessIsBarDotTxt)
3557 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3558 d.addCallback(self.failUnlessIsBarDotTxt)
3559 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3560 d.addCallback(self.failUnlessIsBarDotTxt)
3563 def test_GET_URI_URL_dir(self):
3564 base = "/uri/%s?t=json" % self._foo_uri
3566 d.addCallback(self.failUnlessIsFooJSON)
3569 def test_GET_URI_URL_missing(self):
3570 base = "/uri/%s" % self._bad_file_uri
3571 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3572 http.GONE, None, "NotEnoughSharesError",
3574 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3575 # here? we must arrange for a download to fail after target.open()
3576 # has been called, and then inspect the response to see that it is
3577 # shorter than we expected.
3580 def test_PUT_DIRURL_uri(self):
3581 d = self.s.create_dirnode()
3583 new_uri = dn.get_uri()
3584 # replace /foo with a new (empty) directory
3585 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3586 d.addCallback(lambda res:
3587 self.failUnlessReallyEqual(res.strip(), new_uri))
3588 d.addCallback(lambda res:
3589 self.failUnlessRWChildURIIs(self.public_root,
3593 d.addCallback(_made_dir)
3596 def test_PUT_DIRURL_uri_noreplace(self):
3597 d = self.s.create_dirnode()
3599 new_uri = dn.get_uri()
3600 # replace /foo with a new (empty) directory, but ask that
3601 # replace=false, so it should fail
3602 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3603 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3605 self.public_url + "/foo?t=uri&replace=false",
3607 d.addCallback(lambda res:
3608 self.failUnlessRWChildURIIs(self.public_root,
3612 d.addCallback(_made_dir)
3615 def test_PUT_DIRURL_bad_t(self):
3616 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3617 "400 Bad Request", "PUT to a directory",
3618 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3619 d.addCallback(lambda res:
3620 self.failUnlessRWChildURIIs(self.public_root,
3625 def test_PUT_NEWFILEURL_uri(self):
3626 contents, n, new_uri = self.makefile(8)
3627 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3628 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3629 d.addCallback(lambda res:
3630 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3634 def test_PUT_NEWFILEURL_mdmf(self):
3635 new_contents = self.NEWFILE_CONTENTS * 300000
3636 d = self.PUT(self.public_url + \
3637 "/foo/mdmf.txt?format=mdmf",
3639 d.addCallback(lambda ignored:
3640 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3641 def _got_json(json):
3642 data = simplejson.loads(json)
3644 self.failUnlessIn("format", data)
3645 self.failUnlessEqual(data["format"], "MDMF")
3646 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3647 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3648 d.addCallback(_got_json)
3651 def test_PUT_NEWFILEURL_sdmf(self):
3652 new_contents = self.NEWFILE_CONTENTS * 300000
3653 d = self.PUT(self.public_url + \
3654 "/foo/sdmf.txt?format=sdmf",
3656 d.addCallback(lambda ignored:
3657 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3658 def _got_json(json):
3659 data = simplejson.loads(json)
3661 self.failUnlessIn("format", data)
3662 self.failUnlessEqual(data["format"], "SDMF")
3663 d.addCallback(_got_json)
3666 def test_PUT_NEWFILEURL_bad_format(self):
3667 new_contents = self.NEWFILE_CONTENTS * 300000
3668 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3669 400, "Bad Request", "Unknown format: foo",
3670 self.PUT, self.public_url + \
3671 "/foo/foo.txt?format=foo",
3674 def test_PUT_NEWFILEURL_uri_replace(self):
3675 contents, n, new_uri = self.makefile(8)
3676 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3677 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3678 d.addCallback(lambda res:
3679 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3683 def test_PUT_NEWFILEURL_uri_no_replace(self):
3684 contents, n, new_uri = self.makefile(8)
3685 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3686 d.addBoth(self.shouldFail, error.Error,
3687 "PUT_NEWFILEURL_uri_no_replace",
3689 "There was already a child by that name, and you asked me "
3690 "to not replace it")
3693 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3694 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3695 d.addBoth(self.shouldFail, error.Error,
3696 "POST_put_uri_unknown_bad",
3698 "unknown cap in a write slot")
3701 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3702 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3703 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3704 u"put-future-ro.txt")
3707 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3708 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3709 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3710 u"put-future-imm.txt")
3713 def test_PUT_NEWFILE_URI(self):
3714 file_contents = "New file contents here\n"
3715 d = self.PUT("/uri", file_contents)
3717 assert isinstance(uri, str), uri
3718 self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3719 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3721 return self.GET("/uri/%s" % uri)
3722 d.addCallback(_check)
3724 self.failUnlessReallyEqual(res, file_contents)
3725 d.addCallback(_check2)
3728 def test_PUT_NEWFILE_URI_not_mutable(self):
3729 file_contents = "New file contents here\n"
3730 d = self.PUT("/uri?mutable=false", file_contents)
3732 assert isinstance(uri, str), uri
3733 self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3734 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3736 return self.GET("/uri/%s" % uri)
3737 d.addCallback(_check)
3739 self.failUnlessReallyEqual(res, file_contents)
3740 d.addCallback(_check2)
3743 def test_PUT_NEWFILE_URI_only_PUT(self):
3744 d = self.PUT("/uri?t=bogus", "")
3745 d.addBoth(self.shouldFail, error.Error,
3746 "PUT_NEWFILE_URI_only_PUT",
3748 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3751 def test_PUT_NEWFILE_URI_mutable(self):
3752 file_contents = "New file contents here\n"
3753 d = self.PUT("/uri?mutable=true", file_contents)
3754 def _check1(filecap):
3755 filecap = filecap.strip()
3756 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3757 self.filecap = filecap
3758 u = uri.WriteableSSKFileURI.init_from_string(filecap)
3759 self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3760 n = self.s.create_node_from_uri(filecap)
3761 return n.download_best_version()
3762 d.addCallback(_check1)
3764 self.failUnlessReallyEqual(data, file_contents)
3765 return self.GET("/uri/%s" % urllib.quote(self.filecap))
3766 d.addCallback(_check2)
3768 self.failUnlessReallyEqual(res, file_contents)
3769 d.addCallback(_check3)
3772 def test_PUT_mkdir(self):
3773 d = self.PUT("/uri?t=mkdir", "")
3775 n = self.s.create_node_from_uri(uri.strip())
3776 d2 = self.failUnlessNodeKeysAre(n, [])
3777 d2.addCallback(lambda res:
3778 self.GET("/uri/%s?t=json" % uri))
3780 d.addCallback(_check)
3781 d.addCallback(self.failUnlessIsEmptyJSON)
3784 def test_PUT_mkdir_mdmf(self):
3785 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3787 u = uri.from_string(res)
3788 # Check that this is an MDMF writecap
3789 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3793 def test_PUT_mkdir_sdmf(self):
3794 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3796 u = uri.from_string(res)
3797 self.failUnlessIsInstance(u, uri.DirectoryURI)
3801 def test_PUT_mkdir_bad_format(self):
3802 return self.shouldHTTPError("PUT_mkdir_bad_format",
3803 400, "Bad Request", "Unknown format: foo",
3804 self.PUT, "/uri?t=mkdir&format=foo",
3807 def test_POST_check(self):
3808 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3810 # this returns a string form of the results, which are probably
3811 # None since we're using fake filenodes.
3812 # TODO: verify that the check actually happened, by changing
3813 # FakeCHKFileNode to count how many times .check() has been
3816 d.addCallback(_done)
3820 def test_PUT_update_at_offset(self):
3821 file_contents = "test file" * 100000 # about 900 KiB
3822 d = self.PUT("/uri?mutable=true", file_contents)
3824 self.filecap = filecap
3825 new_data = file_contents[:100]
3826 new = "replaced and so on"
3828 new_data += file_contents[len(new_data):]
3829 assert len(new_data) == len(file_contents)
3830 self.new_data = new_data
3831 d.addCallback(_then)
3832 d.addCallback(lambda ignored:
3833 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3834 "replaced and so on"))
3835 def _get_data(filecap):
3836 n = self.s.create_node_from_uri(filecap)
3837 return n.download_best_version()
3838 d.addCallback(_get_data)
3839 d.addCallback(lambda results:
3840 self.failUnlessEqual(results, self.new_data))
3841 # Now try appending things to the file
3842 d.addCallback(lambda ignored:
3843 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3845 d.addCallback(_get_data)
3846 d.addCallback(lambda results:
3847 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3848 # and try replacing the beginning of the file
3849 d.addCallback(lambda ignored:
3850 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3851 d.addCallback(_get_data)
3852 d.addCallback(lambda results:
3853 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3856 def test_PUT_update_at_invalid_offset(self):
3857 file_contents = "test file" * 100000 # about 900 KiB
3858 d = self.PUT("/uri?mutable=true", file_contents)
3860 self.filecap = filecap
3861 d.addCallback(_then)
3862 # Negative offsets should cause an error.
3863 d.addCallback(lambda ignored:
3864 self.shouldHTTPError("PUT_update_at_invalid_offset",
3868 "/uri/%s?offset=-1" % self.filecap,
3872 def test_PUT_update_at_offset_immutable(self):
3873 file_contents = "Test file" * 100000
3874 d = self.PUT("/uri", file_contents)
3876 self.filecap = filecap
3877 d.addCallback(_then)
3878 d.addCallback(lambda ignored:
3879 self.shouldHTTPError("PUT_update_at_offset_immutable",
3883 "/uri/%s?offset=50" % self.filecap,
3888 def test_bad_method(self):
3889 url = self.webish_url + self.public_url + "/foo/bar.txt"
3890 d = self.shouldHTTPError("bad_method",
3891 501, "Not Implemented",
3892 "I don't know how to treat a BOGUS request.",
3893 client.getPage, url, method="BOGUS")
3896 def test_short_url(self):
3897 url = self.webish_url + "/uri"
3898 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3899 "I don't know how to treat a DELETE request.",
3900 client.getPage, url, method="DELETE")
3903 def test_ophandle_bad(self):
3904 url = self.webish_url + "/operations/bogus?t=status"
3905 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3906 "unknown/expired handle 'bogus'",
3907 client.getPage, url)
3910 def test_ophandle_cancel(self):
3911 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3912 followRedirect=True)
3913 d.addCallback(lambda ignored:
3914 self.GET("/operations/128?t=status&output=JSON"))
3916 data = simplejson.loads(res)
3917 self.failUnless("finished" in data, res)
3918 monitor = self.ws.root.child_operations.handles["128"][0]
3919 d = self.POST("/operations/128?t=cancel&output=JSON")
3921 data = simplejson.loads(res)
3922 self.failUnless("finished" in data, res)
3923 # t=cancel causes the handle to be forgotten
3924 self.failUnless(monitor.is_cancelled())
3925 d.addCallback(_check2)
3927 d.addCallback(_check1)
3928 d.addCallback(lambda ignored:
3929 self.shouldHTTPError("ophandle_cancel",
3930 404, "404 Not Found",
3931 "unknown/expired handle '128'",
3933 "/operations/128?t=status&output=JSON"))
3936 def test_ophandle_retainfor(self):
3937 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3938 followRedirect=True)
3939 d.addCallback(lambda ignored:
3940 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3942 data = simplejson.loads(res)
3943 self.failUnless("finished" in data, res)
3944 d.addCallback(_check1)
3945 # the retain-for=0 will cause the handle to be expired very soon
3946 d.addCallback(lambda ign:
3947 self.clock.advance(2.0))
3948 d.addCallback(lambda ignored:
3949 self.shouldHTTPError("ophandle_retainfor",
3950 404, "404 Not Found",
3951 "unknown/expired handle '129'",
3953 "/operations/129?t=status&output=JSON"))
3956 def test_ophandle_release_after_complete(self):
3957 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3958 followRedirect=True)
3959 d.addCallback(self.wait_for_operation, "130")
3960 d.addCallback(lambda ignored:
3961 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3962 # the release-after-complete=true will cause the handle to be expired
3963 d.addCallback(lambda ignored:
3964 self.shouldHTTPError("ophandle_release_after_complete",
3965 404, "404 Not Found",
3966 "unknown/expired handle '130'",
3968 "/operations/130?t=status&output=JSON"))
3971 def test_uncollected_ophandle_expiration(self):
3972 # uncollected ophandles should expire after 4 days
3973 def _make_uncollected_ophandle(ophandle):
3974 d = self.POST(self.public_url +
3975 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3976 followRedirect=False)
3977 # When we start the operation, the webapi server will want
3978 # to redirect us to the page for the ophandle, so we get
3979 # confirmation that the operation has started. If the
3980 # manifest operation has finished by the time we get there,
3981 # following that redirect (by setting followRedirect=True
3982 # above) has the side effect of collecting the ophandle that
3983 # we've just created, which means that we can't use the
3984 # ophandle to test the uncollected timeout anymore. So,
3985 # instead, catch the 302 here and don't follow it.
3986 d.addBoth(self.should302, "uncollected_ophandle_creation")
3988 # Create an ophandle, don't collect it, then advance the clock by
3989 # 4 days - 1 second and make sure that the ophandle is still there.
3990 d = _make_uncollected_ophandle(131)
3991 d.addCallback(lambda ign:
3992 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3993 d.addCallback(lambda ign:
3994 self.GET("/operations/131?t=status&output=JSON"))
3996 data = simplejson.loads(res)
3997 self.failUnless("finished" in data, res)
3998 d.addCallback(_check1)
3999 # Create an ophandle, don't collect it, then try to collect it
4000 # after 4 days. It should be gone.
4001 d.addCallback(lambda ign:
4002 _make_uncollected_ophandle(132))
4003 d.addCallback(lambda ign:
4004 self.clock.advance(96*60*60))
4005 d.addCallback(lambda ign:
4006 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4007 404, "404 Not Found",
4008 "unknown/expired handle '132'",
4010 "/operations/132?t=status&output=JSON"))
4013 def test_collected_ophandle_expiration(self):
4014 # collected ophandles should expire after 1 day
4015 def _make_collected_ophandle(ophandle):
4016 d = self.POST(self.public_url +
4017 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4018 followRedirect=True)
4019 # By following the initial redirect, we collect the ophandle
4020 # we've just created.
4022 # Create a collected ophandle, then collect it after 23 hours
4023 # and 59 seconds to make sure that it is still there.
4024 d = _make_collected_ophandle(133)
4025 d.addCallback(lambda ign:
4026 self.clock.advance((24*60*60) - 1))
4027 d.addCallback(lambda ign:
4028 self.GET("/operations/133?t=status&output=JSON"))
4030 data = simplejson.loads(res)
4031 self.failUnless("finished" in data, res)
4032 d.addCallback(_check1)
4033 # Create another uncollected ophandle, then try to collect it
4034 # after 24 hours to make sure that it is gone.
4035 d.addCallback(lambda ign:
4036 _make_collected_ophandle(134))
4037 d.addCallback(lambda ign:
4038 self.clock.advance(24*60*60))
4039 d.addCallback(lambda ign:
4040 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4041 404, "404 Not Found",
4042 "unknown/expired handle '134'",
4044 "/operations/134?t=status&output=JSON"))
4047 def test_incident(self):
4048 d = self.POST("/report_incident", details="eek")
4050 self.failIfIn("<html>", res)
4051 self.failUnlessIn("Thank you for your report!", res)
4052 d.addCallback(_done)
4055 def test_static(self):
4056 webdir = os.path.join(self.staticdir, "subdir")
4057 fileutil.make_dirs(webdir)
4058 f = open(os.path.join(webdir, "hello.txt"), "wb")
4062 d = self.GET("/static/subdir/hello.txt")
4064 self.failUnlessReallyEqual(res, "hello")
4065 d.addCallback(_check)
4069 class IntroducerWeb(unittest.TestCase):
4074 d = defer.succeed(None)
4076 d.addCallback(lambda ign: self.node.stopService())
4077 d.addCallback(flushEventualQueue)
4080 def test_welcome(self):
4081 basedir = "web.IntroducerWeb.test_welcome"
4083 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4084 self.node = IntroducerNode(basedir)
4085 self.ws = self.node.getServiceNamed("webish")
4087 d = fireEventually(None)
4088 d.addCallback(lambda ign: self.node.startService())
4089 d.addCallback(lambda ign: self.node.when_tub_ready())
4091 d.addCallback(lambda ign: self.GET("/"))
4093 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4094 self.failUnlessIn(FAVICON_MARKUP, res)
4095 d.addCallback(_check)
4098 def GET(self, urlpath, followRedirect=False, return_response=False,
4100 # if return_response=True, this fires with (data, statuscode,
4101 # respheaders) instead of just data.
4102 assert not isinstance(urlpath, unicode)
4103 url = self.ws.getURL().rstrip('/') + urlpath
4104 factory = HTTPClientGETFactory(url, method="GET",
4105 followRedirect=followRedirect, **kwargs)
4106 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4107 d = factory.deferred
4108 def _got_data(data):
4109 return (data, factory.status, factory.response_headers)
4111 d.addCallback(_got_data)
4112 return factory.deferred
4115 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4116 def test_load_file(self):
4117 # This will raise an exception unless a well-formed XML file is found under that name.
4118 common.getxmlfile('directory.xhtml').load()
4120 def test_parse_replace_arg(self):
4121 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4122 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4123 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4125 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4126 common.parse_replace_arg, "only_fles")
4128 def test_abbreviate_time(self):
4129 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4130 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4131 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4132 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4133 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4134 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4136 def test_compute_rate(self):
4137 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4138 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4139 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4140 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4141 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4142 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4143 self.shouldFail(AssertionError, "test_compute_rate", "",
4144 common.compute_rate, -100, 10)
4145 self.shouldFail(AssertionError, "test_compute_rate", "",
4146 common.compute_rate, 100, -10)
4149 rate = common.compute_rate(10*1000*1000, 1)
4150 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4152 def test_abbreviate_rate(self):
4153 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4154 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4155 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4156 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4158 def test_abbreviate_size(self):
4159 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4160 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4161 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4162 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4163 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4165 def test_plural(self):
4167 return "%d second%s" % (s, status.plural(s))
4168 self.failUnlessReallyEqual(convert(0), "0 seconds")
4169 self.failUnlessReallyEqual(convert(1), "1 second")
4170 self.failUnlessReallyEqual(convert(2), "2 seconds")
4172 return "has share%s: %s" % (status.plural(s), ",".join(s))
4173 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4174 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4175 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4178 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4180 def CHECK(self, ign, which, args, clientnum=0):
4181 fileurl = self.fileurls[which]
4182 url = fileurl + "?" + args
4183 return self.GET(url, method="POST", clientnum=clientnum)
4185 def test_filecheck(self):
4186 self.basedir = "web/Grid/filecheck"
4188 c0 = self.g.clients[0]
4191 d = c0.upload(upload.Data(DATA, convergence=""))
4192 def _stash_uri(ur, which):
4193 self.uris[which] = ur.uri
4194 d.addCallback(_stash_uri, "good")
4195 d.addCallback(lambda ign:
4196 c0.upload(upload.Data(DATA+"1", convergence="")))
4197 d.addCallback(_stash_uri, "sick")
4198 d.addCallback(lambda ign:
4199 c0.upload(upload.Data(DATA+"2", convergence="")))
4200 d.addCallback(_stash_uri, "dead")
4201 def _stash_mutable_uri(n, which):
4202 self.uris[which] = n.get_uri()
4203 assert isinstance(self.uris[which], str)
4204 d.addCallback(lambda ign:
4205 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4206 d.addCallback(_stash_mutable_uri, "corrupt")
4207 d.addCallback(lambda ign:
4208 c0.upload(upload.Data("literal", convergence="")))
4209 d.addCallback(_stash_uri, "small")
4210 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4211 d.addCallback(_stash_mutable_uri, "smalldir")
4213 def _compute_fileurls(ignored):
4215 for which in self.uris:
4216 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4217 d.addCallback(_compute_fileurls)
4219 def _clobber_shares(ignored):
4220 good_shares = self.find_uri_shares(self.uris["good"])
4221 self.failUnlessReallyEqual(len(good_shares), 10)
4222 sick_shares = self.find_uri_shares(self.uris["sick"])
4223 os.unlink(sick_shares[0][2])
4224 dead_shares = self.find_uri_shares(self.uris["dead"])
4225 for i in range(1, 10):
4226 os.unlink(dead_shares[i][2])
4227 c_shares = self.find_uri_shares(self.uris["corrupt"])
4228 cso = CorruptShareOptions()
4229 cso.stdout = StringIO()
4230 cso.parseOptions([c_shares[0][2]])
4232 d.addCallback(_clobber_shares)
4234 d.addCallback(self.CHECK, "good", "t=check")
4235 def _got_html_good(res):
4236 self.failUnlessIn("Healthy", res)
4237 self.failIfIn("Not Healthy", res)
4238 self.failUnlessIn(FAVICON_MARKUP, res)
4239 d.addCallback(_got_html_good)
4240 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4241 def _got_html_good_return_to(res):
4242 self.failUnlessIn("Healthy", res)
4243 self.failIfIn("Not Healthy", res)
4244 self.failUnlessIn('<a href="somewhere">Return to file', res)
4245 d.addCallback(_got_html_good_return_to)
4246 d.addCallback(self.CHECK, "good", "t=check&output=json")
4247 def _got_json_good(res):
4248 r = simplejson.loads(res)
4249 self.failUnlessEqual(r["summary"], "Healthy")
4250 self.failUnless(r["results"]["healthy"])
4251 self.failIf(r["results"]["needs-rebalancing"])
4252 self.failUnless(r["results"]["recoverable"])
4253 d.addCallback(_got_json_good)
4255 d.addCallback(self.CHECK, "small", "t=check")
4256 def _got_html_small(res):
4257 self.failUnlessIn("Literal files are always healthy", res)
4258 self.failIfIn("Not Healthy", res)
4259 d.addCallback(_got_html_small)
4260 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4261 def _got_html_small_return_to(res):
4262 self.failUnlessIn("Literal files are always healthy", res)
4263 self.failIfIn("Not Healthy", res)
4264 self.failUnlessIn('<a href="somewhere">Return to file', res)
4265 d.addCallback(_got_html_small_return_to)
4266 d.addCallback(self.CHECK, "small", "t=check&output=json")
4267 def _got_json_small(res):
4268 r = simplejson.loads(res)
4269 self.failUnlessEqual(r["storage-index"], "")
4270 self.failUnless(r["results"]["healthy"])
4271 d.addCallback(_got_json_small)
4273 d.addCallback(self.CHECK, "smalldir", "t=check")
4274 def _got_html_smalldir(res):
4275 self.failUnlessIn("Literal files are always healthy", res)
4276 self.failIfIn("Not Healthy", res)
4277 d.addCallback(_got_html_smalldir)
4278 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4279 def _got_json_smalldir(res):
4280 r = simplejson.loads(res)
4281 self.failUnlessEqual(r["storage-index"], "")
4282 self.failUnless(r["results"]["healthy"])
4283 d.addCallback(_got_json_smalldir)
4285 d.addCallback(self.CHECK, "sick", "t=check")
4286 def _got_html_sick(res):
4287 self.failUnlessIn("Not Healthy", res)
4288 d.addCallback(_got_html_sick)
4289 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4290 def _got_json_sick(res):
4291 r = simplejson.loads(res)
4292 self.failUnlessEqual(r["summary"],
4293 "Not Healthy: 9 shares (enc 3-of-10)")
4294 self.failIf(r["results"]["healthy"])
4295 self.failIf(r["results"]["needs-rebalancing"])
4296 self.failUnless(r["results"]["recoverable"])
4297 d.addCallback(_got_json_sick)
4299 d.addCallback(self.CHECK, "dead", "t=check")
4300 def _got_html_dead(res):
4301 self.failUnlessIn("Not Healthy", res)
4302 d.addCallback(_got_html_dead)
4303 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4304 def _got_json_dead(res):
4305 r = simplejson.loads(res)
4306 self.failUnlessEqual(r["summary"],
4307 "Not Healthy: 1 shares (enc 3-of-10)")
4308 self.failIf(r["results"]["healthy"])
4309 self.failIf(r["results"]["needs-rebalancing"])
4310 self.failIf(r["results"]["recoverable"])
4311 d.addCallback(_got_json_dead)
4313 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4314 def _got_html_corrupt(res):
4315 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4316 d.addCallback(_got_html_corrupt)
4317 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4318 def _got_json_corrupt(res):
4319 r = simplejson.loads(res)
4320 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4321 self.failIf(r["results"]["healthy"])
4322 self.failUnless(r["results"]["recoverable"])
4323 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4324 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4325 d.addCallback(_got_json_corrupt)
4327 d.addErrback(self.explain_web_error)
4330 def test_repair_html(self):
4331 self.basedir = "web/Grid/repair_html"
4333 c0 = self.g.clients[0]
4336 d = c0.upload(upload.Data(DATA, convergence=""))
4337 def _stash_uri(ur, which):
4338 self.uris[which] = ur.uri
4339 d.addCallback(_stash_uri, "good")
4340 d.addCallback(lambda ign:
4341 c0.upload(upload.Data(DATA+"1", convergence="")))
4342 d.addCallback(_stash_uri, "sick")
4343 d.addCallback(lambda ign:
4344 c0.upload(upload.Data(DATA+"2", convergence="")))
4345 d.addCallback(_stash_uri, "dead")
4346 def _stash_mutable_uri(n, which):
4347 self.uris[which] = n.get_uri()
4348 assert isinstance(self.uris[which], str)
4349 d.addCallback(lambda ign:
4350 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4351 d.addCallback(_stash_mutable_uri, "corrupt")
4353 def _compute_fileurls(ignored):
4355 for which in self.uris:
4356 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4357 d.addCallback(_compute_fileurls)
4359 def _clobber_shares(ignored):
4360 good_shares = self.find_uri_shares(self.uris["good"])
4361 self.failUnlessReallyEqual(len(good_shares), 10)
4362 sick_shares = self.find_uri_shares(self.uris["sick"])
4363 os.unlink(sick_shares[0][2])
4364 dead_shares = self.find_uri_shares(self.uris["dead"])
4365 for i in range(1, 10):
4366 os.unlink(dead_shares[i][2])
4367 c_shares = self.find_uri_shares(self.uris["corrupt"])
4368 cso = CorruptShareOptions()
4369 cso.stdout = StringIO()
4370 cso.parseOptions([c_shares[0][2]])
4372 d.addCallback(_clobber_shares)
4374 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4375 def _got_html_good(res):
4376 self.failUnlessIn("Healthy", res)
4377 self.failIfIn("Not Healthy", res)
4378 self.failUnlessIn("No repair necessary", res)
4379 self.failUnlessIn(FAVICON_MARKUP, res)
4380 d.addCallback(_got_html_good)
4382 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4383 def _got_html_sick(res):
4384 self.failUnlessIn("Healthy : healthy", res)
4385 self.failIfIn("Not Healthy", res)
4386 self.failUnlessIn("Repair successful", res)
4387 d.addCallback(_got_html_sick)
4389 # repair of a dead file will fail, of course, but it isn't yet
4390 # clear how this should be reported. Right now it shows up as
4393 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4394 #def _got_html_dead(res):
4396 # self.failUnlessIn("Healthy : healthy", res)
4397 # self.failIfIn("Not Healthy", res)
4398 # self.failUnlessIn("No repair necessary", res)
4399 #d.addCallback(_got_html_dead)
4401 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4402 def _got_html_corrupt(res):
4403 self.failUnlessIn("Healthy : Healthy", res)
4404 self.failIfIn("Not Healthy", res)
4405 self.failUnlessIn("Repair successful", res)
4406 d.addCallback(_got_html_corrupt)
4408 d.addErrback(self.explain_web_error)
4411 def test_repair_json(self):
4412 self.basedir = "web/Grid/repair_json"
4414 c0 = self.g.clients[0]
4417 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4418 def _stash_uri(ur, which):
4419 self.uris[which] = ur.uri
4420 d.addCallback(_stash_uri, "sick")
4422 def _compute_fileurls(ignored):
4424 for which in self.uris:
4425 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4426 d.addCallback(_compute_fileurls)
4428 def _clobber_shares(ignored):
4429 sick_shares = self.find_uri_shares(self.uris["sick"])
4430 os.unlink(sick_shares[0][2])
4431 d.addCallback(_clobber_shares)
4433 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4434 def _got_json_sick(res):
4435 r = simplejson.loads(res)
4436 self.failUnlessReallyEqual(r["repair-attempted"], True)
4437 self.failUnlessReallyEqual(r["repair-successful"], True)
4438 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4439 "Not Healthy: 9 shares (enc 3-of-10)")
4440 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4441 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4442 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4443 d.addCallback(_got_json_sick)
4445 d.addErrback(self.explain_web_error)
4448 def test_unknown(self, immutable=False):
4449 self.basedir = "web/Grid/unknown"
4451 self.basedir = "web/Grid/unknown-immutable"
4454 c0 = self.g.clients[0]
4458 # the future cap format may contain slashes, which must be tolerated
4459 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4463 name = u"future-imm"
4464 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4465 d = c0.create_immutable_dirnode({name: (future_node, {})})
4468 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4469 d = c0.create_dirnode()
4471 def _stash_root_and_create_file(n):
4473 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4474 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4476 return self.rootnode.set_node(name, future_node)
4477 d.addCallback(_stash_root_and_create_file)
4479 # make sure directory listing tolerates unknown nodes
4480 d.addCallback(lambda ign: self.GET(self.rooturl))
4481 def _check_directory_html(res, expected_type_suffix):
4482 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4483 '<td>%s</td>' % (expected_type_suffix, str(name)),
4485 self.failUnless(re.search(pattern, res), res)
4486 # find the More Info link for name, should be relative
4487 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4488 info_url = mo.group(1)
4489 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4491 d.addCallback(_check_directory_html, "-IMM")
4493 d.addCallback(_check_directory_html, "")
4495 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4496 def _check_directory_json(res, expect_rw_uri):
4497 data = simplejson.loads(res)
4498 self.failUnlessEqual(data[0], "dirnode")
4499 f = data[1]["children"][name]
4500 self.failUnlessEqual(f[0], "unknown")
4502 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4504 self.failIfIn("rw_uri", f[1])
4506 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4508 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4509 self.failUnlessIn("metadata", f[1])
4510 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4512 def _check_info(res, expect_rw_uri, expect_ro_uri):
4513 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4515 self.failUnlessIn(unknown_rwcap, res)
4518 self.failUnlessIn(unknown_immcap, res)
4520 self.failUnlessIn(unknown_rocap, res)
4522 self.failIfIn(unknown_rocap, res)
4523 self.failIfIn("Raw data as", res)
4524 self.failIfIn("Directory writecap", res)
4525 self.failIfIn("Checker Operations", res)
4526 self.failIfIn("Mutable File Operations", res)
4527 self.failIfIn("Directory Operations", res)
4529 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4530 # why they fail. Possibly related to ticket #922.
4532 d.addCallback(lambda ign: self.GET(expected_info_url))
4533 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4534 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4535 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4537 def _check_json(res, expect_rw_uri):
4538 data = simplejson.loads(res)
4539 self.failUnlessEqual(data[0], "unknown")
4541 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4543 self.failIfIn("rw_uri", data[1])
4546 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4547 self.failUnlessReallyEqual(data[1]["mutable"], False)
4549 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4550 self.failUnlessReallyEqual(data[1]["mutable"], True)
4552 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4553 self.failIfIn("mutable", data[1])
4555 # TODO: check metadata contents
4556 self.failUnlessIn("metadata", data[1])
4558 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4559 d.addCallback(_check_json, expect_rw_uri=not immutable)
4561 # and make sure that a read-only version of the directory can be
4562 # rendered too. This version will not have unknown_rwcap, whether
4563 # or not future_node was immutable.
4564 d.addCallback(lambda ign: self.GET(self.rourl))
4566 d.addCallback(_check_directory_html, "-IMM")
4568 d.addCallback(_check_directory_html, "-RO")
4570 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4571 d.addCallback(_check_directory_json, expect_rw_uri=False)
4573 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4574 d.addCallback(_check_json, expect_rw_uri=False)
4576 # TODO: check that getting t=info from the Info link in the ro directory
4577 # works, and does not include the writecap URI.
4580 def test_immutable_unknown(self):
4581 return self.test_unknown(immutable=True)
4583 def test_mutant_dirnodes_are_omitted(self):
4584 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4587 c = self.g.clients[0]
4592 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4593 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4594 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4596 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4597 # test the dirnode and web layers separately.
4599 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4600 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4601 # When the directory is read, the mutants should be silently disposed of, leaving
4602 # their lonely sibling.
4603 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4604 # because immutable directories don't have a writecap and therefore that field
4605 # isn't (and can't be) decrypted.
4606 # TODO: The field still exists in the netstring. Technically we should check what
4607 # happens if something is put there (_unpack_contents should raise ValueError),
4608 # but that can wait.
4610 lonely_child = nm.create_from_cap(lonely_uri)
4611 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4612 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4614 def _by_hook_or_by_crook():
4616 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4617 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4619 mutant_write_in_ro_child.get_write_uri = lambda: None
4620 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4622 kids = {u"lonely": (lonely_child, {}),
4623 u"ro": (mutant_ro_child, {}),
4624 u"write-in-ro": (mutant_write_in_ro_child, {}),
4626 d = c.create_immutable_dirnode(kids)
4629 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4630 self.failIf(dn.is_mutable())
4631 self.failUnless(dn.is_readonly())
4632 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4633 self.failIf(hasattr(dn._node, 'get_writekey'))
4635 self.failUnlessIn("RO-IMM", rep)
4637 self.failUnlessIn("CHK", cap.to_string())
4640 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4641 return download_to_data(dn._node)
4642 d.addCallback(_created)
4644 def _check_data(data):
4645 # Decode the netstring representation of the directory to check that all children
4646 # are present. This is a bit of an abstraction violation, but there's not really
4647 # any other way to do it given that the real DirectoryNode._unpack_contents would
4648 # strip the mutant children out (which is what we're trying to test, later).
4651 while position < len(data):
4652 entries, position = split_netstring(data, 1, position)
4654 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4655 name = name_utf8.decode("utf-8")
4656 self.failUnlessEqual(rwcapdata, "")
4657 self.failUnlessIn(name, kids)
4658 (expected_child, ign) = kids[name]
4659 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4662 self.failUnlessReallyEqual(numkids, 3)
4663 return self.rootnode.list()
4664 d.addCallback(_check_data)
4666 # Now when we use the real directory listing code, the mutants should be absent.
4667 def _check_kids(children):
4668 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4669 lonely_node, lonely_metadata = children[u"lonely"]
4671 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4672 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4673 d.addCallback(_check_kids)
4675 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4676 d.addCallback(lambda n: n.list())
4677 d.addCallback(_check_kids) # again with dirnode recreated from cap
4679 # Make sure the lonely child can be listed in HTML...
4680 d.addCallback(lambda ign: self.GET(self.rooturl))
4681 def _check_html(res):
4682 self.failIfIn("URI:SSK", res)
4683 get_lonely = "".join([r'<td>FILE</td>',
4685 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4687 r'\s+<td align="right">%d</td>' % len("one"),
4689 self.failUnless(re.search(get_lonely, res), res)
4691 # find the More Info link for name, should be relative
4692 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4693 info_url = mo.group(1)
4694 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4695 d.addCallback(_check_html)
4698 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4699 def _check_json(res):
4700 data = simplejson.loads(res)
4701 self.failUnlessEqual(data[0], "dirnode")
4702 listed_children = data[1]["children"]
4703 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4704 ll_type, ll_data = listed_children[u"lonely"]
4705 self.failUnlessEqual(ll_type, "filenode")
4706 self.failIfIn("rw_uri", ll_data)
4707 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4708 d.addCallback(_check_json)
4711 def test_deep_check(self):
4712 self.basedir = "web/Grid/deep_check"
4714 c0 = self.g.clients[0]
4718 d = c0.create_dirnode()
4719 def _stash_root_and_create_file(n):
4721 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4722 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4723 d.addCallback(_stash_root_and_create_file)
4724 def _stash_uri(fn, which):
4725 self.uris[which] = fn.get_uri()
4727 d.addCallback(_stash_uri, "good")
4728 d.addCallback(lambda ign:
4729 self.rootnode.add_file(u"small",
4730 upload.Data("literal",
4732 d.addCallback(_stash_uri, "small")
4733 d.addCallback(lambda ign:
4734 self.rootnode.add_file(u"sick",
4735 upload.Data(DATA+"1",
4737 d.addCallback(_stash_uri, "sick")
4739 # this tests that deep-check and stream-manifest will ignore
4740 # UnknownNode instances. Hopefully this will also cover deep-stats.
4741 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4742 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4744 def _clobber_shares(ignored):
4745 self.delete_shares_numbered(self.uris["sick"], [0,1])
4746 d.addCallback(_clobber_shares)
4754 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4757 units = [simplejson.loads(line)
4758 for line in res.splitlines()
4761 print "response is:", res
4762 print "undecodeable line was '%s'" % line
4764 self.failUnlessReallyEqual(len(units), 5+1)
4765 # should be parent-first
4767 self.failUnlessEqual(u0["path"], [])
4768 self.failUnlessEqual(u0["type"], "directory")
4769 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4770 u0cr = u0["check-results"]
4771 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4773 ugood = [u for u in units
4774 if u["type"] == "file" and u["path"] == [u"good"]][0]
4775 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4776 ugoodcr = ugood["check-results"]
4777 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4780 self.failUnlessEqual(stats["type"], "stats")
4782 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4783 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4784 self.failUnlessReallyEqual(s["count-directories"], 1)
4785 self.failUnlessReallyEqual(s["count-unknown"], 1)
4786 d.addCallback(_done)
4788 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4789 def _check_manifest(res):
4790 self.failUnless(res.endswith("\n"))
4791 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4792 self.failUnlessReallyEqual(len(units), 5+1)
4793 self.failUnlessEqual(units[-1]["type"], "stats")
4795 self.failUnlessEqual(first["path"], [])
4796 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4797 self.failUnlessEqual(first["type"], "directory")
4798 stats = units[-1]["stats"]
4799 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4800 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4801 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4802 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4803 self.failUnlessReallyEqual(stats["count-unknown"], 1)
4804 d.addCallback(_check_manifest)
4806 # now add root/subdir and root/subdir/grandchild, then make subdir
4807 # unrecoverable, then see what happens
4809 d.addCallback(lambda ign:
4810 self.rootnode.create_subdirectory(u"subdir"))
4811 d.addCallback(_stash_uri, "subdir")
4812 d.addCallback(lambda subdir_node:
4813 subdir_node.add_file(u"grandchild",
4814 upload.Data(DATA+"2",
4816 d.addCallback(_stash_uri, "grandchild")
4818 d.addCallback(lambda ign:
4819 self.delete_shares_numbered(self.uris["subdir"],
4827 # root/subdir [unrecoverable]
4828 # root/subdir/grandchild
4830 # how should a streaming-JSON API indicate fatal error?
4831 # answer: emit ERROR: instead of a JSON string
4833 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4834 def _check_broken_manifest(res):
4835 lines = res.splitlines()
4837 for (i,line) in enumerate(lines)
4838 if line.startswith("ERROR:")]
4840 self.fail("no ERROR: in output: %s" % (res,))
4841 first_error = error_lines[0]
4842 error_line = lines[first_error]
4843 error_msg = lines[first_error+1:]
4844 error_msg_s = "\n".join(error_msg) + "\n"
4845 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4847 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4848 units = [simplejson.loads(line) for line in lines[:first_error]]
4849 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4850 last_unit = units[-1]
4851 self.failUnlessEqual(last_unit["path"], ["subdir"])
4852 d.addCallback(_check_broken_manifest)
4854 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4855 def _check_broken_deepcheck(res):
4856 lines = res.splitlines()
4858 for (i,line) in enumerate(lines)
4859 if line.startswith("ERROR:")]
4861 self.fail("no ERROR: in output: %s" % (res,))
4862 first_error = error_lines[0]
4863 error_line = lines[first_error]
4864 error_msg = lines[first_error+1:]
4865 error_msg_s = "\n".join(error_msg) + "\n"
4866 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4868 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4869 units = [simplejson.loads(line) for line in lines[:first_error]]
4870 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4871 last_unit = units[-1]
4872 self.failUnlessEqual(last_unit["path"], ["subdir"])
4873 r = last_unit["check-results"]["results"]
4874 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4875 self.failUnlessReallyEqual(r["count-shares-good"], 1)
4876 self.failUnlessReallyEqual(r["recoverable"], False)
4877 d.addCallback(_check_broken_deepcheck)
4879 d.addErrback(self.explain_web_error)
4882 def test_deep_check_and_repair(self):
4883 self.basedir = "web/Grid/deep_check_and_repair"
4885 c0 = self.g.clients[0]
4889 d = c0.create_dirnode()
4890 def _stash_root_and_create_file(n):
4892 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4893 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4894 d.addCallback(_stash_root_and_create_file)
4895 def _stash_uri(fn, which):
4896 self.uris[which] = fn.get_uri()
4897 d.addCallback(_stash_uri, "good")
4898 d.addCallback(lambda ign:
4899 self.rootnode.add_file(u"small",
4900 upload.Data("literal",
4902 d.addCallback(_stash_uri, "small")
4903 d.addCallback(lambda ign:
4904 self.rootnode.add_file(u"sick",
4905 upload.Data(DATA+"1",
4907 d.addCallback(_stash_uri, "sick")
4908 #d.addCallback(lambda ign:
4909 # self.rootnode.add_file(u"dead",
4910 # upload.Data(DATA+"2",
4912 #d.addCallback(_stash_uri, "dead")
4914 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4915 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4916 #d.addCallback(_stash_uri, "corrupt")
4918 def _clobber_shares(ignored):
4919 good_shares = self.find_uri_shares(self.uris["good"])
4920 self.failUnlessReallyEqual(len(good_shares), 10)
4921 sick_shares = self.find_uri_shares(self.uris["sick"])
4922 os.unlink(sick_shares[0][2])
4923 #dead_shares = self.find_uri_shares(self.uris["dead"])
4924 #for i in range(1, 10):
4925 # os.unlink(dead_shares[i][2])
4927 #c_shares = self.find_uri_shares(self.uris["corrupt"])
4928 #cso = CorruptShareOptions()
4929 #cso.stdout = StringIO()
4930 #cso.parseOptions([c_shares[0][2]])
4932 d.addCallback(_clobber_shares)
4935 # root/good CHK, 10 shares
4937 # root/sick CHK, 9 shares
4939 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4941 units = [simplejson.loads(line)
4942 for line in res.splitlines()
4944 self.failUnlessReallyEqual(len(units), 4+1)
4945 # should be parent-first
4947 self.failUnlessEqual(u0["path"], [])
4948 self.failUnlessEqual(u0["type"], "directory")
4949 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4950 u0crr = u0["check-and-repair-results"]
4951 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4952 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4954 ugood = [u for u in units
4955 if u["type"] == "file" and u["path"] == [u"good"]][0]
4956 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4957 ugoodcrr = ugood["check-and-repair-results"]
4958 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4959 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4961 usick = [u for u in units
4962 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4963 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4964 usickcrr = usick["check-and-repair-results"]
4965 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4966 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4967 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4968 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4971 self.failUnlessEqual(stats["type"], "stats")
4973 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4974 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4975 self.failUnlessReallyEqual(s["count-directories"], 1)
4976 d.addCallback(_done)
4978 d.addErrback(self.explain_web_error)
4981 def _count_leases(self, ignored, which):
4982 u = self.uris[which]
4983 shares = self.find_uri_shares(u)
4985 for shnum, serverid, fn in shares:
4986 sf = get_share_file(fn)
4987 num_leases = len(list(sf.get_leases()))
4988 lease_counts.append( (fn, num_leases) )
4991 def _assert_leasecount(self, lease_counts, expected):
4992 for (fn, num_leases) in lease_counts:
4993 if num_leases != expected:
4994 self.fail("expected %d leases, have %d, on %s" %
4995 (expected, num_leases, fn))
4997 def test_add_lease(self):
4998 self.basedir = "web/Grid/add_lease"
4999 self.set_up_grid(num_clients=2)
5000 c0 = self.g.clients[0]
5003 d = c0.upload(upload.Data(DATA, convergence=""))
5004 def _stash_uri(ur, which):
5005 self.uris[which] = ur.uri
5006 d.addCallback(_stash_uri, "one")
5007 d.addCallback(lambda ign:
5008 c0.upload(upload.Data(DATA+"1", convergence="")))
5009 d.addCallback(_stash_uri, "two")
5010 def _stash_mutable_uri(n, which):
5011 self.uris[which] = n.get_uri()
5012 assert isinstance(self.uris[which], str)
5013 d.addCallback(lambda ign:
5014 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5015 d.addCallback(_stash_mutable_uri, "mutable")
5017 def _compute_fileurls(ignored):
5019 for which in self.uris:
5020 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5021 d.addCallback(_compute_fileurls)
5023 d.addCallback(self._count_leases, "one")
5024 d.addCallback(self._assert_leasecount, 1)
5025 d.addCallback(self._count_leases, "two")
5026 d.addCallback(self._assert_leasecount, 1)
5027 d.addCallback(self._count_leases, "mutable")
5028 d.addCallback(self._assert_leasecount, 1)
5030 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5031 def _got_html_good(res):
5032 self.failUnlessIn("Healthy", res)
5033 self.failIfIn("Not Healthy", res)
5034 d.addCallback(_got_html_good)
5036 d.addCallback(self._count_leases, "one")
5037 d.addCallback(self._assert_leasecount, 1)
5038 d.addCallback(self._count_leases, "two")
5039 d.addCallback(self._assert_leasecount, 1)
5040 d.addCallback(self._count_leases, "mutable")
5041 d.addCallback(self._assert_leasecount, 1)
5043 # this CHECK uses the original client, which uses the same
5044 # lease-secrets, so it will just renew the original lease
5045 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5046 d.addCallback(_got_html_good)
5048 d.addCallback(self._count_leases, "one")
5049 d.addCallback(self._assert_leasecount, 1)
5050 d.addCallback(self._count_leases, "two")
5051 d.addCallback(self._assert_leasecount, 1)
5052 d.addCallback(self._count_leases, "mutable")
5053 d.addCallback(self._assert_leasecount, 1)
5055 # this CHECK uses an alternate client, which adds a second lease
5056 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5057 d.addCallback(_got_html_good)
5059 d.addCallback(self._count_leases, "one")
5060 d.addCallback(self._assert_leasecount, 2)
5061 d.addCallback(self._count_leases, "two")
5062 d.addCallback(self._assert_leasecount, 1)
5063 d.addCallback(self._count_leases, "mutable")
5064 d.addCallback(self._assert_leasecount, 1)
5066 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5067 d.addCallback(_got_html_good)
5069 d.addCallback(self._count_leases, "one")
5070 d.addCallback(self._assert_leasecount, 2)
5071 d.addCallback(self._count_leases, "two")
5072 d.addCallback(self._assert_leasecount, 1)
5073 d.addCallback(self._count_leases, "mutable")
5074 d.addCallback(self._assert_leasecount, 1)
5076 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5078 d.addCallback(_got_html_good)
5080 d.addCallback(self._count_leases, "one")
5081 d.addCallback(self._assert_leasecount, 2)
5082 d.addCallback(self._count_leases, "two")
5083 d.addCallback(self._assert_leasecount, 1)
5084 d.addCallback(self._count_leases, "mutable")
5085 d.addCallback(self._assert_leasecount, 2)
5087 d.addErrback(self.explain_web_error)
5090 def test_deep_add_lease(self):
5091 self.basedir = "web/Grid/deep_add_lease"
5092 self.set_up_grid(num_clients=2)
5093 c0 = self.g.clients[0]
5097 d = c0.create_dirnode()
5098 def _stash_root_and_create_file(n):
5100 self.uris["root"] = n.get_uri()
5101 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5102 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5103 d.addCallback(_stash_root_and_create_file)
5104 def _stash_uri(fn, which):
5105 self.uris[which] = fn.get_uri()
5106 d.addCallback(_stash_uri, "one")
5107 d.addCallback(lambda ign:
5108 self.rootnode.add_file(u"small",
5109 upload.Data("literal",
5111 d.addCallback(_stash_uri, "small")
5113 d.addCallback(lambda ign:
5114 c0.create_mutable_file(publish.MutableData("mutable")))
5115 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5116 d.addCallback(_stash_uri, "mutable")
5118 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5120 units = [simplejson.loads(line)
5121 for line in res.splitlines()
5123 # root, one, small, mutable, stats
5124 self.failUnlessReallyEqual(len(units), 4+1)
5125 d.addCallback(_done)
5127 d.addCallback(self._count_leases, "root")
5128 d.addCallback(self._assert_leasecount, 1)
5129 d.addCallback(self._count_leases, "one")
5130 d.addCallback(self._assert_leasecount, 1)
5131 d.addCallback(self._count_leases, "mutable")
5132 d.addCallback(self._assert_leasecount, 1)
5134 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5135 d.addCallback(_done)
5137 d.addCallback(self._count_leases, "root")
5138 d.addCallback(self._assert_leasecount, 1)
5139 d.addCallback(self._count_leases, "one")
5140 d.addCallback(self._assert_leasecount, 1)
5141 d.addCallback(self._count_leases, "mutable")
5142 d.addCallback(self._assert_leasecount, 1)
5144 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5146 d.addCallback(_done)
5148 d.addCallback(self._count_leases, "root")
5149 d.addCallback(self._assert_leasecount, 2)
5150 d.addCallback(self._count_leases, "one")
5151 d.addCallback(self._assert_leasecount, 2)
5152 d.addCallback(self._count_leases, "mutable")
5153 d.addCallback(self._assert_leasecount, 2)
5155 d.addErrback(self.explain_web_error)
5159 def test_exceptions(self):
5160 self.basedir = "web/Grid/exceptions"
5161 self.set_up_grid(num_clients=1, num_servers=2)
5162 c0 = self.g.clients[0]
5163 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5166 d = c0.create_dirnode()
5168 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5169 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5171 d.addCallback(_stash_root)
5172 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5174 self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5175 self.delete_shares_numbered(ur.uri, range(1,10))
5177 u = uri.from_string(ur.uri)
5178 u.key = testutil.flip_bit(u.key, 0)
5179 baduri = u.to_string()
5180 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5181 d.addCallback(_stash_bad)
5182 d.addCallback(lambda ign: c0.create_dirnode())
5183 def _mangle_dirnode_1share(n):
5185 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5186 self.fileurls["dir-1share-json"] = url + "?t=json"
5187 self.delete_shares_numbered(u, range(1,10))
5188 d.addCallback(_mangle_dirnode_1share)
5189 d.addCallback(lambda ign: c0.create_dirnode())
5190 def _mangle_dirnode_0share(n):
5192 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5193 self.fileurls["dir-0share-json"] = url + "?t=json"
5194 self.delete_shares_numbered(u, range(0,10))
5195 d.addCallback(_mangle_dirnode_0share)
5197 # NotEnoughSharesError should be reported sensibly, with a
5198 # text/plain explanation of the problem, and perhaps some
5199 # information on which shares *could* be found.
5201 d.addCallback(lambda ignored:
5202 self.shouldHTTPError("GET unrecoverable",
5203 410, "Gone", "NoSharesError",
5204 self.GET, self.fileurls["0shares"]))
5205 def _check_zero_shares(body):
5206 self.failIfIn("<html>", body)
5207 body = " ".join(body.strip().split())
5208 exp = ("NoSharesError: no shares could be found. "
5209 "Zero shares usually indicates a corrupt URI, or that "
5210 "no servers were connected, but it might also indicate "
5211 "severe corruption. You should perform a filecheck on "
5212 "this object to learn more. The full error message is: "
5213 "no shares (need 3). Last failure: None")
5214 self.failUnlessReallyEqual(exp, body)
5215 d.addCallback(_check_zero_shares)
5218 d.addCallback(lambda ignored:
5219 self.shouldHTTPError("GET 1share",
5220 410, "Gone", "NotEnoughSharesError",
5221 self.GET, self.fileurls["1share"]))
5222 def _check_one_share(body):
5223 self.failIfIn("<html>", body)
5224 body = " ".join(body.strip().split())
5225 msgbase = ("NotEnoughSharesError: This indicates that some "
5226 "servers were unavailable, or that shares have been "
5227 "lost to server departure, hard drive failure, or disk "
5228 "corruption. You should perform a filecheck on "
5229 "this object to learn more. The full error message is:"
5231 msg1 = msgbase + (" ran out of shares:"
5234 " overdue= unused= need 3. Last failure: None")
5235 msg2 = msgbase + (" ran out of shares:"
5237 " pending=Share(sh0-on-xgru5)"
5238 " overdue= unused= need 3. Last failure: None")
5239 self.failUnless(body == msg1 or body == msg2, body)
5240 d.addCallback(_check_one_share)
5242 d.addCallback(lambda ignored:
5243 self.shouldHTTPError("GET imaginary",
5244 404, "Not Found", None,
5245 self.GET, self.fileurls["imaginary"]))
5246 def _missing_child(body):
5247 self.failUnlessIn("No such child: imaginary", body)
5248 d.addCallback(_missing_child)
5250 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5251 def _check_0shares_dir_html(body):
5252 self.failUnlessIn("<html>", body)
5253 # we should see the regular page, but without the child table or
5255 body = " ".join(body.strip().split())
5256 self.failUnlessIn('href="?t=info">More info on this directory',
5258 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5259 "could not be retrieved, because there were insufficient "
5260 "good shares. This might indicate that no servers were "
5261 "connected, insufficient servers were connected, the URI "
5262 "was corrupt, or that shares have been lost due to server "
5263 "departure, hard drive failure, or disk corruption. You "
5264 "should perform a filecheck on this object to learn more.")
5265 self.failUnlessIn(exp, body)
5266 self.failUnlessIn("No upload forms: directory is unreadable", body)
5267 d.addCallback(_check_0shares_dir_html)
5269 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5270 def _check_1shares_dir_html(body):
5271 # at some point, we'll split UnrecoverableFileError into 0-shares
5272 # and some-shares like we did for immutable files (since there
5273 # are different sorts of advice to offer in each case). For now,
5274 # they present the same way.
5275 self.failUnlessIn("<html>", body)
5276 body = " ".join(body.strip().split())
5277 self.failUnlessIn('href="?t=info">More info on this directory',
5279 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5280 "could not be retrieved, because there were insufficient "
5281 "good shares. This might indicate that no servers were "
5282 "connected, insufficient servers were connected, the URI "
5283 "was corrupt, or that shares have been lost due to server "
5284 "departure, hard drive failure, or disk corruption. You "
5285 "should perform a filecheck on this object to learn more.")
5286 self.failUnlessIn(exp, body)
5287 self.failUnlessIn("No upload forms: directory is unreadable", body)
5288 d.addCallback(_check_1shares_dir_html)
5290 d.addCallback(lambda ignored:
5291 self.shouldHTTPError("GET dir-0share-json",
5292 410, "Gone", "UnrecoverableFileError",
5294 self.fileurls["dir-0share-json"]))
5295 def _check_unrecoverable_file(body):
5296 self.failIfIn("<html>", body)
5297 body = " ".join(body.strip().split())
5298 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5299 "could not be retrieved, because there were insufficient "
5300 "good shares. This might indicate that no servers were "
5301 "connected, insufficient servers were connected, the URI "
5302 "was corrupt, or that shares have been lost due to server "
5303 "departure, hard drive failure, or disk corruption. You "
5304 "should perform a filecheck on this object to learn more.")
5305 self.failUnlessReallyEqual(exp, body)
5306 d.addCallback(_check_unrecoverable_file)
5308 d.addCallback(lambda ignored:
5309 self.shouldHTTPError("GET dir-1share-json",
5310 410, "Gone", "UnrecoverableFileError",
5312 self.fileurls["dir-1share-json"]))
5313 d.addCallback(_check_unrecoverable_file)
5315 d.addCallback(lambda ignored:
5316 self.shouldHTTPError("GET imaginary",
5317 404, "Not Found", None,
5318 self.GET, self.fileurls["imaginary"]))
5320 # attach a webapi child that throws a random error, to test how it
5322 w = c0.getServiceNamed("webish")
5323 w.root.putChild("ERRORBOOM", ErrorBoom())
5325 # "Accept: */*" : should get a text/html stack trace
5326 # "Accept: text/plain" : should get a text/plain stack trace
5327 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5328 # no Accept header: should get a text/html stack trace
5330 d.addCallback(lambda ignored:
5331 self.shouldHTTPError("GET errorboom_html",
5332 500, "Internal Server Error", None,
5333 self.GET, "ERRORBOOM",
5334 headers={"accept": ["*/*"]}))
5335 def _internal_error_html1(body):
5336 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5337 d.addCallback(_internal_error_html1)
5339 d.addCallback(lambda ignored:
5340 self.shouldHTTPError("GET errorboom_text",
5341 500, "Internal Server Error", None,
5342 self.GET, "ERRORBOOM",
5343 headers={"accept": ["text/plain"]}))
5344 def _internal_error_text2(body):
5345 self.failIfIn("<html>", body)
5346 self.failUnless(body.startswith("Traceback "), body)
5347 d.addCallback(_internal_error_text2)
5349 CLI_accepts = "text/plain, application/octet-stream"
5350 d.addCallback(lambda ignored:
5351 self.shouldHTTPError("GET errorboom_text",
5352 500, "Internal Server Error", None,
5353 self.GET, "ERRORBOOM",
5354 headers={"accept": [CLI_accepts]}))
5355 def _internal_error_text3(body):
5356 self.failIfIn("<html>", body)
5357 self.failUnless(body.startswith("Traceback "), body)
5358 d.addCallback(_internal_error_text3)
5360 d.addCallback(lambda ignored:
5361 self.shouldHTTPError("GET errorboom_text",
5362 500, "Internal Server Error", None,
5363 self.GET, "ERRORBOOM"))
5364 def _internal_error_html4(body):
5365 self.failUnlessIn("<html>", body)
5366 d.addCallback(_internal_error_html4)
5368 def _flush_errors(res):
5369 # Trial: please ignore the CompletelyUnhandledError in the logs
5370 self.flushLoggedErrors(CompletelyUnhandledError)
5372 d.addBoth(_flush_errors)
5376 def test_blacklist(self):
5377 # download from a blacklisted URI, get an error
5378 self.basedir = "web/Grid/blacklist"
5380 c0 = self.g.clients[0]
5381 c0_basedir = c0.basedir
5382 fn = os.path.join(c0_basedir, "access.blacklist")
5384 DATA = "off-limits " * 50
5386 d = c0.upload(upload.Data(DATA, convergence=""))
5387 def _stash_uri_and_create_dir(ur):
5389 self.url = "uri/"+self.uri
5390 u = uri.from_string_filenode(self.uri)
5391 self.si = u.get_storage_index()
5392 childnode = c0.create_node_from_uri(self.uri, None)
5393 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5394 d.addCallback(_stash_uri_and_create_dir)
5395 def _stash_dir(node):
5396 self.dir_node = node
5397 self.dir_uri = node.get_uri()
5398 self.dir_url = "uri/"+self.dir_uri
5399 d.addCallback(_stash_dir)
5400 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5401 def _check_dir_html(body):
5402 self.failUnlessIn("<html>", body)
5403 self.failUnlessIn("blacklisted.txt</a>", body)
5404 d.addCallback(_check_dir_html)
5405 d.addCallback(lambda ign: self.GET(self.url))
5406 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5408 def _blacklist(ign):
5410 f.write(" # this is a comment\n")
5412 f.write("\n") # also exercise blank lines
5413 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5415 # clients should be checking the blacklist each time, so we don't
5416 # need to restart the client
5417 d.addCallback(_blacklist)
5418 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5420 "Access Prohibited: off-limits",
5421 self.GET, self.url))
5423 # We should still be able to list the parent directory, in HTML...
5424 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5425 def _check_dir_html2(body):
5426 self.failUnlessIn("<html>", body)
5427 self.failUnlessIn("blacklisted.txt</strike>", body)
5428 d.addCallback(_check_dir_html2)
5430 # ... and in JSON (used by CLI).
5431 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5432 def _check_dir_json(res):
5433 data = simplejson.loads(res)
5434 self.failUnless(isinstance(data, list), data)
5435 self.failUnlessEqual(data[0], "dirnode")
5436 self.failUnless(isinstance(data[1], dict), data)
5437 self.failUnlessIn("children", data[1])
5438 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5439 childdata = data[1]["children"]["blacklisted.txt"]
5440 self.failUnless(isinstance(childdata, list), data)
5441 self.failUnlessEqual(childdata[0], "filenode")
5442 self.failUnless(isinstance(childdata[1], dict), data)
5443 d.addCallback(_check_dir_json)
5445 def _unblacklist(ign):
5446 open(fn, "w").close()
5447 # the Blacklist object watches mtime to tell when the file has
5448 # changed, but on windows this test will run faster than the
5449 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5450 # to force a reload.
5451 self.g.clients[0].blacklist.last_mtime -= 2.0
5452 d.addCallback(_unblacklist)
5454 # now a read should work
5455 d.addCallback(lambda ign: self.GET(self.url))
5456 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5458 # read again to exercise the blacklist-is-unchanged logic
5459 d.addCallback(lambda ign: self.GET(self.url))
5460 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5462 # now add a blacklisted directory, and make sure files under it are
5465 childnode = c0.create_node_from_uri(self.uri, None)
5466 return c0.create_dirnode({u"child": (childnode,{}) })
5467 d.addCallback(_add_dir)
5468 def _get_dircap(dn):
5469 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5470 self.dir_url_base = "uri/"+dn.get_write_uri()
5471 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5472 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5473 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5474 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5475 d.addCallback(_get_dircap)
5476 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5477 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5478 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5479 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5480 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5481 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5482 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5483 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5484 d.addCallback(lambda ign: self.GET(self.child_url))
5485 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5487 def _block_dir(ign):
5489 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5491 self.g.clients[0].blacklist.last_mtime -= 2.0
5492 d.addCallback(_block_dir)
5493 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5495 "Access Prohibited: dir-off-limits",
5496 self.GET, self.dir_url_base))
5497 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5499 "Access Prohibited: dir-off-limits",
5500 self.GET, self.dir_url_json1))
5501 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5503 "Access Prohibited: dir-off-limits",
5504 self.GET, self.dir_url_json2))
5505 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5507 "Access Prohibited: dir-off-limits",
5508 self.GET, self.dir_url_json_ro))
5509 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5511 "Access Prohibited: dir-off-limits",
5512 self.GET, self.child_url))
5516 class CompletelyUnhandledError(Exception):
5518 class ErrorBoom(rend.Page):
5519 def beforeRender(self, ctx):
5520 raise CompletelyUnhandledError("whoops")