1 import os.path, re, urllib, time
3 from StringIO import StringIO
4 from twisted.application import service
5 from twisted.trial import unittest
6 from twisted.internet import defer, reactor
7 from twisted.internet.task import Clock
8 from twisted.web import client, error, http
9 from twisted.python import failure, log
10 from nevow import rend
11 from allmydata import interfaces, uri, webish, dirnode
12 from allmydata.storage.shares import get_share_file
13 from allmydata.storage_client import StorageFarmBroker
14 from allmydata.immutable import upload
15 from allmydata.immutable.downloader.status import DownloadStatus
16 from allmydata.dirnode import DirectoryNode
17 from allmydata.nodemaker import NodeMaker
18 from allmydata.unknown import UnknownNode
19 from allmydata.web import status, common
20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
21 from allmydata.util import fileutil, base32, hashutil
22 from allmydata.util.consumer import download_to_data
23 from allmydata.util.netstring import split_netstring
24 from allmydata.util.encodingutil import to_str
25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
26 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
27 make_mutable_file_uri, create_mutable_filenode
28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
29 from allmydata.mutable import servermap, publish, retrieve
30 import allmydata.test.common_util as testutil
31 from allmydata.test.no_network import GridTestMixin
32 from allmydata.test.common_web import HTTPClientGETFactory, \
34 from allmydata.client import Client, SecretHolder
36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
37 # create a webserver that works against them
39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
45 class FakeStatsProvider:
47 stats = {'stats': {}, 'counters': {}}
50 class FakeNodeMaker(NodeMaker):
55 'max_segment_size':128*1024 # 1024=KiB
57 def _create_lit(self, cap):
58 return FakeCHKFileNode(cap)
59 def _create_immutable(self, cap):
60 return FakeCHKFileNode(cap)
61 def _create_mutable(self, cap):
62 return FakeMutableFileNode(None,
64 self.encoding_params, None).init_from_cap(cap)
65 def create_mutable_file(self, contents="", keysize=None,
66 version=SDMF_VERSION):
67 n = FakeMutableFileNode(None, None, self.encoding_params, None)
68 return n.create(contents, version=version)
70 class FakeUploader(service.Service):
72 def upload(self, uploadable, history=None):
73 d = uploadable.get_size()
74 d.addCallback(lambda size: uploadable.read(size))
77 n = create_chk_filenode(data)
78 results = upload.UploadResults()
79 results.uri = n.get_uri()
81 d.addCallback(_got_data)
83 def get_helper_info(self):
87 def __init__(self, binaryserverid):
88 self.binaryserverid = binaryserverid
89 def get_name(self): return "short"
90 def get_longname(self): return "long"
91 def get_serverid(self): return self.binaryserverid
94 ds = DownloadStatus("storage_index", 1234)
97 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
98 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
99 storage_index = hashutil.storage_index_hash("SI")
100 e0 = ds.add_segment_request(0, now)
102 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
103 e1 = ds.add_segment_request(1, now+2)
105 # two outstanding requests
106 e2 = ds.add_segment_request(2, now+4)
107 e3 = ds.add_segment_request(3, now+5)
108 del e2,e3 # hush pyflakes
110 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
111 e = ds.add_segment_request(4, now)
113 e.deliver(now, 0, 140, 0.5)
115 e = ds.add_dyhb_request(serverA, now)
116 e.finished([1,2], now+1)
117 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
119 e = ds.add_read_event(0, 120, now)
120 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
122 e = ds.add_read_event(120, 30, now+2) # left unfinished
124 e = ds.add_block_request(serverA, 1, 100, 20, now)
125 e.finished(20, now+1)
126 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
128 # make sure that add_read_event() can come first too
129 ds1 = DownloadStatus(storage_index, 1234)
130 e = ds1.add_read_event(0, 120, now)
131 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
137 _all_upload_status = [upload.UploadStatus()]
138 _all_download_status = [build_one_ds()]
139 _all_mapupdate_statuses = [servermap.UpdateStatus()]
140 _all_publish_statuses = [publish.PublishStatus()]
141 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
143 def list_all_upload_statuses(self):
144 return self._all_upload_status
145 def list_all_download_statuses(self):
146 return self._all_download_status
147 def list_all_mapupdate_statuses(self):
148 return self._all_mapupdate_statuses
149 def list_all_publish_statuses(self):
150 return self._all_publish_statuses
151 def list_all_retrieve_statuses(self):
152 return self._all_retrieve_statuses
153 def list_all_helper_statuses(self):
156 class FakeClient(Client):
158 # don't upcall to Client.__init__, since we only want to initialize a
160 service.MultiService.__init__(self)
161 self.nodeid = "fake_nodeid"
162 self.nickname = "fake_nickname"
163 self.introducer_furl = "None"
164 self.stats_provider = FakeStatsProvider()
165 self._secret_holder = SecretHolder("lease secret", "convergence secret")
167 self.convergence = "some random string"
168 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
169 self.introducer_client = None
170 self.history = FakeHistory()
171 self.uploader = FakeUploader()
172 self.uploader.setServiceParent(self)
173 self.blacklist = None
174 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
177 self.mutable_file_default = SDMF_VERSION
179 def startService(self):
180 return service.MultiService.startService(self)
181 def stopService(self):
182 return service.MultiService.stopService(self)
184 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
186 class WebMixin(object):
188 self.s = FakeClient()
189 self.s.startService()
190 self.staticdir = self.mktemp()
192 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
194 self.ws.setServiceParent(self.s)
195 self.webish_port = self.ws.getPortnum()
196 self.webish_url = self.ws.getURL()
197 assert self.webish_url.endswith("/")
198 self.webish_url = self.webish_url[:-1] # these tests add their own /
200 l = [ self.s.create_dirnode() for x in range(6) ]
201 d = defer.DeferredList(l)
203 self.public_root = res[0][1]
204 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
205 self.public_url = "/uri/" + self.public_root.get_uri()
206 self.private_root = res[1][1]
210 self._foo_uri = foo.get_uri()
211 self._foo_readonly_uri = foo.get_readonly_uri()
212 self._foo_verifycap = foo.get_verify_cap().to_string()
213 # NOTE: we ignore the deferred on all set_uri() calls, because we
214 # know the fake nodes do these synchronously
215 self.public_root.set_uri(u"foo", foo.get_uri(),
216 foo.get_readonly_uri())
218 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
219 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
220 self._bar_txt_verifycap = n.get_verify_cap().to_string()
223 # XXX: Do we ever use this?
224 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
226 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
229 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
230 assert self._quux_txt_uri.startswith("URI:MDMF")
231 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
233 foo.set_uri(u"empty", res[3][1].get_uri(),
234 res[3][1].get_readonly_uri())
235 sub_uri = res[4][1].get_uri()
236 self._sub_uri = sub_uri
237 foo.set_uri(u"sub", sub_uri, sub_uri)
238 sub = self.s.create_node_from_uri(sub_uri)
240 _ign, n, blocking_uri = self.makefile(1)
241 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
243 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
244 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
245 # still think of it as an umlaut
246 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
248 _ign, n, baz_file = self.makefile(2)
249 self._baz_file_uri = baz_file
250 sub.set_uri(u"baz.txt", baz_file, baz_file)
252 _ign, n, self._bad_file_uri = self.makefile(3)
253 # this uri should not be downloadable
254 del FakeCHKFileNode.all_contents[self._bad_file_uri]
257 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
258 rodir.get_readonly_uri())
259 rodir.set_uri(u"nor", baz_file, baz_file)
265 # public/foo/quux.txt
266 # public/foo/blockingfile
269 # public/foo/sub/baz.txt
271 # public/reedownlee/nor
272 self.NEWFILE_CONTENTS = "newfile contents\n"
274 return foo.get_metadata_for(u"bar.txt")
276 def _got_metadata(metadata):
277 self._bar_txt_metadata = metadata
278 d.addCallback(_got_metadata)
281 def makefile(self, number):
282 contents = "contents of file %s\n" % number
283 n = create_chk_filenode(contents)
284 return contents, n, n.get_uri()
286 def makefile_mutable(self, number, mdmf=False):
287 contents = "contents of mutable file %s\n" % number
288 n = create_mutable_filenode(contents, mdmf)
289 return contents, n, n.get_uri(), n.get_readonly_uri()
292 return self.s.stopService()
294 def failUnlessIsBarDotTxt(self, res):
295 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
297 def failUnlessIsQuuxDotTxt(self, res):
298 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
300 def failUnlessIsBazDotTxt(self, res):
301 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
303 def failUnlessIsBarJSON(self, res):
304 data = simplejson.loads(res)
305 self.failUnless(isinstance(data, list))
306 self.failUnlessEqual(data[0], "filenode")
307 self.failUnless(isinstance(data[1], dict))
308 self.failIf(data[1]["mutable"])
309 self.failIf("rw_uri" in data[1]) # immutable
310 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
311 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
312 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
314 def failUnlessIsQuuxJSON(self, res, readonly=False):
315 data = simplejson.loads(res)
316 self.failUnless(isinstance(data, list))
317 self.failUnlessEqual(data[0], "filenode")
318 self.failUnless(isinstance(data[1], dict))
320 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
322 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
323 self.failUnless(metadata['mutable'])
325 self.failIf("rw_uri" in metadata)
327 self.failUnless("rw_uri" in metadata)
328 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
329 self.failUnless("ro_uri" in metadata)
330 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
331 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
333 def failUnlessIsFooJSON(self, res):
334 data = simplejson.loads(res)
335 self.failUnless(isinstance(data, list))
336 self.failUnlessEqual(data[0], "dirnode", res)
337 self.failUnless(isinstance(data[1], dict))
338 self.failUnless(data[1]["mutable"])
339 self.failUnless("rw_uri" in data[1]) # mutable
340 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
341 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
342 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
344 kidnames = sorted([unicode(n) for n in data[1]["children"]])
345 self.failUnlessEqual(kidnames,
346 [u"bar.txt", u"baz.txt", u"blockingfile",
347 u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
348 kids = dict( [(unicode(name),value)
350 in data[1]["children"].iteritems()] )
351 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
352 self.failUnlessIn("metadata", kids[u"sub"][1])
353 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
354 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
355 self.failUnlessIn("linkcrtime", tahoe_md)
356 self.failUnlessIn("linkmotime", tahoe_md)
357 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
358 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
359 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
360 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
361 self._bar_txt_verifycap)
362 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
363 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
364 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
365 self._bar_txt_metadata["tahoe"]["linkcrtime"])
366 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
368 self.failUnlessIn("quux.txt", kids)
369 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
371 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
372 self._quux_txt_readonly_uri)
374 def GET(self, urlpath, followRedirect=False, return_response=False,
376 # if return_response=True, this fires with (data, statuscode,
377 # respheaders) instead of just data.
378 assert not isinstance(urlpath, unicode)
379 url = self.webish_url + urlpath
380 factory = HTTPClientGETFactory(url, method="GET",
381 followRedirect=followRedirect, **kwargs)
382 reactor.connectTCP("localhost", self.webish_port, factory)
385 return (data, factory.status, factory.response_headers)
387 d.addCallback(_got_data)
388 return factory.deferred
390 def HEAD(self, urlpath, return_response=False, **kwargs):
391 # this requires some surgery, because twisted.web.client doesn't want
392 # to give us back the response headers.
393 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
394 reactor.connectTCP("localhost", self.webish_port, factory)
397 return (data, factory.status, factory.response_headers)
399 d.addCallback(_got_data)
400 return factory.deferred
402 def PUT(self, urlpath, data, **kwargs):
403 url = self.webish_url + urlpath
404 return client.getPage(url, method="PUT", postdata=data, **kwargs)
406 def DELETE(self, urlpath):
407 url = self.webish_url + urlpath
408 return client.getPage(url, method="DELETE")
410 def POST(self, urlpath, followRedirect=False, **fields):
411 sepbase = "boogabooga"
415 form.append('Content-Disposition: form-data; name="_charset"')
419 for name, value in fields.iteritems():
420 if isinstance(value, tuple):
421 filename, value = value
422 form.append('Content-Disposition: form-data; name="%s"; '
423 'filename="%s"' % (name, filename.encode("utf-8")))
425 form.append('Content-Disposition: form-data; name="%s"' % name)
427 if isinstance(value, unicode):
428 value = value.encode("utf-8")
431 assert isinstance(value, str)
438 body = "\r\n".join(form) + "\r\n"
439 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
440 return self.POST2(urlpath, body, headers, followRedirect)
442 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
443 url = self.webish_url + urlpath
444 return client.getPage(url, method="POST", postdata=body,
445 headers=headers, followRedirect=followRedirect)
447 def shouldFail(self, res, expected_failure, which,
448 substring=None, response_substring=None):
449 if isinstance(res, failure.Failure):
450 res.trap(expected_failure)
452 self.failUnless(substring in str(res),
453 "substring '%s' not in '%s'"
454 % (substring, str(res)))
455 if response_substring:
456 self.failUnless(response_substring in res.value.response,
457 "response substring '%s' not in '%s'"
458 % (response_substring, res.value.response))
460 self.fail("%s was supposed to raise %s, not get '%s'" %
461 (which, expected_failure, res))
463 def shouldFail2(self, expected_failure, which, substring,
465 callable, *args, **kwargs):
466 assert substring is None or isinstance(substring, str)
467 assert response_substring is None or isinstance(response_substring, str)
468 d = defer.maybeDeferred(callable, *args, **kwargs)
470 if isinstance(res, failure.Failure):
471 res.trap(expected_failure)
473 self.failUnless(substring in str(res),
474 "%s: substring '%s' not in '%s'"
475 % (which, substring, str(res)))
476 if response_substring:
477 self.failUnless(response_substring in res.value.response,
478 "%s: response substring '%s' not in '%s'"
480 response_substring, res.value.response))
482 self.fail("%s was supposed to raise %s, not get '%s'" %
483 (which, expected_failure, res))
487 def should404(self, res, which):
488 if isinstance(res, failure.Failure):
489 res.trap(error.Error)
490 self.failUnlessReallyEqual(res.value.status, "404")
492 self.fail("%s was supposed to Error(404), not get '%s'" %
495 def should302(self, res, which):
496 if isinstance(res, failure.Failure):
497 res.trap(error.Error)
498 self.failUnlessReallyEqual(res.value.status, "302")
500 self.fail("%s was supposed to Error(302), not get '%s'" %
504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
505 def test_create(self):
508 def test_welcome(self):
511 self.failUnless('Welcome To Tahoe-LAFS' in res, res)
513 self.s.basedir = 'web/test_welcome'
514 fileutil.make_dirs("web/test_welcome")
515 fileutil.make_dirs("web/test_welcome/private")
517 d.addCallback(_check)
520 def test_provisioning(self):
521 d = self.GET("/provisioning/")
523 self.failUnless('Provisioning Tool' in res)
524 fields = {'filled': True,
525 "num_users": int(50e3),
526 "files_per_user": 1000,
527 "space_per_user": int(1e9),
528 "sharing_ratio": 1.0,
529 "encoding_parameters": "3-of-10-5",
531 "ownership_mode": "A",
532 "download_rate": 100,
537 return self.POST("/provisioning/", **fields)
539 d.addCallback(_check)
541 self.failUnless('Provisioning Tool' in res)
542 self.failUnless("Share space consumed: 167.01TB" in res)
544 fields = {'filled': True,
545 "num_users": int(50e6),
546 "files_per_user": 1000,
547 "space_per_user": int(5e9),
548 "sharing_ratio": 1.0,
549 "encoding_parameters": "25-of-100-50",
550 "num_servers": 30000,
551 "ownership_mode": "E",
552 "drive_failure_model": "U",
554 "download_rate": 1000,
559 return self.POST("/provisioning/", **fields)
560 d.addCallback(_check2)
562 self.failUnless("Share space consumed: huge!" in res)
563 fields = {'filled': True}
564 return self.POST("/provisioning/", **fields)
565 d.addCallback(_check3)
567 self.failUnless("Share space consumed:" in res)
568 d.addCallback(_check4)
571 def test_reliability_tool(self):
573 from allmydata import reliability
574 _hush_pyflakes = reliability
577 raise unittest.SkipTest("reliability tool requires NumPy")
579 d = self.GET("/reliability/")
581 self.failUnless('Reliability Tool' in res)
582 fields = {'drive_lifetime': "8Y",
587 "check_period": "1M",
588 "report_period": "3M",
591 return self.POST("/reliability/", **fields)
593 d.addCallback(_check)
595 self.failUnless('Reliability Tool' in res)
596 r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
597 self.failUnless(re.search(r, res), res)
598 d.addCallback(_check2)
601 def test_status(self):
602 h = self.s.get_history()
603 dl_num = h.list_all_download_statuses()[0].get_counter()
604 ul_num = h.list_all_upload_statuses()[0].get_counter()
605 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
606 pub_num = h.list_all_publish_statuses()[0].get_counter()
607 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
608 d = self.GET("/status", followRedirect=True)
610 self.failUnless('Upload and Download Status' in res, res)
611 self.failUnless('"down-%d"' % dl_num in res, res)
612 self.failUnless('"up-%d"' % ul_num in res, res)
613 self.failUnless('"mapupdate-%d"' % mu_num in res, res)
614 self.failUnless('"publish-%d"' % pub_num in res, res)
615 self.failUnless('"retrieve-%d"' % ret_num in res, res)
616 d.addCallback(_check)
617 d.addCallback(lambda res: self.GET("/status/?t=json"))
618 def _check_json(res):
619 data = simplejson.loads(res)
620 self.failUnless(isinstance(data, dict))
621 #active = data["active"]
622 # TODO: test more. We need a way to fake an active operation
624 d.addCallback(_check_json)
626 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
628 self.failUnless("File Download Status" in res, res)
629 d.addCallback(_check_dl)
630 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
631 def _check_dl_json(res):
632 data = simplejson.loads(res)
633 self.failUnless(isinstance(data, dict))
634 self.failUnless("read" in data)
635 self.failUnlessEqual(data["read"][0]["length"], 120)
636 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
637 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
638 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
639 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
640 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
641 # serverids[] keys are strings, since that's what JSON does, but
642 # we'd really like them to be ints
643 self.failUnlessEqual(data["serverids"]["0"], "phwr")
644 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
645 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
646 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
647 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
648 self.failUnless("dyhb" in data)
649 d.addCallback(_check_dl_json)
650 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
652 self.failUnless("File Upload Status" in res, res)
653 d.addCallback(_check_ul)
654 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
655 def _check_mapupdate(res):
656 self.failUnless("Mutable File Servermap Update Status" in res, res)
657 d.addCallback(_check_mapupdate)
658 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
659 def _check_publish(res):
660 self.failUnless("Mutable File Publish Status" in res, res)
661 d.addCallback(_check_publish)
662 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
663 def _check_retrieve(res):
664 self.failUnless("Mutable File Retrieve Status" in res, res)
665 d.addCallback(_check_retrieve)
669 def test_status_numbers(self):
670 drrm = status.DownloadResultsRendererMixin()
671 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
672 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
673 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
674 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
675 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
676 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
677 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
678 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
679 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
681 urrm = status.UploadResultsRendererMixin()
682 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
683 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
684 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
685 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
686 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
687 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
688 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
689 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
690 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
692 def test_GET_FILEURL(self):
693 d = self.GET(self.public_url + "/foo/bar.txt")
694 d.addCallback(self.failUnlessIsBarDotTxt)
697 def test_GET_FILEURL_range(self):
698 headers = {"range": "bytes=1-10"}
699 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
700 return_response=True)
701 def _got((res, status, headers)):
702 self.failUnlessReallyEqual(int(status), 206)
703 self.failUnless(headers.has_key("content-range"))
704 self.failUnlessReallyEqual(headers["content-range"][0],
705 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
706 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
710 def test_GET_FILEURL_partial_range(self):
711 headers = {"range": "bytes=5-"}
712 length = len(self.BAR_CONTENTS)
713 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
714 return_response=True)
715 def _got((res, status, headers)):
716 self.failUnlessReallyEqual(int(status), 206)
717 self.failUnless(headers.has_key("content-range"))
718 self.failUnlessReallyEqual(headers["content-range"][0],
719 "bytes 5-%d/%d" % (length-1, length))
720 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
724 def test_GET_FILEURL_partial_end_range(self):
725 headers = {"range": "bytes=-5"}
726 length = len(self.BAR_CONTENTS)
727 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
728 return_response=True)
729 def _got((res, status, headers)):
730 self.failUnlessReallyEqual(int(status), 206)
731 self.failUnless(headers.has_key("content-range"))
732 self.failUnlessReallyEqual(headers["content-range"][0],
733 "bytes %d-%d/%d" % (length-5, length-1, length))
734 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
738 def test_GET_FILEURL_partial_range_overrun(self):
739 headers = {"range": "bytes=100-200"}
740 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
741 "416 Requested Range not satisfiable",
742 "First beyond end of file",
743 self.GET, self.public_url + "/foo/bar.txt",
747 def test_HEAD_FILEURL_range(self):
748 headers = {"range": "bytes=1-10"}
749 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
750 return_response=True)
751 def _got((res, status, headers)):
752 self.failUnlessReallyEqual(res, "")
753 self.failUnlessReallyEqual(int(status), 206)
754 self.failUnless(headers.has_key("content-range"))
755 self.failUnlessReallyEqual(headers["content-range"][0],
756 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
760 def test_HEAD_FILEURL_partial_range(self):
761 headers = {"range": "bytes=5-"}
762 length = len(self.BAR_CONTENTS)
763 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
764 return_response=True)
765 def _got((res, status, headers)):
766 self.failUnlessReallyEqual(int(status), 206)
767 self.failUnless(headers.has_key("content-range"))
768 self.failUnlessReallyEqual(headers["content-range"][0],
769 "bytes 5-%d/%d" % (length-1, length))
773 def test_HEAD_FILEURL_partial_end_range(self):
774 headers = {"range": "bytes=-5"}
775 length = len(self.BAR_CONTENTS)
776 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
777 return_response=True)
778 def _got((res, status, headers)):
779 self.failUnlessReallyEqual(int(status), 206)
780 self.failUnless(headers.has_key("content-range"))
781 self.failUnlessReallyEqual(headers["content-range"][0],
782 "bytes %d-%d/%d" % (length-5, length-1, length))
786 def test_HEAD_FILEURL_partial_range_overrun(self):
787 headers = {"range": "bytes=100-200"}
788 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
789 "416 Requested Range not satisfiable",
791 self.HEAD, self.public_url + "/foo/bar.txt",
795 def test_GET_FILEURL_range_bad(self):
796 headers = {"range": "BOGUS=fizbop-quarnak"}
797 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
798 return_response=True)
799 def _got((res, status, headers)):
800 self.failUnlessReallyEqual(int(status), 200)
801 self.failUnless(not headers.has_key("content-range"))
802 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
806 def test_HEAD_FILEURL(self):
807 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
808 def _got((res, status, headers)):
809 self.failUnlessReallyEqual(res, "")
810 self.failUnlessReallyEqual(headers["content-length"][0],
811 str(len(self.BAR_CONTENTS)))
812 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
816 def test_GET_FILEURL_named(self):
817 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
818 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
819 d = self.GET(base + "/@@name=/blah.txt")
820 d.addCallback(self.failUnlessIsBarDotTxt)
821 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
822 d.addCallback(self.failUnlessIsBarDotTxt)
823 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
824 d.addCallback(self.failUnlessIsBarDotTxt)
825 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
826 d.addCallback(self.failUnlessIsBarDotTxt)
827 save_url = base + "?save=true&filename=blah.txt"
828 d.addCallback(lambda res: self.GET(save_url))
829 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
830 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
831 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
832 u_url = base + "?save=true&filename=" + u_fn_e
833 d.addCallback(lambda res: self.GET(u_url))
834 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
837 def test_PUT_FILEURL_named_bad(self):
838 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
839 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
841 "/file can only be used with GET or HEAD",
842 self.PUT, base + "/@@name=/blah.txt", "")
846 def test_GET_DIRURL_named_bad(self):
847 base = "/file/%s" % urllib.quote(self._foo_uri)
848 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
851 self.GET, base + "/@@name=/blah.txt")
854 def test_GET_slash_file_bad(self):
855 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
857 "/file must be followed by a file-cap and a name",
861 def test_GET_unhandled_URI_named(self):
862 contents, n, newuri = self.makefile(12)
863 verifier_cap = n.get_verify_cap().to_string()
864 base = "/file/%s" % urllib.quote(verifier_cap)
865 # client.create_node_from_uri() can't handle verify-caps
866 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
867 "400 Bad Request", "is not a file-cap",
871 def test_GET_unhandled_URI(self):
872 contents, n, newuri = self.makefile(12)
873 verifier_cap = n.get_verify_cap().to_string()
874 base = "/uri/%s" % urllib.quote(verifier_cap)
875 # client.create_node_from_uri() can't handle verify-caps
876 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
878 "GET unknown URI type: can only do t=info",
882 def test_GET_FILE_URI(self):
883 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
885 d.addCallback(self.failUnlessIsBarDotTxt)
888 def test_GET_FILE_URI_mdmf(self):
889 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
891 d.addCallback(self.failUnlessIsQuuxDotTxt)
894 def test_GET_FILE_URI_mdmf_extensions(self):
895 base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
897 d.addCallback(self.failUnlessIsQuuxDotTxt)
900 def test_GET_FILE_URI_mdmf_bare_cap(self):
901 cap_elements = self._quux_txt_uri.split(":")
902 # 6 == expected cap length with two extensions.
903 self.failUnlessEqual(len(cap_elements), 6)
905 # Now lop off the extension parameters and stitch everything
907 quux_uri = ":".join(cap_elements[:len(cap_elements) - 2])
909 # Now GET that. We should get back quux.
910 base = "/uri/%s" % urllib.quote(quux_uri)
912 d.addCallback(self.failUnlessIsQuuxDotTxt)
915 def test_GET_FILE_URI_mdmf_readonly(self):
916 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
918 d.addCallback(self.failUnlessIsQuuxDotTxt)
921 def test_GET_FILE_URI_badchild(self):
922 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
923 errmsg = "Files have no children, certainly not named 'boguschild'"
924 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
925 "400 Bad Request", errmsg,
929 def test_PUT_FILE_URI_badchild(self):
930 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
931 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
932 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
933 "400 Bad Request", errmsg,
937 def test_PUT_FILE_URI_mdmf(self):
938 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
939 self._quux_new_contents = "new_contents"
941 d.addCallback(lambda res:
942 self.failUnlessIsQuuxDotTxt(res))
943 d.addCallback(lambda ignored:
944 self.PUT(base, self._quux_new_contents))
945 d.addCallback(lambda ignored:
947 d.addCallback(lambda res:
948 self.failUnlessReallyEqual(res, self._quux_new_contents))
951 def test_PUT_FILE_URI_mdmf_extensions(self):
952 base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
953 self._quux_new_contents = "new_contents"
955 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
956 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
957 d.addCallback(lambda ignored: self.GET(base))
958 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
962 def test_PUT_FILE_URI_mdmf_bare_cap(self):
963 elements = self._quux_txt_uri.split(":")
964 self.failUnlessEqual(len(elements), 6)
966 quux_uri = ":".join(elements[:len(elements) - 2])
967 base = "/uri/%s" % urllib.quote(quux_uri)
968 self._quux_new_contents = "new_contents" * 50000
971 d.addCallback(self.failUnlessIsQuuxDotTxt)
972 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
973 d.addCallback(lambda ignored: self.GET(base))
974 d.addCallback(lambda res:
975 self.failUnlessEqual(res, self._quux_new_contents))
978 def test_PUT_FILE_URI_mdmf_readonly(self):
979 # We're not allowed to PUT things to a readonly cap.
980 base = "/uri/%s" % self._quux_txt_readonly_uri
982 d.addCallback(lambda res:
983 self.failUnlessIsQuuxDotTxt(res))
984 # What should we get here? We get a 500 error now; that's not right.
985 d.addCallback(lambda ignored:
986 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
987 "400 Bad Request", "read-only cap",
988 self.PUT, base, "new data"))
991 def test_PUT_FILE_URI_sdmf_readonly(self):
992 # We're not allowed to put things to a readonly cap.
993 base = "/uri/%s" % self._baz_txt_readonly_uri
995 d.addCallback(lambda res:
996 self.failUnlessIsBazDotTxt(res))
997 d.addCallback(lambda ignored:
998 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
999 "400 Bad Request", "read-only cap",
1000 self.PUT, base, "new_data"))
1003 # TODO: version of this with a Unicode filename
1004 def test_GET_FILEURL_save(self):
1005 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1006 return_response=True)
1007 def _got((res, statuscode, headers)):
1008 content_disposition = headers["content-disposition"][0]
1009 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1010 self.failUnlessIsBarDotTxt(res)
1014 def test_GET_FILEURL_missing(self):
1015 d = self.GET(self.public_url + "/foo/missing")
1016 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1019 def test_GET_FILEURL_info_mdmf(self):
1020 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1022 self.failUnlessIn("mutable file (mdmf)", res)
1023 self.failUnlessIn(self._quux_txt_uri, res)
1024 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1028 def test_GET_FILEURL_info_mdmf_readonly(self):
1029 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1031 self.failUnlessIn("mutable file (mdmf)", res)
1032 self.failIfIn(self._quux_txt_uri, res)
1033 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1037 def test_GET_FILEURL_info_sdmf(self):
1038 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1040 self.failUnlessIn("mutable file (sdmf)", res)
1041 self.failUnlessIn(self._baz_txt_uri, res)
1045 def test_GET_FILEURL_info_mdmf_extensions(self):
1046 d = self.GET("/uri/%s:3:131073?t=info" % self._quux_txt_uri)
1048 self.failUnlessIn("mutable file (mdmf)", res)
1049 self.failUnlessIn(self._quux_txt_uri, res)
1050 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1054 def test_GET_FILEURL_info_mdmf_bare_cap(self):
1055 elements = self._quux_txt_uri.split(":")
1056 self.failUnlessEqual(len(elements), 6)
1058 quux_uri = ":".join(elements[:len(elements) - 2])
1059 base = "/uri/%s?t=info" % urllib.quote(quux_uri)
1062 self.failUnlessIn("mutable file (mdmf)", res)
1063 self.failUnlessIn(quux_uri, res)
1067 def test_PUT_overwrite_only_files(self):
1068 # create a directory, put a file in that directory.
1069 contents, n, filecap = self.makefile(8)
1070 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1071 d.addCallback(lambda res:
1072 self.PUT(self.public_url + "/foo/dir/file1.txt",
1073 self.NEWFILE_CONTENTS))
1074 # try to overwrite the file with replace=only-files
1075 # (this should work)
1076 d.addCallback(lambda res:
1077 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1079 d.addCallback(lambda res:
1080 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1081 "There was already a child by that name, and you asked me "
1082 "to not replace it",
1083 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1087 def test_PUT_NEWFILEURL(self):
1088 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1089 # TODO: we lose the response code, so we can't check this
1090 #self.failUnlessReallyEqual(responsecode, 201)
1091 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1092 d.addCallback(lambda res:
1093 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1094 self.NEWFILE_CONTENTS))
1097 def test_PUT_NEWFILEURL_not_mutable(self):
1098 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1099 self.NEWFILE_CONTENTS)
1100 # TODO: we lose the response code, so we can't check this
1101 #self.failUnlessReallyEqual(responsecode, 201)
1102 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1103 d.addCallback(lambda res:
1104 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1105 self.NEWFILE_CONTENTS))
1108 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1109 # this should get us a few segments of an MDMF mutable file,
1110 # which we can then test for.
1111 contents = self.NEWFILE_CONTENTS * 300000
1112 d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1114 def _got_filecap(filecap):
1115 self.failUnless(filecap.startswith("URI:MDMF"))
1117 d.addCallback(_got_filecap)
1118 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1119 d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
1122 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1123 contents = self.NEWFILE_CONTENTS * 300000
1124 d = self.PUT("/uri?mutable=true&mutable-type=sdmf",
1126 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1127 d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
1130 def test_PUT_NEWFILEURL_unlinked_bad_mutable_type(self):
1131 contents = self.NEWFILE_CONTENTS * 300000
1132 return self.shouldHTTPError("test bad mutable type",
1133 400, "Bad Request", "Unknown type: foo",
1134 self.PUT, "/uri?mutable=true&mutable-type=foo",
1137 def test_PUT_NEWFILEURL_range_bad(self):
1138 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1139 target = self.public_url + "/foo/new.txt"
1140 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1141 "501 Not Implemented",
1142 "Content-Range in PUT not yet supported",
1143 # (and certainly not for immutable files)
1144 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1146 d.addCallback(lambda res:
1147 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1150 def test_PUT_NEWFILEURL_mutable(self):
1151 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1152 self.NEWFILE_CONTENTS)
1153 # TODO: we lose the response code, so we can't check this
1154 #self.failUnlessReallyEqual(responsecode, 201)
1155 def _check_uri(res):
1156 u = uri.from_string_mutable_filenode(res)
1157 self.failUnless(u.is_mutable())
1158 self.failIf(u.is_readonly())
1160 d.addCallback(_check_uri)
1161 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1162 d.addCallback(lambda res:
1163 self.failUnlessMutableChildContentsAre(self._foo_node,
1165 self.NEWFILE_CONTENTS))
1168 def test_PUT_NEWFILEURL_mutable_toobig(self):
1169 # It is okay to upload large mutable files, so we should be able
1171 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1172 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1175 def test_PUT_NEWFILEURL_replace(self):
1176 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1177 # TODO: we lose the response code, so we can't check this
1178 #self.failUnlessReallyEqual(responsecode, 200)
1179 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1180 d.addCallback(lambda res:
1181 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1182 self.NEWFILE_CONTENTS))
1185 def test_PUT_NEWFILEURL_bad_t(self):
1186 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1187 "PUT to a file: bad t=bogus",
1188 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1192 def test_PUT_NEWFILEURL_no_replace(self):
1193 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1194 self.NEWFILE_CONTENTS)
1195 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1197 "There was already a child by that name, and you asked me "
1198 "to not replace it")
1201 def test_PUT_NEWFILEURL_mkdirs(self):
1202 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1204 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1205 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1206 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1207 d.addCallback(lambda res:
1208 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1209 self.NEWFILE_CONTENTS))
1212 def test_PUT_NEWFILEURL_blocked(self):
1213 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1214 self.NEWFILE_CONTENTS)
1215 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1217 "Unable to create directory 'blockingfile': a file was in the way")
1220 def test_PUT_NEWFILEURL_emptyname(self):
1221 # an empty pathname component (i.e. a double-slash) is disallowed
1222 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1224 "The webapi does not allow empty pathname components",
1225 self.PUT, self.public_url + "/foo//new.txt", "")
1228 def test_DELETE_FILEURL(self):
1229 d = self.DELETE(self.public_url + "/foo/bar.txt")
1230 d.addCallback(lambda res:
1231 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1234 def test_DELETE_FILEURL_missing(self):
1235 d = self.DELETE(self.public_url + "/foo/missing")
1236 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1239 def test_DELETE_FILEURL_missing2(self):
1240 d = self.DELETE(self.public_url + "/missing/missing")
1241 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1244 def failUnlessHasBarDotTxtMetadata(self, res):
1245 data = simplejson.loads(res)
1246 self.failUnless(isinstance(data, list))
1247 self.failUnlessIn("metadata", data[1])
1248 self.failUnlessIn("tahoe", data[1]["metadata"])
1249 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1250 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1251 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1252 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1254 def test_GET_FILEURL_json(self):
1255 # twisted.web.http.parse_qs ignores any query args without an '=', so
1256 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1257 # instead. This may make it tricky to emulate the S3 interface
1259 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1261 self.failUnlessIsBarJSON(data)
1262 self.failUnlessHasBarDotTxtMetadata(data)
1264 d.addCallback(_check1)
1267 def test_GET_FILEURL_json_mutable_type(self):
1268 # The JSON should include mutable-type, which says whether the
1269 # file is SDMF or MDMF
1270 d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1271 self.NEWFILE_CONTENTS * 300000)
1272 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1273 def _got_json(json, version):
1274 data = simplejson.loads(json)
1275 assert "filenode" == data[0]
1277 assert isinstance(data, dict)
1279 self.failUnlessIn("mutable-type", data)
1280 self.failUnlessEqual(data['mutable-type'], version)
1282 d.addCallback(_got_json, "mdmf")
1283 # Now make an SDMF file and check that it is reported correctly.
1284 d.addCallback(lambda ignored:
1285 self.PUT("/uri?mutable=true&mutable-type=sdmf",
1286 self.NEWFILE_CONTENTS * 300000))
1287 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1288 d.addCallback(_got_json, "sdmf")
1291 def test_GET_FILEURL_json_mdmf_extensions(self):
1292 # A GET invoked against a URL that includes an MDMF cap with
1293 # extensions should fetch the same JSON information as a GET
1294 # invoked against a bare cap.
1295 self._quux_txt_uri = "%s:3:131073" % self._quux_txt_uri
1296 self._quux_txt_readonly_uri = "%s:3:131073" % self._quux_txt_readonly_uri
1297 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1298 d.addCallback(self.failUnlessIsQuuxJSON)
1301 def test_GET_FILEURL_json_mdmf_bare_cap(self):
1302 elements = self._quux_txt_uri.split(":")
1303 self.failUnlessEqual(len(elements), 6)
1305 quux_uri = ":".join(elements[:len(elements) - 2])
1306 # so failUnlessIsQuuxJSON will work.
1307 self._quux_txt_uri = quux_uri
1309 # we need to alter the readonly URI in the same way, again so
1310 # failUnlessIsQuuxJSON will work
1311 elements = self._quux_txt_readonly_uri.split(":")
1312 self.failUnlessEqual(len(elements), 6)
1313 quux_ro_uri = ":".join(elements[:len(elements) - 2])
1314 self._quux_txt_readonly_uri = quux_ro_uri
1316 base = "/uri/%s?t=json" % urllib.quote(quux_uri)
1318 d.addCallback(self.failUnlessIsQuuxJSON)
1321 def test_GET_FILEURL_json_mdmf_bare_readonly_cap(self):
1322 elements = self._quux_txt_readonly_uri.split(":")
1323 self.failUnlessEqual(len(elements), 6)
1325 quux_readonly_uri = ":".join(elements[:len(elements) - 2])
1326 # so failUnlessIsQuuxJSON will work
1327 self._quux_txt_readonly_uri = quux_readonly_uri
1328 base = "/uri/%s?t=json" % quux_readonly_uri
1330 # XXX: We may need to make a method that knows how to check for
1331 # readonly JSON, or else alter that one so that it knows how to
1333 d.addCallback(self.failUnlessIsQuuxJSON, readonly=True)
1336 def test_GET_FILEURL_json_mdmf(self):
1337 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1338 d.addCallback(self.failUnlessIsQuuxJSON)
1341 def test_GET_FILEURL_json_missing(self):
1342 d = self.GET(self.public_url + "/foo/missing?json")
1343 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1346 def test_GET_FILEURL_uri(self):
1347 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1349 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1350 d.addCallback(_check)
1351 d.addCallback(lambda res:
1352 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1354 # for now, for files, uris and readonly-uris are the same
1355 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1356 d.addCallback(_check2)
1359 def test_GET_FILEURL_badtype(self):
1360 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1363 self.public_url + "/foo/bar.txt?t=bogus")
1366 def test_CSS_FILE(self):
1367 d = self.GET("/tahoe_css", followRedirect=True)
1369 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1370 self.failUnless(CSS_STYLE.search(res), res)
1371 d.addCallback(_check)
1374 def test_GET_FILEURL_uri_missing(self):
1375 d = self.GET(self.public_url + "/foo/missing?t=uri")
1376 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1379 def test_GET_DIRECTORY_html(self):
1380 d = self.GET(self.public_url + "/foo", followRedirect=True)
1382 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1383 # These are radio buttons that allow a user to toggle
1384 # whether a particular mutable file is SDMF or MDMF.
1385 self.failUnlessIn("mutable-type-mdmf", res)
1386 self.failUnlessIn("mutable-type-sdmf", res)
1387 # Similarly, these toggle whether a particular directory
1388 # should be MDMF or SDMF.
1389 self.failUnlessIn("mutable-directory-mdmf", res)
1390 self.failUnlessIn("mutable-directory-sdmf", res)
1391 self.failUnlessIn("quux", res)
1392 d.addCallback(_check)
1395 def test_GET_root_html(self):
1396 # make sure that we have the option to upload an unlinked
1397 # mutable file in SDMF and MDMF formats.
1399 def _got_html(html):
1400 # These are radio buttons that allow the user to toggle
1401 # whether a particular mutable file is MDMF or SDMF.
1402 self.failUnlessIn("mutable-type-mdmf", html)
1403 self.failUnlessIn("mutable-type-sdmf", html)
1404 # We should also have the ability to create a mutable directory.
1405 self.failUnlessIn("mkdir", html)
1406 # ...and we should have the ability to say whether that's an
1407 # MDMF or SDMF directory
1408 self.failUnlessIn("mutable-directory-mdmf", html)
1409 self.failUnlessIn("mutable-directory-sdmf", html)
1410 d.addCallback(_got_html)
1413 def test_mutable_type_defaults(self):
1414 # The checked="checked" attribute of the inputs corresponding to
1415 # the mutable-type parameter should change as expected with the
1416 # value configured in tahoe.cfg.
1418 # By default, the value configured with the client is
1419 # SDMF_VERSION, so that should be checked.
1420 assert self.s.mutable_file_default == SDMF_VERSION
1423 def _got_html(html, value):
1424 i = 'input checked="checked" type="radio" id="mutable-type-%s"'
1425 self.failUnlessIn(i % value, html)
1426 d.addCallback(_got_html, "sdmf")
1427 d.addCallback(lambda ignored:
1428 self.GET(self.public_url + "/foo", followRedirect=True))
1429 d.addCallback(_got_html, "sdmf")
1430 # Now switch the configuration value to MDMF. The MDMF radio
1431 # buttons should now be checked on these pages.
1432 def _swap_values(ignored):
1433 self.s.mutable_file_default = MDMF_VERSION
1434 d.addCallback(_swap_values)
1435 d.addCallback(lambda ignored: self.GET("/"))
1436 d.addCallback(_got_html, "mdmf")
1437 d.addCallback(lambda ignored:
1438 self.GET(self.public_url + "/foo", followRedirect=True))
1439 d.addCallback(_got_html, "mdmf")
1442 def test_GET_DIRURL(self):
1443 # the addSlash means we get a redirect here
1444 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1446 d = self.GET(self.public_url + "/foo", followRedirect=True)
1448 self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1450 # the FILE reference points to a URI, but it should end in bar.txt
1451 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1452 (ROOT, urllib.quote(self._bar_txt_uri)))
1453 get_bar = "".join([r'<td>FILE</td>',
1455 r'<a href="%s">bar.txt</a>' % bar_url,
1457 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1459 self.failUnless(re.search(get_bar, res), res)
1460 for label in ['unlink', 'rename']:
1461 for line in res.split("\n"):
1462 # find the line that contains the relevant button for bar.txt
1463 if ("form action" in line and
1464 ('value="%s"' % (label,)) in line and
1465 'value="bar.txt"' in line):
1466 # the form target should use a relative URL
1467 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1468 self.failUnlessIn('action="%s"' % foo_url, line)
1469 # and the when_done= should too
1470 #done_url = urllib.quote(???)
1471 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1473 # 'unlink' needs to use POST because it directly has a side effect
1474 if label == 'unlink':
1475 self.failUnlessIn('method="post"', line)
1478 self.fail("unable to find '%s bar.txt' line" % (label,), res)
1480 # the DIR reference just points to a URI
1481 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1482 get_sub = ((r'<td>DIR</td>')
1483 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1484 self.failUnless(re.search(get_sub, res), res)
1485 d.addCallback(_check)
1487 # look at a readonly directory
1488 d.addCallback(lambda res:
1489 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1491 self.failUnless("(read-only)" in res, res)
1492 self.failIf("Upload a file" in res, res)
1493 d.addCallback(_check2)
1495 # and at a directory that contains a readonly directory
1496 d.addCallback(lambda res:
1497 self.GET(self.public_url, followRedirect=True))
1499 self.failUnless(re.search('<td>DIR-RO</td>'
1500 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1501 d.addCallback(_check3)
1503 # and an empty directory
1504 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1506 self.failUnless("directory is empty" in res, res)
1507 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1508 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1509 d.addCallback(_check4)
1511 # and at a literal directory
1512 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1513 d.addCallback(lambda res:
1514 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1516 self.failUnless('(immutable)' in res, res)
1517 self.failUnless(re.search('<td>FILE</td>'
1518 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1519 d.addCallback(_check5)
1522 def test_GET_DIRURL_badtype(self):
1523 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1527 self.public_url + "/foo?t=bogus")
1530 def test_GET_DIRURL_json(self):
1531 d = self.GET(self.public_url + "/foo?t=json")
1532 d.addCallback(self.failUnlessIsFooJSON)
1535 def test_GET_DIRURL_json_mutable_type(self):
1536 d = self.PUT(self.public_url + \
1537 "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
1538 self.NEWFILE_CONTENTS * 300000)
1539 d.addCallback(lambda ignored:
1540 self.PUT(self.public_url + \
1541 "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
1542 self.NEWFILE_CONTENTS * 300000))
1543 # Now we have an MDMF and SDMF file in the directory. If we GET
1544 # its JSON, we should see their encodings.
1545 d.addCallback(lambda ignored:
1546 self.GET(self.public_url + "/foo?t=json"))
1547 def _got_json(json):
1548 data = simplejson.loads(json)
1549 assert data[0] == "dirnode"
1552 kids = data['children']
1554 mdmf_data = kids['mdmf.txt'][1]
1555 self.failUnlessIn("mutable-type", mdmf_data)
1556 self.failUnlessEqual(mdmf_data['mutable-type'], "mdmf")
1558 sdmf_data = kids['sdmf.txt'][1]
1559 self.failUnlessIn("mutable-type", sdmf_data)
1560 self.failUnlessEqual(sdmf_data['mutable-type'], "sdmf")
1561 d.addCallback(_got_json)
1565 def test_POST_DIRURL_manifest_no_ophandle(self):
1566 d = self.shouldFail2(error.Error,
1567 "test_POST_DIRURL_manifest_no_ophandle",
1569 "slow operation requires ophandle=",
1570 self.POST, self.public_url, t="start-manifest")
1573 def test_POST_DIRURL_manifest(self):
1574 d = defer.succeed(None)
1575 def getman(ignored, output):
1576 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1577 followRedirect=True)
1578 d.addCallback(self.wait_for_operation, "125")
1579 d.addCallback(self.get_operation_results, "125", output)
1581 d.addCallback(getman, None)
1582 def _got_html(manifest):
1583 self.failUnless("Manifest of SI=" in manifest)
1584 self.failUnless("<td>sub</td>" in manifest)
1585 self.failUnless(self._sub_uri in manifest)
1586 self.failUnless("<td>sub/baz.txt</td>" in manifest)
1587 d.addCallback(_got_html)
1589 # both t=status and unadorned GET should be identical
1590 d.addCallback(lambda res: self.GET("/operations/125"))
1591 d.addCallback(_got_html)
1593 d.addCallback(getman, "html")
1594 d.addCallback(_got_html)
1595 d.addCallback(getman, "text")
1596 def _got_text(manifest):
1597 self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1598 self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1599 d.addCallback(_got_text)
1600 d.addCallback(getman, "JSON")
1602 data = res["manifest"]
1604 for (path_list, cap) in data:
1605 got[tuple(path_list)] = cap
1606 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1607 self.failUnless((u"sub",u"baz.txt") in got)
1608 self.failUnless("finished" in res)
1609 self.failUnless("origin" in res)
1610 self.failUnless("storage-index" in res)
1611 self.failUnless("verifycaps" in res)
1612 self.failUnless("stats" in res)
1613 d.addCallback(_got_json)
1616 def test_POST_DIRURL_deepsize_no_ophandle(self):
1617 d = self.shouldFail2(error.Error,
1618 "test_POST_DIRURL_deepsize_no_ophandle",
1620 "slow operation requires ophandle=",
1621 self.POST, self.public_url, t="start-deep-size")
1624 def test_POST_DIRURL_deepsize(self):
1625 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1626 followRedirect=True)
1627 d.addCallback(self.wait_for_operation, "126")
1628 d.addCallback(self.get_operation_results, "126", "json")
1629 def _got_json(data):
1630 self.failUnlessReallyEqual(data["finished"], True)
1632 self.failUnless(size > 1000)
1633 d.addCallback(_got_json)
1634 d.addCallback(self.get_operation_results, "126", "text")
1636 mo = re.search(r'^size: (\d+)$', res, re.M)
1637 self.failUnless(mo, res)
1638 size = int(mo.group(1))
1639 # with directories, the size varies.
1640 self.failUnless(size > 1000)
1641 d.addCallback(_got_text)
1644 def test_POST_DIRURL_deepstats_no_ophandle(self):
1645 d = self.shouldFail2(error.Error,
1646 "test_POST_DIRURL_deepstats_no_ophandle",
1648 "slow operation requires ophandle=",
1649 self.POST, self.public_url, t="start-deep-stats")
1652 def test_POST_DIRURL_deepstats(self):
1653 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1654 followRedirect=True)
1655 d.addCallback(self.wait_for_operation, "127")
1656 d.addCallback(self.get_operation_results, "127", "json")
1657 def _got_json(stats):
1658 expected = {"count-immutable-files": 3,
1659 "count-mutable-files": 2,
1660 "count-literal-files": 0,
1662 "count-directories": 3,
1663 "size-immutable-files": 57,
1664 "size-literal-files": 0,
1665 #"size-directories": 1912, # varies
1666 #"largest-directory": 1590,
1667 "largest-directory-children": 7,
1668 "largest-immutable-file": 19,
1670 for k,v in expected.iteritems():
1671 self.failUnlessReallyEqual(stats[k], v,
1672 "stats[%s] was %s, not %s" %
1674 self.failUnlessReallyEqual(stats["size-files-histogram"],
1676 d.addCallback(_got_json)
1679 def test_POST_DIRURL_stream_manifest(self):
1680 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1682 self.failUnless(res.endswith("\n"))
1683 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1684 self.failUnlessReallyEqual(len(units), 9)
1685 self.failUnlessEqual(units[-1]["type"], "stats")
1687 self.failUnlessEqual(first["path"], [])
1688 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1689 self.failUnlessEqual(first["type"], "directory")
1690 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1691 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1692 self.failIfEqual(baz["storage-index"], None)
1693 self.failIfEqual(baz["verifycap"], None)
1694 self.failIfEqual(baz["repaircap"], None)
1695 # XXX: Add quux and baz to this test.
1697 d.addCallback(_check)
1700 def test_GET_DIRURL_uri(self):
1701 d = self.GET(self.public_url + "/foo?t=uri")
1703 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1704 d.addCallback(_check)
1707 def test_GET_DIRURL_readonly_uri(self):
1708 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1710 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1711 d.addCallback(_check)
1714 def test_PUT_NEWDIRURL(self):
1715 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1716 d.addCallback(lambda res:
1717 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1718 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1719 d.addCallback(self.failUnlessNodeKeysAre, [])
1722 def test_PUT_NEWDIRURL_mdmf(self):
1723 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1724 d.addCallback(lambda res:
1725 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1726 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1727 d.addCallback(lambda node:
1728 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1731 def test_PUT_NEWDIRURL_sdmf(self):
1732 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf",
1734 d.addCallback(lambda res:
1735 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1736 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1737 d.addCallback(lambda node:
1738 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1741 def test_PUT_NEWDIRURL_bad_mutable_type(self):
1742 return self.shouldHTTPError("test bad mutable type",
1743 400, "Bad Request", "Unknown type: foo",
1744 self.PUT, self.public_url + \
1745 "/foo/newdir=?t=mkdir&mutable-type=foo", "")
1747 def test_POST_NEWDIRURL(self):
1748 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1749 d.addCallback(lambda res:
1750 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1751 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1752 d.addCallback(self.failUnlessNodeKeysAre, [])
1755 def test_POST_NEWDIRURL_mdmf(self):
1756 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1757 d.addCallback(lambda res:
1758 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1759 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1760 d.addCallback(lambda node:
1761 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1764 def test_POST_NEWDIRURL_sdmf(self):
1765 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf", "")
1766 d.addCallback(lambda res:
1767 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1768 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1769 d.addCallback(lambda node:
1770 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1773 def test_POST_NEWDIRURL_bad_mutable_type(self):
1774 return self.shouldHTTPError("test bad mutable type",
1775 400, "Bad Request", "Unknown type: foo",
1776 self.POST2, self.public_url + \
1777 "/foo/newdir?t=mkdir&mutable-type=foo", "")
1779 def test_POST_NEWDIRURL_emptyname(self):
1780 # an empty pathname component (i.e. a double-slash) is disallowed
1781 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1783 "The webapi does not allow empty pathname components, i.e. a double slash",
1784 self.POST, self.public_url + "//?t=mkdir")
1787 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1788 (newkids, caps) = self._create_initial_children()
1789 query = "/foo/newdir?t=mkdir-with-children"
1790 if version == MDMF_VERSION:
1791 query += "&mutable-type=mdmf"
1792 elif version == SDMF_VERSION:
1793 query += "&mutable-type=sdmf"
1795 version = SDMF_VERSION # for later
1796 d = self.POST2(self.public_url + query,
1797 simplejson.dumps(newkids))
1799 n = self.s.create_node_from_uri(uri.strip())
1800 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1801 self.failUnlessEqual(n._node.get_version(), version)
1802 d2.addCallback(lambda ign:
1803 self.failUnlessROChildURIIs(n, u"child-imm",
1805 d2.addCallback(lambda ign:
1806 self.failUnlessRWChildURIIs(n, u"child-mutable",
1808 d2.addCallback(lambda ign:
1809 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1811 d2.addCallback(lambda ign:
1812 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1813 caps['unknown_rocap']))
1814 d2.addCallback(lambda ign:
1815 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1816 caps['unknown_rwcap']))
1817 d2.addCallback(lambda ign:
1818 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1819 caps['unknown_immcap']))
1820 d2.addCallback(lambda ign:
1821 self.failUnlessRWChildURIIs(n, u"dirchild",
1823 d2.addCallback(lambda ign:
1824 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1826 d2.addCallback(lambda ign:
1827 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1828 caps['emptydircap']))
1830 d.addCallback(_check)
1831 d.addCallback(lambda res:
1832 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1833 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1834 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1835 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1836 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1839 def test_POST_NEWDIRURL_initial_children(self):
1840 return self._do_POST_NEWDIRURL_initial_children_test()
1842 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1843 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1845 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1846 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1848 def test_POST_NEWDIRURL_initial_children_bad_mutable_type(self):
1849 (newkids, caps) = self._create_initial_children()
1850 return self.shouldHTTPError("test bad mutable type",
1851 400, "Bad Request", "Unknown type: foo",
1852 self.POST2, self.public_url + \
1853 "/foo/newdir?t=mkdir-with-children&mutable-type=foo",
1854 simplejson.dumps(newkids))
1856 def test_POST_NEWDIRURL_immutable(self):
1857 (newkids, caps) = self._create_immutable_children()
1858 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1859 simplejson.dumps(newkids))
1861 n = self.s.create_node_from_uri(uri.strip())
1862 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1863 d2.addCallback(lambda ign:
1864 self.failUnlessROChildURIIs(n, u"child-imm",
1866 d2.addCallback(lambda ign:
1867 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1868 caps['unknown_immcap']))
1869 d2.addCallback(lambda ign:
1870 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1872 d2.addCallback(lambda ign:
1873 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1875 d2.addCallback(lambda ign:
1876 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1877 caps['emptydircap']))
1879 d.addCallback(_check)
1880 d.addCallback(lambda res:
1881 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1882 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1883 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1884 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1885 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1886 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1887 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1888 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1889 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1890 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1891 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1892 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1893 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1894 d.addErrback(self.explain_web_error)
1897 def test_POST_NEWDIRURL_immutable_bad(self):
1898 (newkids, caps) = self._create_initial_children()
1899 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1901 "needed to be immutable but was not",
1903 self.public_url + "/foo/newdir?t=mkdir-immutable",
1904 simplejson.dumps(newkids))
1907 def test_PUT_NEWDIRURL_exists(self):
1908 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1909 d.addCallback(lambda res:
1910 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1911 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1912 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1915 def test_PUT_NEWDIRURL_blocked(self):
1916 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1917 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1919 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1920 d.addCallback(lambda res:
1921 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1922 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1923 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1926 def test_PUT_NEWDIRURL_mkdir_p(self):
1927 d = defer.succeed(None)
1928 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1929 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1930 d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1931 def mkdir_p(mkpnode):
1932 url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1934 def made_subsub(ssuri):
1935 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1936 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1938 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1940 d.addCallback(made_subsub)
1942 d.addCallback(mkdir_p)
1945 def test_PUT_NEWDIRURL_mkdirs(self):
1946 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1947 d.addCallback(lambda res:
1948 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1949 d.addCallback(lambda res:
1950 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1951 d.addCallback(lambda res:
1952 self._foo_node.get_child_at_path(u"subdir/newdir"))
1953 d.addCallback(self.failUnlessNodeKeysAre, [])
1956 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1957 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=mdmf", "")
1958 d.addCallback(lambda ignored:
1959 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1960 d.addCallback(lambda ignored:
1961 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1962 d.addCallback(lambda ignored:
1963 self._foo_node.get_child_at_path(u"subdir"))
1964 def _got_subdir(subdir):
1965 # XXX: What we want?
1966 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1967 self.failUnlessNodeHasChild(subdir, u"newdir")
1968 return subdir.get_child_at_path(u"newdir")
1969 d.addCallback(_got_subdir)
1970 d.addCallback(lambda newdir:
1971 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1974 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1975 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=sdmf", "")
1976 d.addCallback(lambda ignored:
1977 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1978 d.addCallback(lambda ignored:
1979 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1980 d.addCallback(lambda ignored:
1981 self._foo_node.get_child_at_path(u"subdir"))
1982 def _got_subdir(subdir):
1983 # XXX: What we want?
1984 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1985 self.failUnlessNodeHasChild(subdir, u"newdir")
1986 return subdir.get_child_at_path(u"newdir")
1987 d.addCallback(_got_subdir)
1988 d.addCallback(lambda newdir:
1989 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1992 def test_PUT_NEWDIRURL_mkdirs_bad_mutable_type(self):
1993 return self.shouldHTTPError("test bad mutable type",
1994 400, "Bad Request", "Unknown type: foo",
1995 self.PUT, self.public_url + \
1996 "/foo/subdir/newdir?t=mkdir&mutable-type=foo",
1999 def test_DELETE_DIRURL(self):
2000 d = self.DELETE(self.public_url + "/foo")
2001 d.addCallback(lambda res:
2002 self.failIfNodeHasChild(self.public_root, u"foo"))
2005 def test_DELETE_DIRURL_missing(self):
2006 d = self.DELETE(self.public_url + "/foo/missing")
2007 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2008 d.addCallback(lambda res:
2009 self.failUnlessNodeHasChild(self.public_root, u"foo"))
2012 def test_DELETE_DIRURL_missing2(self):
2013 d = self.DELETE(self.public_url + "/missing")
2014 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2017 def dump_root(self):
2019 w = webish.DirnodeWalkerMixin()
2020 def visitor(childpath, childnode, metadata):
2022 d = w.walk(self.public_root, visitor)
2025 def failUnlessNodeKeysAre(self, node, expected_keys):
2026 for k in expected_keys:
2027 assert isinstance(k, unicode)
2029 def _check(children):
2030 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2031 d.addCallback(_check)
2033 def failUnlessNodeHasChild(self, node, name):
2034 assert isinstance(name, unicode)
2036 def _check(children):
2037 self.failUnless(name in children)
2038 d.addCallback(_check)
2040 def failIfNodeHasChild(self, node, name):
2041 assert isinstance(name, unicode)
2043 def _check(children):
2044 self.failIf(name in children)
2045 d.addCallback(_check)
2048 def failUnlessChildContentsAre(self, node, name, expected_contents):
2049 assert isinstance(name, unicode)
2050 d = node.get_child_at_path(name)
2051 d.addCallback(lambda node: download_to_data(node))
2052 def _check(contents):
2053 self.failUnlessReallyEqual(contents, expected_contents)
2054 d.addCallback(_check)
2057 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2058 assert isinstance(name, unicode)
2059 d = node.get_child_at_path(name)
2060 d.addCallback(lambda node: node.download_best_version())
2061 def _check(contents):
2062 self.failUnlessReallyEqual(contents, expected_contents)
2063 d.addCallback(_check)
2066 def failUnlessRWChildURIIs(self, node, name, expected_uri):
2067 assert isinstance(name, unicode)
2068 d = node.get_child_at_path(name)
2070 self.failUnless(child.is_unknown() or not child.is_readonly())
2071 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2072 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2073 expected_ro_uri = self._make_readonly(expected_uri)
2075 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2076 d.addCallback(_check)
2079 def failUnlessROChildURIIs(self, node, name, expected_uri):
2080 assert isinstance(name, unicode)
2081 d = node.get_child_at_path(name)
2083 self.failUnless(child.is_unknown() or child.is_readonly())
2084 self.failUnlessReallyEqual(child.get_write_uri(), None)
2085 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2086 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2087 d.addCallback(_check)
2090 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2091 assert isinstance(name, unicode)
2092 d = node.get_child_at_path(name)
2094 self.failUnless(child.is_unknown() or not child.is_readonly())
2095 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2096 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2097 expected_ro_uri = self._make_readonly(got_uri)
2099 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2100 d.addCallback(_check)
2103 def failUnlessURIMatchesROChild(self, got_uri, node, name):
2104 assert isinstance(name, unicode)
2105 d = node.get_child_at_path(name)
2107 self.failUnless(child.is_unknown() or child.is_readonly())
2108 self.failUnlessReallyEqual(child.get_write_uri(), None)
2109 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2110 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2111 d.addCallback(_check)
2114 def failUnlessCHKURIHasContents(self, got_uri, contents):
2115 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2117 def test_POST_upload(self):
2118 d = self.POST(self.public_url + "/foo", t="upload",
2119 file=("new.txt", self.NEWFILE_CONTENTS))
2121 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2122 d.addCallback(lambda res:
2123 self.failUnlessChildContentsAre(fn, u"new.txt",
2124 self.NEWFILE_CONTENTS))
2127 def test_POST_upload_unicode(self):
2128 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2129 d = self.POST(self.public_url + "/foo", t="upload",
2130 file=(filename, self.NEWFILE_CONTENTS))
2132 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2133 d.addCallback(lambda res:
2134 self.failUnlessChildContentsAre(fn, filename,
2135 self.NEWFILE_CONTENTS))
2136 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2137 d.addCallback(lambda res: self.GET(target_url))
2138 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2139 self.NEWFILE_CONTENTS,
2143 def test_POST_upload_unicode_named(self):
2144 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2145 d = self.POST(self.public_url + "/foo", t="upload",
2147 file=("overridden", self.NEWFILE_CONTENTS))
2149 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2150 d.addCallback(lambda res:
2151 self.failUnlessChildContentsAre(fn, filename,
2152 self.NEWFILE_CONTENTS))
2153 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2154 d.addCallback(lambda res: self.GET(target_url))
2155 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2156 self.NEWFILE_CONTENTS,
2160 def test_POST_upload_no_link(self):
2161 d = self.POST("/uri", t="upload",
2162 file=("new.txt", self.NEWFILE_CONTENTS))
2163 def _check_upload_results(page):
2164 # this should be a page which describes the results of the upload
2165 # that just finished.
2166 self.failUnless("Upload Results:" in page)
2167 self.failUnless("URI:" in page)
2168 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2169 mo = uri_re.search(page)
2170 self.failUnless(mo, page)
2171 new_uri = mo.group(1)
2173 d.addCallback(_check_upload_results)
2174 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2177 def test_POST_upload_no_link_whendone(self):
2178 d = self.POST("/uri", t="upload", when_done="/",
2179 file=("new.txt", self.NEWFILE_CONTENTS))
2180 d.addBoth(self.shouldRedirect, "/")
2183 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2184 d = defer.maybeDeferred(callable, *args, **kwargs)
2186 if isinstance(res, failure.Failure):
2187 res.trap(error.PageRedirect)
2188 statuscode = res.value.status
2189 target = res.value.location
2190 return checker(statuscode, target)
2191 self.fail("%s: callable was supposed to redirect, not return '%s'"
2196 def test_POST_upload_no_link_whendone_results(self):
2197 def check(statuscode, target):
2198 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2199 self.failUnless(target.startswith(self.webish_url), target)
2200 return client.getPage(target, method="GET")
2201 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2203 self.POST, "/uri", t="upload",
2204 when_done="/uri/%(uri)s",
2205 file=("new.txt", self.NEWFILE_CONTENTS))
2206 d.addCallback(lambda res:
2207 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2210 def test_POST_upload_no_link_mutable(self):
2211 d = self.POST("/uri", t="upload", mutable="true",
2212 file=("new.txt", self.NEWFILE_CONTENTS))
2213 def _check(filecap):
2214 filecap = filecap.strip()
2215 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2216 self.filecap = filecap
2217 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2218 self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2219 n = self.s.create_node_from_uri(filecap)
2220 return n.download_best_version()
2221 d.addCallback(_check)
2223 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2224 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2225 d.addCallback(_check2)
2227 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2228 return self.GET("/file/%s" % urllib.quote(self.filecap))
2229 d.addCallback(_check3)
2231 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2232 d.addCallback(_check4)
2235 def test_POST_upload_no_link_mutable_toobig(self):
2236 # The SDMF size limit is no longer in place, so we should be
2237 # able to upload mutable files that are as large as we want them
2239 d = self.POST("/uri", t="upload", mutable="true",
2240 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2244 def test_POST_upload_mutable_type_unlinked(self):
2245 d = self.POST("/uri?t=upload&mutable=true&mutable-type=sdmf",
2246 file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2247 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2248 def _got_json(json, version):
2249 data = simplejson.loads(json)
2252 self.failUnlessIn("mutable-type", data)
2253 self.failUnlessEqual(data['mutable-type'], version)
2254 d.addCallback(_got_json, "sdmf")
2255 d.addCallback(lambda ignored:
2256 self.POST("/uri?t=upload&mutable=true&mutable-type=mdmf",
2257 file=('mdmf.txt', self.NEWFILE_CONTENTS * 300000)))
2258 def _got_filecap(filecap):
2259 self.failUnless(filecap.startswith("URI:MDMF"))
2261 d.addCallback(_got_filecap)
2262 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2263 d.addCallback(_got_json, "mdmf")
2266 def test_POST_upload_mutable_type_unlinked_bad_mutable_type(self):
2267 return self.shouldHTTPError("test bad mutable type",
2268 400, "Bad Request", "Unknown type: foo",
2270 "/uri?5=upload&mutable=true&mutable-type=foo",
2271 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2273 def test_POST_upload_mutable_type(self):
2274 d = self.POST(self.public_url + \
2275 "/foo?t=upload&mutable=true&mutable-type=sdmf",
2276 file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2278 def _got_cap(filecap, filename):
2279 filenameu = unicode(filename)
2280 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2281 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2282 def _got_mdmf_cap(filecap):
2283 self.failUnless(filecap.startswith("URI:MDMF"))
2285 d.addCallback(_got_cap, "sdmf.txt")
2286 def _got_json(json, version):
2287 data = simplejson.loads(json)
2290 self.failUnlessIn("mutable-type", data)
2291 self.failUnlessEqual(data['mutable-type'], version)
2292 d.addCallback(_got_json, "sdmf")
2293 d.addCallback(lambda ignored:
2294 self.POST(self.public_url + \
2295 "/foo?t=upload&mutable=true&mutable-type=mdmf",
2296 file=("mdmf.txt", self.NEWFILE_CONTENTS * 300000)))
2297 d.addCallback(_got_mdmf_cap)
2298 d.addCallback(_got_cap, "mdmf.txt")
2299 d.addCallback(_got_json, "mdmf")
2302 def test_POST_upload_bad_mutable_type(self):
2303 return self.shouldHTTPError("test bad mutable type",
2304 400, "Bad Request", "Unknown type: foo",
2305 self.POST, self.public_url + \
2306 "/foo?t=upload&mutable=true&mutable-type=foo",
2307 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2309 def test_POST_upload_mutable(self):
2310 # this creates a mutable file
2311 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2312 file=("new.txt", self.NEWFILE_CONTENTS))
2314 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2315 d.addCallback(lambda res:
2316 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2317 self.NEWFILE_CONTENTS))
2318 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2320 self.failUnless(IMutableFileNode.providedBy(newnode))
2321 self.failUnless(newnode.is_mutable())
2322 self.failIf(newnode.is_readonly())
2323 self._mutable_node = newnode
2324 self._mutable_uri = newnode.get_uri()
2327 # now upload it again and make sure that the URI doesn't change
2328 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2329 d.addCallback(lambda res:
2330 self.POST(self.public_url + "/foo", t="upload",
2332 file=("new.txt", NEWER_CONTENTS)))
2333 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2334 d.addCallback(lambda res:
2335 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2337 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2339 self.failUnless(IMutableFileNode.providedBy(newnode))
2340 self.failUnless(newnode.is_mutable())
2341 self.failIf(newnode.is_readonly())
2342 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2343 d.addCallback(_got2)
2345 # upload a second time, using PUT instead of POST
2346 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2347 d.addCallback(lambda res:
2348 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2349 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2350 d.addCallback(lambda res:
2351 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2354 # finally list the directory, since mutable files are displayed
2355 # slightly differently
2357 d.addCallback(lambda res:
2358 self.GET(self.public_url + "/foo/",
2359 followRedirect=True))
2360 def _check_page(res):
2361 # TODO: assert more about the contents
2362 self.failUnless("SSK" in res)
2364 d.addCallback(_check_page)
2366 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2368 self.failUnless(IMutableFileNode.providedBy(newnode))
2369 self.failUnless(newnode.is_mutable())
2370 self.failIf(newnode.is_readonly())
2371 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2372 d.addCallback(_got3)
2374 # look at the JSON form of the enclosing directory
2375 d.addCallback(lambda res:
2376 self.GET(self.public_url + "/foo/?t=json",
2377 followRedirect=True))
2378 def _check_page_json(res):
2379 parsed = simplejson.loads(res)
2380 self.failUnlessEqual(parsed[0], "dirnode")
2381 children = dict( [(unicode(name),value)
2383 in parsed[1]["children"].iteritems()] )
2384 self.failUnless(u"new.txt" in children)
2385 new_json = children[u"new.txt"]
2386 self.failUnlessEqual(new_json[0], "filenode")
2387 self.failUnless(new_json[1]["mutable"])
2388 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2389 ro_uri = self._mutable_node.get_readonly().to_string()
2390 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2391 d.addCallback(_check_page_json)
2393 # and the JSON form of the file
2394 d.addCallback(lambda res:
2395 self.GET(self.public_url + "/foo/new.txt?t=json"))
2396 def _check_file_json(res):
2397 parsed = simplejson.loads(res)
2398 self.failUnlessEqual(parsed[0], "filenode")
2399 self.failUnless(parsed[1]["mutable"])
2400 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2401 ro_uri = self._mutable_node.get_readonly().to_string()
2402 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2403 d.addCallback(_check_file_json)
2405 # and look at t=uri and t=readonly-uri
2406 d.addCallback(lambda res:
2407 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2408 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2409 d.addCallback(lambda res:
2410 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2411 def _check_ro_uri(res):
2412 ro_uri = self._mutable_node.get_readonly().to_string()
2413 self.failUnlessReallyEqual(res, ro_uri)
2414 d.addCallback(_check_ro_uri)
2416 # make sure we can get to it from /uri/URI
2417 d.addCallback(lambda res:
2418 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2419 d.addCallback(lambda res:
2420 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2422 # and that HEAD computes the size correctly
2423 d.addCallback(lambda res:
2424 self.HEAD(self.public_url + "/foo/new.txt",
2425 return_response=True))
2426 def _got_headers((res, status, headers)):
2427 self.failUnlessReallyEqual(res, "")
2428 self.failUnlessReallyEqual(headers["content-length"][0],
2429 str(len(NEW2_CONTENTS)))
2430 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2431 d.addCallback(_got_headers)
2433 # make sure that outdated size limits aren't enforced anymore.
2434 d.addCallback(lambda ignored:
2435 self.POST(self.public_url + "/foo", t="upload",
2438 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2439 d.addErrback(self.dump_error)
2442 def test_POST_upload_mutable_toobig(self):
2443 # SDMF had a size limti that was removed a while ago. MDMF has
2444 # never had a size limit. Test to make sure that we do not
2445 # encounter errors when trying to upload large mutable files,
2446 # since there should be no coded prohibitions regarding large
2448 d = self.POST(self.public_url + "/foo",
2449 t="upload", mutable="true",
2450 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2453 def dump_error(self, f):
2454 # if the web server returns an error code (like 400 Bad Request),
2455 # web.client.getPage puts the HTTP response body into the .response
2456 # attribute of the exception object that it gives back. It does not
2457 # appear in the Failure's repr(), so the ERROR that trial displays
2458 # will be rather terse and unhelpful. addErrback this method to the
2459 # end of your chain to get more information out of these errors.
2460 if f.check(error.Error):
2461 print "web.error.Error:"
2463 print f.value.response
2466 def test_POST_upload_replace(self):
2467 d = self.POST(self.public_url + "/foo", t="upload",
2468 file=("bar.txt", self.NEWFILE_CONTENTS))
2470 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2471 d.addCallback(lambda res:
2472 self.failUnlessChildContentsAre(fn, u"bar.txt",
2473 self.NEWFILE_CONTENTS))
2476 def test_POST_upload_no_replace_ok(self):
2477 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2478 file=("new.txt", self.NEWFILE_CONTENTS))
2479 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2480 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2481 self.NEWFILE_CONTENTS))
2484 def test_POST_upload_no_replace_queryarg(self):
2485 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2486 file=("bar.txt", self.NEWFILE_CONTENTS))
2487 d.addBoth(self.shouldFail, error.Error,
2488 "POST_upload_no_replace_queryarg",
2490 "There was already a child by that name, and you asked me "
2491 "to not replace it")
2492 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2493 d.addCallback(self.failUnlessIsBarDotTxt)
2496 def test_POST_upload_no_replace_field(self):
2497 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2498 file=("bar.txt", self.NEWFILE_CONTENTS))
2499 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2501 "There was already a child by that name, and you asked me "
2502 "to not replace it")
2503 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2504 d.addCallback(self.failUnlessIsBarDotTxt)
2507 def test_POST_upload_whendone(self):
2508 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2509 file=("new.txt", self.NEWFILE_CONTENTS))
2510 d.addBoth(self.shouldRedirect, "/THERE")
2512 d.addCallback(lambda res:
2513 self.failUnlessChildContentsAre(fn, u"new.txt",
2514 self.NEWFILE_CONTENTS))
2517 def test_POST_upload_named(self):
2519 d = self.POST(self.public_url + "/foo", t="upload",
2520 name="new.txt", file=self.NEWFILE_CONTENTS)
2521 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2522 d.addCallback(lambda res:
2523 self.failUnlessChildContentsAre(fn, u"new.txt",
2524 self.NEWFILE_CONTENTS))
2527 def test_POST_upload_named_badfilename(self):
2528 d = self.POST(self.public_url + "/foo", t="upload",
2529 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2530 d.addBoth(self.shouldFail, error.Error,
2531 "test_POST_upload_named_badfilename",
2533 "name= may not contain a slash",
2535 # make sure that nothing was added
2536 d.addCallback(lambda res:
2537 self.failUnlessNodeKeysAre(self._foo_node,
2538 [u"bar.txt", u"baz.txt", u"blockingfile",
2539 u"empty", u"n\u00fc.txt", u"quux.txt",
2543 def test_POST_FILEURL_check(self):
2544 bar_url = self.public_url + "/foo/bar.txt"
2545 d = self.POST(bar_url, t="check")
2547 self.failUnless("Healthy :" in res)
2548 d.addCallback(_check)
2549 redir_url = "http://allmydata.org/TARGET"
2550 def _check2(statuscode, target):
2551 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2552 self.failUnlessReallyEqual(target, redir_url)
2553 d.addCallback(lambda res:
2554 self.shouldRedirect2("test_POST_FILEURL_check",
2558 when_done=redir_url))
2559 d.addCallback(lambda res:
2560 self.POST(bar_url, t="check", return_to=redir_url))
2562 self.failUnless("Healthy :" in res)
2563 self.failUnless("Return to file" in res)
2564 self.failUnless(redir_url in res)
2565 d.addCallback(_check3)
2567 d.addCallback(lambda res:
2568 self.POST(bar_url, t="check", output="JSON"))
2569 def _check_json(res):
2570 data = simplejson.loads(res)
2571 self.failUnless("storage-index" in data)
2572 self.failUnless(data["results"]["healthy"])
2573 d.addCallback(_check_json)
2577 def test_POST_FILEURL_check_and_repair(self):
2578 bar_url = self.public_url + "/foo/bar.txt"
2579 d = self.POST(bar_url, t="check", repair="true")
2581 self.failUnless("Healthy :" in res)
2582 d.addCallback(_check)
2583 redir_url = "http://allmydata.org/TARGET"
2584 def _check2(statuscode, target):
2585 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2586 self.failUnlessReallyEqual(target, redir_url)
2587 d.addCallback(lambda res:
2588 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2591 t="check", repair="true",
2592 when_done=redir_url))
2593 d.addCallback(lambda res:
2594 self.POST(bar_url, t="check", return_to=redir_url))
2596 self.failUnless("Healthy :" in res)
2597 self.failUnless("Return to file" in res)
2598 self.failUnless(redir_url in res)
2599 d.addCallback(_check3)
2602 def test_POST_DIRURL_check(self):
2603 foo_url = self.public_url + "/foo/"
2604 d = self.POST(foo_url, t="check")
2606 self.failUnless("Healthy :" in res, res)
2607 d.addCallback(_check)
2608 redir_url = "http://allmydata.org/TARGET"
2609 def _check2(statuscode, target):
2610 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2611 self.failUnlessReallyEqual(target, redir_url)
2612 d.addCallback(lambda res:
2613 self.shouldRedirect2("test_POST_DIRURL_check",
2617 when_done=redir_url))
2618 d.addCallback(lambda res:
2619 self.POST(foo_url, t="check", return_to=redir_url))
2621 self.failUnless("Healthy :" in res, res)
2622 self.failUnless("Return to file/directory" in res)
2623 self.failUnless(redir_url in res)
2624 d.addCallback(_check3)
2626 d.addCallback(lambda res:
2627 self.POST(foo_url, t="check", output="JSON"))
2628 def _check_json(res):
2629 data = simplejson.loads(res)
2630 self.failUnless("storage-index" in data)
2631 self.failUnless(data["results"]["healthy"])
2632 d.addCallback(_check_json)
2636 def test_POST_DIRURL_check_and_repair(self):
2637 foo_url = self.public_url + "/foo/"
2638 d = self.POST(foo_url, t="check", repair="true")
2640 self.failUnless("Healthy :" in res, res)
2641 d.addCallback(_check)
2642 redir_url = "http://allmydata.org/TARGET"
2643 def _check2(statuscode, target):
2644 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2645 self.failUnlessReallyEqual(target, redir_url)
2646 d.addCallback(lambda res:
2647 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2650 t="check", repair="true",
2651 when_done=redir_url))
2652 d.addCallback(lambda res:
2653 self.POST(foo_url, t="check", return_to=redir_url))
2655 self.failUnless("Healthy :" in res)
2656 self.failUnless("Return to file/directory" in res)
2657 self.failUnless(redir_url in res)
2658 d.addCallback(_check3)
2661 def test_POST_FILEURL_mdmf_check(self):
2662 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2663 d = self.POST(quux_url, t="check")
2665 self.failUnlessIn("Healthy", res)
2666 d.addCallback(_check)
2667 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2668 d.addCallback(lambda ignored:
2669 self.POST(quux_extension_url, t="check"))
2670 d.addCallback(_check)
2673 def test_POST_FILEURL_mdmf_check_and_repair(self):
2674 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2675 d = self.POST(quux_url, t="check", repair="true")
2677 self.failUnlessIn("Healthy", res)
2678 d.addCallback(_check)
2679 quux_extension_url = "/uri/%s" %\
2680 urllib.quote("%s:3:131073" % self._quux_txt_uri)
2681 d.addCallback(lambda ignored:
2682 self.POST(quux_extension_url, t="check", repair="true"))
2683 d.addCallback(_check)
2686 def wait_for_operation(self, ignored, ophandle):
2687 url = "/operations/" + ophandle
2688 url += "?t=status&output=JSON"
2691 data = simplejson.loads(res)
2692 if not data["finished"]:
2693 d = self.stall(delay=1.0)
2694 d.addCallback(self.wait_for_operation, ophandle)
2700 def get_operation_results(self, ignored, ophandle, output=None):
2701 url = "/operations/" + ophandle
2704 url += "&output=" + output
2707 if output and output.lower() == "json":
2708 return simplejson.loads(res)
2713 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2714 d = self.shouldFail2(error.Error,
2715 "test_POST_DIRURL_deepcheck_no_ophandle",
2717 "slow operation requires ophandle=",
2718 self.POST, self.public_url, t="start-deep-check")
2721 def test_POST_DIRURL_deepcheck(self):
2722 def _check_redirect(statuscode, target):
2723 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2724 self.failUnless(target.endswith("/operations/123"))
2725 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2726 self.POST, self.public_url,
2727 t="start-deep-check", ophandle="123")
2728 d.addCallback(self.wait_for_operation, "123")
2729 def _check_json(data):
2730 self.failUnlessReallyEqual(data["finished"], True)
2731 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2732 self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2733 d.addCallback(_check_json)
2734 d.addCallback(self.get_operation_results, "123", "html")
2735 def _check_html(res):
2736 self.failUnless("Objects Checked: <span>10</span>" in res)
2737 self.failUnless("Objects Healthy: <span>10</span>" in res)
2738 d.addCallback(_check_html)
2740 d.addCallback(lambda res:
2741 self.GET("/operations/123/"))
2742 d.addCallback(_check_html) # should be the same as without the slash
2744 d.addCallback(lambda res:
2745 self.shouldFail2(error.Error, "one", "404 Not Found",
2746 "No detailed results for SI bogus",
2747 self.GET, "/operations/123/bogus"))
2749 foo_si = self._foo_node.get_storage_index()
2750 foo_si_s = base32.b2a(foo_si)
2751 d.addCallback(lambda res:
2752 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2753 def _check_foo_json(res):
2754 data = simplejson.loads(res)
2755 self.failUnlessEqual(data["storage-index"], foo_si_s)
2756 self.failUnless(data["results"]["healthy"])
2757 d.addCallback(_check_foo_json)
2760 def test_POST_DIRURL_deepcheck_and_repair(self):
2761 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2762 ophandle="124", output="json", followRedirect=True)
2763 d.addCallback(self.wait_for_operation, "124")
2764 def _check_json(data):
2765 self.failUnlessReallyEqual(data["finished"], True)
2766 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2767 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2768 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2769 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2770 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2771 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2772 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2773 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2774 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2775 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2776 d.addCallback(_check_json)
2777 d.addCallback(self.get_operation_results, "124", "html")
2778 def _check_html(res):
2779 self.failUnless("Objects Checked: <span>10</span>" in res)
2781 self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2782 self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2783 self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2785 self.failUnless("Repairs Attempted: <span>0</span>" in res)
2786 self.failUnless("Repairs Successful: <span>0</span>" in res)
2787 self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2789 self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2790 self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2791 self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2792 d.addCallback(_check_html)
2795 def test_POST_FILEURL_bad_t(self):
2796 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2797 "POST to file: bad t=bogus",
2798 self.POST, self.public_url + "/foo/bar.txt",
2802 def test_POST_mkdir(self): # return value?
2803 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2804 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2805 d.addCallback(self.failUnlessNodeKeysAre, [])
2808 def test_POST_mkdir_mdmf(self):
2809 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=mdmf")
2810 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2811 d.addCallback(lambda node:
2812 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2815 def test_POST_mkdir_sdmf(self):
2816 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=sdmf")
2817 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2818 d.addCallback(lambda node:
2819 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2822 def test_POST_mkdir_bad_mutable_type(self):
2823 return self.shouldHTTPError("test bad mutable type",
2824 400, "Bad Request", "Unknown type: foo",
2825 self.POST, self.public_url + \
2826 "/foo?t=mkdir&name=newdir&mutable-type=foo")
2828 def test_POST_mkdir_initial_children(self):
2829 (newkids, caps) = self._create_initial_children()
2830 d = self.POST2(self.public_url +
2831 "/foo?t=mkdir-with-children&name=newdir",
2832 simplejson.dumps(newkids))
2833 d.addCallback(lambda res:
2834 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2835 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2836 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2837 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2838 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2841 def test_POST_mkdir_initial_children_mdmf(self):
2842 (newkids, caps) = self._create_initial_children()
2843 d = self.POST2(self.public_url +
2844 "/foo?t=mkdir-with-children&name=newdir&mutable-type=mdmf",
2845 simplejson.dumps(newkids))
2846 d.addCallback(lambda res:
2847 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2848 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2849 d.addCallback(lambda node:
2850 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2851 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2852 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2857 def test_POST_mkdir_initial_children_sdmf(self):
2858 (newkids, caps) = self._create_initial_children()
2859 d = self.POST2(self.public_url +
2860 "/foo?t=mkdir-with-children&name=newdir&mutable-type=sdmf",
2861 simplejson.dumps(newkids))
2862 d.addCallback(lambda res:
2863 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2864 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2865 d.addCallback(lambda node:
2866 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2867 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2868 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2872 def test_POST_mkdir_initial_children_bad_mutable_type(self):
2873 (newkids, caps) = self._create_initial_children()
2874 return self.shouldHTTPError("test bad mutable type",
2875 400, "Bad Request", "Unknown type: foo",
2876 self.POST, self.public_url + \
2877 "/foo?t=mkdir-with-children&name=newdir&mutable-type=foo",
2878 simplejson.dumps(newkids))
2880 def test_POST_mkdir_immutable(self):
2881 (newkids, caps) = self._create_immutable_children()
2882 d = self.POST2(self.public_url +
2883 "/foo?t=mkdir-immutable&name=newdir",
2884 simplejson.dumps(newkids))
2885 d.addCallback(lambda res:
2886 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2887 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2888 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2889 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2890 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2891 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2892 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2893 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2894 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2895 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2896 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2897 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2898 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2901 def test_POST_mkdir_immutable_bad(self):
2902 (newkids, caps) = self._create_initial_children()
2903 d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2905 "needed to be immutable but was not",
2908 "/foo?t=mkdir-immutable&name=newdir",
2909 simplejson.dumps(newkids))
2912 def test_POST_mkdir_2(self):
2913 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2914 d.addCallback(lambda res:
2915 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2916 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917 d.addCallback(self.failUnlessNodeKeysAre, [])
2920 def test_POST_mkdirs_2(self):
2921 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2922 d.addCallback(lambda res:
2923 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2924 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2925 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2926 d.addCallback(self.failUnlessNodeKeysAre, [])
2929 def test_POST_mkdir_no_parentdir_noredirect(self):
2930 d = self.POST("/uri?t=mkdir")
2931 def _after_mkdir(res):
2932 uri.DirectoryURI.init_from_string(res)
2933 d.addCallback(_after_mkdir)
2936 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2937 d = self.POST("/uri?t=mkdir&mutable-type=mdmf")
2938 def _after_mkdir(res):
2939 u = uri.from_string(res)
2940 # Check that this is an MDMF writecap
2941 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2942 d.addCallback(_after_mkdir)
2945 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2946 d = self.POST("/uri?t=mkdir&mutable-type=sdmf")
2947 def _after_mkdir(res):
2948 u = uri.from_string(res)
2949 self.failUnlessIsInstance(u, uri.DirectoryURI)
2950 d.addCallback(_after_mkdir)
2953 def test_POST_mkdir_no_parentdir_noredirect_bad_mutable_type(self):
2954 return self.shouldHTTPError("test bad mutable type",
2955 400, "Bad Request", "Unknown type: foo",
2956 self.POST, self.public_url + \
2957 "/uri?t=mkdir&mutable-type=foo")
2959 def test_POST_mkdir_no_parentdir_noredirect2(self):
2960 # make sure form-based arguments (as on the welcome page) still work
2961 d = self.POST("/uri", t="mkdir")
2962 def _after_mkdir(res):
2963 uri.DirectoryURI.init_from_string(res)
2964 d.addCallback(_after_mkdir)
2965 d.addErrback(self.explain_web_error)
2968 def test_POST_mkdir_no_parentdir_redirect(self):
2969 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2970 d.addBoth(self.shouldRedirect, None, statuscode='303')
2971 def _check_target(target):
2972 target = urllib.unquote(target)
2973 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2974 d.addCallback(_check_target)
2977 def test_POST_mkdir_no_parentdir_redirect2(self):
2978 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2979 d.addBoth(self.shouldRedirect, None, statuscode='303')
2980 def _check_target(target):
2981 target = urllib.unquote(target)
2982 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2983 d.addCallback(_check_target)
2984 d.addErrback(self.explain_web_error)
2987 def _make_readonly(self, u):
2988 ro_uri = uri.from_string(u).get_readonly()
2991 return ro_uri.to_string()
2993 def _create_initial_children(self):
2994 contents, n, filecap1 = self.makefile(12)
2995 md1 = {"metakey1": "metavalue1"}
2996 filecap2 = make_mutable_file_uri()
2997 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2998 filecap3 = node3.get_readonly_uri()
2999 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3000 dircap = DirectoryNode(node4, None, None).get_uri()
3001 mdmfcap = make_mutable_file_uri(mdmf=True)
3002 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3003 emptydircap = "URI:DIR2-LIT:"
3004 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
3005 "ro_uri": self._make_readonly(filecap1),
3006 "metadata": md1, }],
3007 u"child-mutable": ["filenode", {"rw_uri": filecap2,
3008 "ro_uri": self._make_readonly(filecap2)}],
3009 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3010 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
3011 "ro_uri": unknown_rocap}],
3012 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
3013 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3014 u"dirchild": ["dirnode", {"rw_uri": dircap,
3015 "ro_uri": self._make_readonly(dircap)}],
3016 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3017 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3018 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3019 "ro_uri": self._make_readonly(mdmfcap)}],
3021 return newkids, {'filecap1': filecap1,
3022 'filecap2': filecap2,
3023 'filecap3': filecap3,
3024 'unknown_rwcap': unknown_rwcap,
3025 'unknown_rocap': unknown_rocap,
3026 'unknown_immcap': unknown_immcap,
3028 'litdircap': litdircap,
3029 'emptydircap': emptydircap,
3032 def _create_immutable_children(self):
3033 contents, n, filecap1 = self.makefile(12)
3034 md1 = {"metakey1": "metavalue1"}
3035 tnode = create_chk_filenode("immutable directory contents\n"*10)
3036 dnode = DirectoryNode(tnode, None, None)
3037 assert not dnode.is_mutable()
3038 immdircap = dnode.get_uri()
3039 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3040 emptydircap = "URI:DIR2-LIT:"
3041 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
3042 "metadata": md1, }],
3043 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3044 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
3045 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3046 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3048 return newkids, {'filecap1': filecap1,
3049 'unknown_immcap': unknown_immcap,
3050 'immdircap': immdircap,
3051 'litdircap': litdircap,
3052 'emptydircap': emptydircap}
3054 def test_POST_mkdir_no_parentdir_initial_children(self):
3055 (newkids, caps) = self._create_initial_children()
3056 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3057 def _after_mkdir(res):
3058 self.failUnless(res.startswith("URI:DIR"), res)
3059 n = self.s.create_node_from_uri(res)
3060 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3061 d2.addCallback(lambda ign:
3062 self.failUnlessROChildURIIs(n, u"child-imm",
3064 d2.addCallback(lambda ign:
3065 self.failUnlessRWChildURIIs(n, u"child-mutable",
3067 d2.addCallback(lambda ign:
3068 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3070 d2.addCallback(lambda ign:
3071 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3072 caps['unknown_rwcap']))
3073 d2.addCallback(lambda ign:
3074 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3075 caps['unknown_rocap']))
3076 d2.addCallback(lambda ign:
3077 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3078 caps['unknown_immcap']))
3079 d2.addCallback(lambda ign:
3080 self.failUnlessRWChildURIIs(n, u"dirchild",
3083 d.addCallback(_after_mkdir)
3086 def test_POST_mkdir_no_parentdir_unexpected_children(self):
3087 # the regular /uri?t=mkdir operation is specified to ignore its body.
3088 # Only t=mkdir-with-children pays attention to it.
3089 (newkids, caps) = self._create_initial_children()
3090 d = self.shouldHTTPError("POST t=mkdir unexpected children",
3092 "t=mkdir does not accept children=, "
3093 "try t=mkdir-with-children instead",
3094 self.POST2, "/uri?t=mkdir", # without children
3095 simplejson.dumps(newkids))
3098 def test_POST_noparent_bad(self):
3099 d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
3100 "/uri accepts only PUT, PUT?t=mkdir, "
3101 "POST?t=upload, and POST?t=mkdir",
3102 self.POST, "/uri?t=bogus")
3105 def test_POST_mkdir_no_parentdir_immutable(self):
3106 (newkids, caps) = self._create_immutable_children()
3107 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3108 def _after_mkdir(res):
3109 self.failUnless(res.startswith("URI:DIR"), res)
3110 n = self.s.create_node_from_uri(res)
3111 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3112 d2.addCallback(lambda ign:
3113 self.failUnlessROChildURIIs(n, u"child-imm",
3115 d2.addCallback(lambda ign:
3116 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3117 caps['unknown_immcap']))
3118 d2.addCallback(lambda ign:
3119 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3121 d2.addCallback(lambda ign:
3122 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3124 d2.addCallback(lambda ign:
3125 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3126 caps['emptydircap']))
3128 d.addCallback(_after_mkdir)
3131 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3132 (newkids, caps) = self._create_initial_children()
3133 d = self.shouldFail2(error.Error,
3134 "test_POST_mkdir_no_parentdir_immutable_bad",
3136 "needed to be immutable but was not",
3138 "/uri?t=mkdir-immutable",
3139 simplejson.dumps(newkids))
3142 def test_welcome_page_mkdir_button(self):
3143 # Fetch the welcome page.
3145 def _after_get_welcome_page(res):
3146 MKDIR_BUTTON_RE = re.compile(
3147 '<form action="([^"]*)" method="post".*?'
3148 '<input type="hidden" name="t" value="([^"]*)" />'
3149 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3150 '<input type="submit" value="Create a directory" />',
3152 mo = MKDIR_BUTTON_RE.search(res)
3153 formaction = mo.group(1)
3155 formaname = mo.group(3)
3156 formavalue = mo.group(4)
3157 return (formaction, formt, formaname, formavalue)
3158 d.addCallback(_after_get_welcome_page)
3159 def _after_parse_form(res):
3160 (formaction, formt, formaname, formavalue) = res
3161 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3162 d.addCallback(_after_parse_form)
3163 d.addBoth(self.shouldRedirect, None, statuscode='303')
3166 def test_POST_mkdir_replace(self): # return value?
3167 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3168 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3169 d.addCallback(self.failUnlessNodeKeysAre, [])
3172 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3173 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3174 d.addBoth(self.shouldFail, error.Error,
3175 "POST_mkdir_no_replace_queryarg",
3177 "There was already a child by that name, and you asked me "
3178 "to not replace it")
3179 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3180 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3183 def test_POST_mkdir_no_replace_field(self): # return value?
3184 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3186 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3188 "There was already a child by that name, and you asked me "
3189 "to not replace it")
3190 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3191 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3194 def test_POST_mkdir_whendone_field(self):
3195 d = self.POST(self.public_url + "/foo",
3196 t="mkdir", name="newdir", when_done="/THERE")
3197 d.addBoth(self.shouldRedirect, "/THERE")
3198 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3199 d.addCallback(self.failUnlessNodeKeysAre, [])
3202 def test_POST_mkdir_whendone_queryarg(self):
3203 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3204 t="mkdir", name="newdir")
3205 d.addBoth(self.shouldRedirect, "/THERE")
3206 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3207 d.addCallback(self.failUnlessNodeKeysAre, [])
3210 def test_POST_bad_t(self):
3211 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
3212 "POST to a directory with bad t=BOGUS",
3213 self.POST, self.public_url + "/foo", t="BOGUS")
3216 def test_POST_set_children(self, command_name="set_children"):
3217 contents9, n9, newuri9 = self.makefile(9)
3218 contents10, n10, newuri10 = self.makefile(10)
3219 contents11, n11, newuri11 = self.makefile(11)
3222 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3225 "ctime": 1002777696.7564139,
3226 "mtime": 1002777696.7564139
3229 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3232 "ctime": 1002777696.7564139,
3233 "mtime": 1002777696.7564139
3236 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3239 "ctime": 1002777696.7564139,
3240 "mtime": 1002777696.7564139
3243 }""" % (newuri9, newuri10, newuri11)
3245 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3247 d = client.getPage(url, method="POST", postdata=reqbody)
3249 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3250 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3251 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3253 d.addCallback(_then)
3254 d.addErrback(self.dump_error)
3257 def test_POST_set_children_with_hyphen(self):
3258 return self.test_POST_set_children(command_name="set-children")
3260 def test_POST_link_uri(self):
3261 contents, n, newuri = self.makefile(8)
3262 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3263 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3264 d.addCallback(lambda res:
3265 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3269 def test_POST_link_uri_replace(self):
3270 contents, n, newuri = self.makefile(8)
3271 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3272 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3273 d.addCallback(lambda res:
3274 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3278 def test_POST_link_uri_unknown_bad(self):
3279 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3280 d.addBoth(self.shouldFail, error.Error,
3281 "POST_link_uri_unknown_bad",
3283 "unknown cap in a write slot")
3286 def test_POST_link_uri_unknown_ro_good(self):
3287 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3288 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3291 def test_POST_link_uri_unknown_imm_good(self):
3292 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3293 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3296 def test_POST_link_uri_no_replace_queryarg(self):
3297 contents, n, newuri = self.makefile(8)
3298 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3299 name="bar.txt", uri=newuri)
3300 d.addBoth(self.shouldFail, error.Error,
3301 "POST_link_uri_no_replace_queryarg",
3303 "There was already a child by that name, and you asked me "
3304 "to not replace it")
3305 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3306 d.addCallback(self.failUnlessIsBarDotTxt)
3309 def test_POST_link_uri_no_replace_field(self):
3310 contents, n, newuri = self.makefile(8)
3311 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3312 name="bar.txt", uri=newuri)
3313 d.addBoth(self.shouldFail, error.Error,
3314 "POST_link_uri_no_replace_field",
3316 "There was already a child by that name, and you asked me "
3317 "to not replace it")
3318 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3319 d.addCallback(self.failUnlessIsBarDotTxt)
3322 def test_POST_delete(self, command_name='delete'):
3323 d = self._foo_node.list()
3324 def _check_before(children):
3325 self.failUnless(u"bar.txt" in children)
3326 d.addCallback(_check_before)
3327 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3328 d.addCallback(lambda res: self._foo_node.list())
3329 def _check_after(children):
3330 self.failIf(u"bar.txt" in children)
3331 d.addCallback(_check_after)
3334 def test_POST_unlink(self):
3335 return self.test_POST_delete(command_name='unlink')
3337 def test_POST_rename_file(self):
3338 d = self.POST(self.public_url + "/foo", t="rename",
3339 from_name="bar.txt", to_name='wibble.txt')
3340 d.addCallback(lambda res:
3341 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3342 d.addCallback(lambda res:
3343 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3344 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3345 d.addCallback(self.failUnlessIsBarDotTxt)
3346 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3347 d.addCallback(self.failUnlessIsBarJSON)
3350 def test_POST_rename_file_redundant(self):
3351 d = self.POST(self.public_url + "/foo", t="rename",
3352 from_name="bar.txt", to_name='bar.txt')
3353 d.addCallback(lambda res:
3354 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3355 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3356 d.addCallback(self.failUnlessIsBarDotTxt)
3357 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3358 d.addCallback(self.failUnlessIsBarJSON)
3361 def test_POST_rename_file_replace(self):
3362 # rename a file and replace a directory with it
3363 d = self.POST(self.public_url + "/foo", t="rename",
3364 from_name="bar.txt", to_name='empty')
3365 d.addCallback(lambda res:
3366 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3367 d.addCallback(lambda res:
3368 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3369 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3370 d.addCallback(self.failUnlessIsBarDotTxt)
3371 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3372 d.addCallback(self.failUnlessIsBarJSON)
3375 def test_POST_rename_file_no_replace_queryarg(self):
3376 # rename a file and replace a directory with it
3377 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3378 from_name="bar.txt", to_name='empty')
3379 d.addBoth(self.shouldFail, error.Error,
3380 "POST_rename_file_no_replace_queryarg",
3382 "There was already a child by that name, and you asked me "
3383 "to not replace it")
3384 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3385 d.addCallback(self.failUnlessIsEmptyJSON)
3388 def test_POST_rename_file_no_replace_field(self):
3389 # rename a file and replace a directory with it
3390 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3391 from_name="bar.txt", to_name='empty')
3392 d.addBoth(self.shouldFail, error.Error,
3393 "POST_rename_file_no_replace_field",
3395 "There was already a child by that name, and you asked me "
3396 "to not replace it")
3397 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3398 d.addCallback(self.failUnlessIsEmptyJSON)
3401 def failUnlessIsEmptyJSON(self, res):
3402 data = simplejson.loads(res)
3403 self.failUnlessEqual(data[0], "dirnode", data)
3404 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3406 def test_POST_rename_file_slash_fail(self):
3407 d = self.POST(self.public_url + "/foo", t="rename",
3408 from_name="bar.txt", to_name='kirk/spock.txt')
3409 d.addBoth(self.shouldFail, error.Error,
3410 "test_POST_rename_file_slash_fail",
3412 "to_name= may not contain a slash",
3414 d.addCallback(lambda res:
3415 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3418 def test_POST_rename_dir(self):
3419 d = self.POST(self.public_url, t="rename",
3420 from_name="foo", to_name='plunk')
3421 d.addCallback(lambda res:
3422 self.failIfNodeHasChild(self.public_root, u"foo"))
3423 d.addCallback(lambda res:
3424 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3425 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3426 d.addCallback(self.failUnlessIsFooJSON)
3429 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3430 """ If target is not None then the redirection has to go to target. If
3431 statuscode is not None then the redirection has to be accomplished with
3432 that HTTP status code."""
3433 if not isinstance(res, failure.Failure):
3434 to_where = (target is None) and "somewhere" or ("to " + target)
3435 self.fail("%s: we were expecting to get redirected %s, not get an"
3436 " actual page: %s" % (which, to_where, res))
3437 res.trap(error.PageRedirect)
3438 if statuscode is not None:
3439 self.failUnlessReallyEqual(res.value.status, statuscode,
3440 "%s: not a redirect" % which)
3441 if target is not None:
3442 # the PageRedirect does not seem to capture the uri= query arg
3443 # properly, so we can't check for it.
3444 realtarget = self.webish_url + target
3445 self.failUnlessReallyEqual(res.value.location, realtarget,
3446 "%s: wrong target" % which)
3447 return res.value.location
3449 def test_GET_URI_form(self):
3450 base = "/uri?uri=%s" % self._bar_txt_uri
3451 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3452 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3454 d.addBoth(self.shouldRedirect, targetbase)
3455 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3456 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3457 d.addCallback(lambda res: self.GET(base+"&t=json"))
3458 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3459 d.addCallback(self.log, "about to get file by uri")
3460 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3461 d.addCallback(self.failUnlessIsBarDotTxt)
3462 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3463 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3464 followRedirect=True))
3465 d.addCallback(self.failUnlessIsFooJSON)
3466 d.addCallback(self.log, "got dir by uri")
3470 def test_GET_URI_form_bad(self):
3471 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3472 "400 Bad Request", "GET /uri requires uri=",
3476 def test_GET_rename_form(self):
3477 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3478 followRedirect=True)
3480 self.failUnless('name="when_done" value="."' in res, res)
3481 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3482 d.addCallback(_check)
3485 def log(self, res, msg):
3486 #print "MSG: %s RES: %s" % (msg, res)
3490 def test_GET_URI_URL(self):
3491 base = "/uri/%s" % self._bar_txt_uri
3493 d.addCallback(self.failUnlessIsBarDotTxt)
3494 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3495 d.addCallback(self.failUnlessIsBarDotTxt)
3496 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3497 d.addCallback(self.failUnlessIsBarDotTxt)
3500 def test_GET_URI_URL_dir(self):
3501 base = "/uri/%s?t=json" % self._foo_uri
3503 d.addCallback(self.failUnlessIsFooJSON)
3506 def test_GET_URI_URL_missing(self):
3507 base = "/uri/%s" % self._bad_file_uri
3508 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3509 http.GONE, None, "NotEnoughSharesError",
3511 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3512 # here? we must arrange for a download to fail after target.open()
3513 # has been called, and then inspect the response to see that it is
3514 # shorter than we expected.
3517 def test_PUT_DIRURL_uri(self):
3518 d = self.s.create_dirnode()
3520 new_uri = dn.get_uri()
3521 # replace /foo with a new (empty) directory
3522 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3523 d.addCallback(lambda res:
3524 self.failUnlessReallyEqual(res.strip(), new_uri))
3525 d.addCallback(lambda res:
3526 self.failUnlessRWChildURIIs(self.public_root,
3530 d.addCallback(_made_dir)
3533 def test_PUT_DIRURL_uri_noreplace(self):
3534 d = self.s.create_dirnode()
3536 new_uri = dn.get_uri()
3537 # replace /foo with a new (empty) directory, but ask that
3538 # replace=false, so it should fail
3539 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3540 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3542 self.public_url + "/foo?t=uri&replace=false",
3544 d.addCallback(lambda res:
3545 self.failUnlessRWChildURIIs(self.public_root,
3549 d.addCallback(_made_dir)
3552 def test_PUT_DIRURL_bad_t(self):
3553 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3554 "400 Bad Request", "PUT to a directory",
3555 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3556 d.addCallback(lambda res:
3557 self.failUnlessRWChildURIIs(self.public_root,
3562 def test_PUT_NEWFILEURL_uri(self):
3563 contents, n, new_uri = self.makefile(8)
3564 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3565 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3566 d.addCallback(lambda res:
3567 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3571 def test_PUT_NEWFILEURL_mdmf(self):
3572 new_contents = self.NEWFILE_CONTENTS * 300000
3573 d = self.PUT(self.public_url + \
3574 "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
3576 d.addCallback(lambda ignored:
3577 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3578 def _got_json(json):
3579 data = simplejson.loads(json)
3581 self.failUnlessIn("mutable-type", data)
3582 self.failUnlessEqual(data['mutable-type'], "mdmf")
3583 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3584 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3585 d.addCallback(_got_json)
3588 def test_PUT_NEWFILEURL_sdmf(self):
3589 new_contents = self.NEWFILE_CONTENTS * 300000
3590 d = self.PUT(self.public_url + \
3591 "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
3593 d.addCallback(lambda ignored:
3594 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3595 def _got_json(json):
3596 data = simplejson.loads(json)
3598 self.failUnlessIn("mutable-type", data)
3599 self.failUnlessEqual(data['mutable-type'], "sdmf")
3600 d.addCallback(_got_json)
3603 def test_PUT_NEWFILEURL_bad_mutable_type(self):
3604 new_contents = self.NEWFILE_CONTENTS * 300000
3605 return self.shouldHTTPError("test bad mutable type",
3606 400, "Bad Request", "Unknown type: foo",
3607 self.PUT, self.public_url + \
3608 "/foo/foo.txt?mutable=true&mutable-type=foo",
3611 def test_PUT_NEWFILEURL_uri_replace(self):
3612 contents, n, new_uri = self.makefile(8)
3613 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3614 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3615 d.addCallback(lambda res:
3616 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3620 def test_PUT_NEWFILEURL_uri_no_replace(self):
3621 contents, n, new_uri = self.makefile(8)
3622 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3623 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
3625 "There was already a child by that name, and you asked me "
3626 "to not replace it")
3629 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3630 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3631 d.addBoth(self.shouldFail, error.Error,
3632 "POST_put_uri_unknown_bad",
3634 "unknown cap in a write slot")
3637 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3638 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3639 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3640 u"put-future-ro.txt")
3643 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3644 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3645 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3646 u"put-future-imm.txt")
3649 def test_PUT_NEWFILE_URI(self):
3650 file_contents = "New file contents here\n"
3651 d = self.PUT("/uri", file_contents)
3653 assert isinstance(uri, str), uri
3654 self.failUnless(uri in FakeCHKFileNode.all_contents)
3655 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3657 return self.GET("/uri/%s" % uri)
3658 d.addCallback(_check)
3660 self.failUnlessReallyEqual(res, file_contents)
3661 d.addCallback(_check2)
3664 def test_PUT_NEWFILE_URI_not_mutable(self):
3665 file_contents = "New file contents here\n"
3666 d = self.PUT("/uri?mutable=false", file_contents)
3668 assert isinstance(uri, str), uri
3669 self.failUnless(uri in FakeCHKFileNode.all_contents)
3670 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3672 return self.GET("/uri/%s" % uri)
3673 d.addCallback(_check)
3675 self.failUnlessReallyEqual(res, file_contents)
3676 d.addCallback(_check2)
3679 def test_PUT_NEWFILE_URI_only_PUT(self):
3680 d = self.PUT("/uri?t=bogus", "")
3681 d.addBoth(self.shouldFail, error.Error,
3682 "PUT_NEWFILE_URI_only_PUT",
3684 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3687 def test_PUT_NEWFILE_URI_mutable(self):
3688 file_contents = "New file contents here\n"
3689 d = self.PUT("/uri?mutable=true", file_contents)
3690 def _check1(filecap):
3691 filecap = filecap.strip()
3692 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3693 self.filecap = filecap
3694 u = uri.WriteableSSKFileURI.init_from_string(filecap)
3695 self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3696 n = self.s.create_node_from_uri(filecap)
3697 return n.download_best_version()
3698 d.addCallback(_check1)
3700 self.failUnlessReallyEqual(data, file_contents)
3701 return self.GET("/uri/%s" % urllib.quote(self.filecap))
3702 d.addCallback(_check2)
3704 self.failUnlessReallyEqual(res, file_contents)
3705 d.addCallback(_check3)
3708 def test_PUT_mkdir(self):
3709 d = self.PUT("/uri?t=mkdir", "")
3711 n = self.s.create_node_from_uri(uri.strip())
3712 d2 = self.failUnlessNodeKeysAre(n, [])
3713 d2.addCallback(lambda res:
3714 self.GET("/uri/%s?t=json" % uri))
3716 d.addCallback(_check)
3717 d.addCallback(self.failUnlessIsEmptyJSON)
3720 def test_PUT_mkdir_mdmf(self):
3721 d = self.PUT("/uri?t=mkdir&mutable-type=mdmf", "")
3723 u = uri.from_string(res)
3724 # Check that this is an MDMF writecap
3725 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3729 def test_PUT_mkdir_sdmf(self):
3730 d = self.PUT("/uri?t=mkdir&mutable-type=sdmf", "")
3732 u = uri.from_string(res)
3733 self.failUnlessIsInstance(u, uri.DirectoryURI)
3737 def test_PUT_mkdir_bad_mutable_type(self):
3738 return self.shouldHTTPError("bad mutable type",
3739 400, "Bad Request", "Unknown type: foo",
3740 self.PUT, "/uri?t=mkdir&mutable-type=foo",
3743 def test_POST_check(self):
3744 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3746 # this returns a string form of the results, which are probably
3747 # None since we're using fake filenodes.
3748 # TODO: verify that the check actually happened, by changing
3749 # FakeCHKFileNode to count how many times .check() has been
3752 d.addCallback(_done)
3756 def test_PUT_update_at_offset(self):
3757 file_contents = "test file" * 100000 # about 900 KiB
3758 d = self.PUT("/uri?mutable=true", file_contents)
3760 self.filecap = filecap
3761 new_data = file_contents[:100]
3762 new = "replaced and so on"
3764 new_data += file_contents[len(new_data):]
3765 assert len(new_data) == len(file_contents)
3766 self.new_data = new_data
3767 d.addCallback(_then)
3768 d.addCallback(lambda ignored:
3769 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3770 "replaced and so on"))
3771 def _get_data(filecap):
3772 n = self.s.create_node_from_uri(filecap)
3773 return n.download_best_version()
3774 d.addCallback(_get_data)
3775 d.addCallback(lambda results:
3776 self.failUnlessEqual(results, self.new_data))
3777 # Now try appending things to the file
3778 d.addCallback(lambda ignored:
3779 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3781 d.addCallback(_get_data)
3782 d.addCallback(lambda results:
3783 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3784 # and try replacing the beginning of the file
3785 d.addCallback(lambda ignored:
3786 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3787 d.addCallback(_get_data)
3788 d.addCallback(lambda results:
3789 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3792 def test_PUT_update_at_invalid_offset(self):
3793 file_contents = "test file" * 100000 # about 900 KiB
3794 d = self.PUT("/uri?mutable=true", file_contents)
3796 self.filecap = filecap
3797 d.addCallback(_then)
3798 # Negative offsets should cause an error.
3799 d.addCallback(lambda ignored:
3800 self.shouldHTTPError("test mutable invalid offset negative",
3804 "/uri/%s?offset=-1" % self.filecap,
3808 def test_PUT_update_at_offset_immutable(self):
3809 file_contents = "Test file" * 100000
3810 d = self.PUT("/uri", file_contents)
3812 self.filecap = filecap
3813 d.addCallback(_then)
3814 d.addCallback(lambda ignored:
3815 self.shouldHTTPError("test immutable update",
3819 "/uri/%s?offset=50" % self.filecap,
3824 def test_bad_method(self):
3825 url = self.webish_url + self.public_url + "/foo/bar.txt"
3826 d = self.shouldHTTPError("test_bad_method",
3827 501, "Not Implemented",
3828 "I don't know how to treat a BOGUS request.",
3829 client.getPage, url, method="BOGUS")
3832 def test_short_url(self):
3833 url = self.webish_url + "/uri"
3834 d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3835 "I don't know how to treat a DELETE request.",
3836 client.getPage, url, method="DELETE")
3839 def test_ophandle_bad(self):
3840 url = self.webish_url + "/operations/bogus?t=status"
3841 d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3842 "unknown/expired handle 'bogus'",
3843 client.getPage, url)
3846 def test_ophandle_cancel(self):
3847 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3848 followRedirect=True)
3849 d.addCallback(lambda ignored:
3850 self.GET("/operations/128?t=status&output=JSON"))
3852 data = simplejson.loads(res)
3853 self.failUnless("finished" in data, res)
3854 monitor = self.ws.root.child_operations.handles["128"][0]
3855 d = self.POST("/operations/128?t=cancel&output=JSON")
3857 data = simplejson.loads(res)
3858 self.failUnless("finished" in data, res)
3859 # t=cancel causes the handle to be forgotten
3860 self.failUnless(monitor.is_cancelled())
3861 d.addCallback(_check2)
3863 d.addCallback(_check1)
3864 d.addCallback(lambda ignored:
3865 self.shouldHTTPError("test_ophandle_cancel",
3866 404, "404 Not Found",
3867 "unknown/expired handle '128'",
3869 "/operations/128?t=status&output=JSON"))
3872 def test_ophandle_retainfor(self):
3873 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3874 followRedirect=True)
3875 d.addCallback(lambda ignored:
3876 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3878 data = simplejson.loads(res)
3879 self.failUnless("finished" in data, res)
3880 d.addCallback(_check1)
3881 # the retain-for=0 will cause the handle to be expired very soon
3882 d.addCallback(lambda ign:
3883 self.clock.advance(2.0))
3884 d.addCallback(lambda ignored:
3885 self.shouldHTTPError("test_ophandle_retainfor",
3886 404, "404 Not Found",
3887 "unknown/expired handle '129'",
3889 "/operations/129?t=status&output=JSON"))
3892 def test_ophandle_release_after_complete(self):
3893 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3894 followRedirect=True)
3895 d.addCallback(self.wait_for_operation, "130")
3896 d.addCallback(lambda ignored:
3897 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3898 # the release-after-complete=true will cause the handle to be expired
3899 d.addCallback(lambda ignored:
3900 self.shouldHTTPError("test_ophandle_release_after_complete",
3901 404, "404 Not Found",
3902 "unknown/expired handle '130'",
3904 "/operations/130?t=status&output=JSON"))
3907 def test_uncollected_ophandle_expiration(self):
3908 # uncollected ophandles should expire after 4 days
3909 def _make_uncollected_ophandle(ophandle):
3910 d = self.POST(self.public_url +
3911 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3912 followRedirect=False)
3913 # When we start the operation, the webapi server will want
3914 # to redirect us to the page for the ophandle, so we get
3915 # confirmation that the operation has started. If the
3916 # manifest operation has finished by the time we get there,
3917 # following that redirect (by setting followRedirect=True
3918 # above) has the side effect of collecting the ophandle that
3919 # we've just created, which means that we can't use the
3920 # ophandle to test the uncollected timeout anymore. So,
3921 # instead, catch the 302 here and don't follow it.
3922 d.addBoth(self.should302, "uncollected_ophandle_creation")
3924 # Create an ophandle, don't collect it, then advance the clock by
3925 # 4 days - 1 second and make sure that the ophandle is still there.
3926 d = _make_uncollected_ophandle(131)
3927 d.addCallback(lambda ign:
3928 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3929 d.addCallback(lambda ign:
3930 self.GET("/operations/131?t=status&output=JSON"))
3932 data = simplejson.loads(res)
3933 self.failUnless("finished" in data, res)
3934 d.addCallback(_check1)
3935 # Create an ophandle, don't collect it, then try to collect it
3936 # after 4 days. It should be gone.
3937 d.addCallback(lambda ign:
3938 _make_uncollected_ophandle(132))
3939 d.addCallback(lambda ign:
3940 self.clock.advance(96*60*60))
3941 d.addCallback(lambda ign:
3942 self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3943 404, "404 Not Found",
3944 "unknown/expired handle '132'",
3946 "/operations/132?t=status&output=JSON"))
3949 def test_collected_ophandle_expiration(self):
3950 # collected ophandles should expire after 1 day
3951 def _make_collected_ophandle(ophandle):
3952 d = self.POST(self.public_url +
3953 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3954 followRedirect=True)
3955 # By following the initial redirect, we collect the ophandle
3956 # we've just created.
3958 # Create a collected ophandle, then collect it after 23 hours
3959 # and 59 seconds to make sure that it is still there.
3960 d = _make_collected_ophandle(133)
3961 d.addCallback(lambda ign:
3962 self.clock.advance((24*60*60) - 1))
3963 d.addCallback(lambda ign:
3964 self.GET("/operations/133?t=status&output=JSON"))
3966 data = simplejson.loads(res)
3967 self.failUnless("finished" in data, res)
3968 d.addCallback(_check1)
3969 # Create another uncollected ophandle, then try to collect it
3970 # after 24 hours to make sure that it is gone.
3971 d.addCallback(lambda ign:
3972 _make_collected_ophandle(134))
3973 d.addCallback(lambda ign:
3974 self.clock.advance(24*60*60))
3975 d.addCallback(lambda ign:
3976 self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3977 404, "404 Not Found",
3978 "unknown/expired handle '134'",
3980 "/operations/134?t=status&output=JSON"))
3983 def test_incident(self):
3984 d = self.POST("/report_incident", details="eek")
3986 self.failUnless("Thank you for your report!" in res, res)
3987 d.addCallback(_done)
3990 def test_static(self):
3991 webdir = os.path.join(self.staticdir, "subdir")
3992 fileutil.make_dirs(webdir)
3993 f = open(os.path.join(webdir, "hello.txt"), "wb")
3997 d = self.GET("/static/subdir/hello.txt")
3999 self.failUnlessReallyEqual(res, "hello")
4000 d.addCallback(_check)
4004 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4005 def test_load_file(self):
4006 # This will raise an exception unless a well-formed XML file is found under that name.
4007 common.getxmlfile('directory.xhtml').load()
4009 def test_parse_replace_arg(self):
4010 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4011 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4012 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4014 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4015 common.parse_replace_arg, "only_fles")
4017 def test_abbreviate_time(self):
4018 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4019 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4020 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4021 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4022 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4023 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4025 def test_compute_rate(self):
4026 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4027 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4028 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4029 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4030 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4031 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4032 self.shouldFail(AssertionError, "test_compute_rate", "",
4033 common.compute_rate, -100, 10)
4034 self.shouldFail(AssertionError, "test_compute_rate", "",
4035 common.compute_rate, 100, -10)
4038 rate = common.compute_rate(10*1000*1000, 1)
4039 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4041 def test_abbreviate_rate(self):
4042 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4043 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4044 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4045 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4047 def test_abbreviate_size(self):
4048 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4049 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4050 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4051 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4052 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4054 def test_plural(self):
4056 return "%d second%s" % (s, status.plural(s))
4057 self.failUnlessReallyEqual(convert(0), "0 seconds")
4058 self.failUnlessReallyEqual(convert(1), "1 second")
4059 self.failUnlessReallyEqual(convert(2), "2 seconds")
4061 return "has share%s: %s" % (status.plural(s), ",".join(s))
4062 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4063 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4064 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4067 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4069 def CHECK(self, ign, which, args, clientnum=0):
4070 fileurl = self.fileurls[which]
4071 url = fileurl + "?" + args
4072 return self.GET(url, method="POST", clientnum=clientnum)
4074 def test_filecheck(self):
4075 self.basedir = "web/Grid/filecheck"
4077 c0 = self.g.clients[0]
4080 d = c0.upload(upload.Data(DATA, convergence=""))
4081 def _stash_uri(ur, which):
4082 self.uris[which] = ur.uri
4083 d.addCallback(_stash_uri, "good")
4084 d.addCallback(lambda ign:
4085 c0.upload(upload.Data(DATA+"1", convergence="")))
4086 d.addCallback(_stash_uri, "sick")
4087 d.addCallback(lambda ign:
4088 c0.upload(upload.Data(DATA+"2", convergence="")))
4089 d.addCallback(_stash_uri, "dead")
4090 def _stash_mutable_uri(n, which):
4091 self.uris[which] = n.get_uri()
4092 assert isinstance(self.uris[which], str)
4093 d.addCallback(lambda ign:
4094 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4095 d.addCallback(_stash_mutable_uri, "corrupt")
4096 d.addCallback(lambda ign:
4097 c0.upload(upload.Data("literal", convergence="")))
4098 d.addCallback(_stash_uri, "small")
4099 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4100 d.addCallback(_stash_mutable_uri, "smalldir")
4102 def _compute_fileurls(ignored):
4104 for which in self.uris:
4105 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4106 d.addCallback(_compute_fileurls)
4108 def _clobber_shares(ignored):
4109 good_shares = self.find_uri_shares(self.uris["good"])
4110 self.failUnlessReallyEqual(len(good_shares), 10)
4111 sick_shares = self.find_uri_shares(self.uris["sick"])
4112 os.unlink(sick_shares[0][2])
4113 dead_shares = self.find_uri_shares(self.uris["dead"])
4114 for i in range(1, 10):
4115 os.unlink(dead_shares[i][2])
4116 c_shares = self.find_uri_shares(self.uris["corrupt"])
4117 cso = CorruptShareOptions()
4118 cso.stdout = StringIO()
4119 cso.parseOptions([c_shares[0][2]])
4121 d.addCallback(_clobber_shares)
4123 d.addCallback(self.CHECK, "good", "t=check")
4124 def _got_html_good(res):
4125 self.failUnless("Healthy" in res, res)
4126 self.failIf("Not Healthy" in res, res)
4127 d.addCallback(_got_html_good)
4128 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4129 def _got_html_good_return_to(res):
4130 self.failUnless("Healthy" in res, res)
4131 self.failIf("Not Healthy" in res, res)
4132 self.failUnless('<a href="somewhere">Return to file'
4134 d.addCallback(_got_html_good_return_to)
4135 d.addCallback(self.CHECK, "good", "t=check&output=json")
4136 def _got_json_good(res):
4137 r = simplejson.loads(res)
4138 self.failUnlessEqual(r["summary"], "Healthy")
4139 self.failUnless(r["results"]["healthy"])
4140 self.failIf(r["results"]["needs-rebalancing"])
4141 self.failUnless(r["results"]["recoverable"])
4142 d.addCallback(_got_json_good)
4144 d.addCallback(self.CHECK, "small", "t=check")
4145 def _got_html_small(res):
4146 self.failUnless("Literal files are always healthy" in res, res)
4147 self.failIf("Not Healthy" in res, res)
4148 d.addCallback(_got_html_small)
4149 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4150 def _got_html_small_return_to(res):
4151 self.failUnless("Literal files are always healthy" in res, res)
4152 self.failIf("Not Healthy" in res, res)
4153 self.failUnless('<a href="somewhere">Return to file'
4155 d.addCallback(_got_html_small_return_to)
4156 d.addCallback(self.CHECK, "small", "t=check&output=json")
4157 def _got_json_small(res):
4158 r = simplejson.loads(res)
4159 self.failUnlessEqual(r["storage-index"], "")
4160 self.failUnless(r["results"]["healthy"])
4161 d.addCallback(_got_json_small)
4163 d.addCallback(self.CHECK, "smalldir", "t=check")
4164 def _got_html_smalldir(res):
4165 self.failUnless("Literal files are always healthy" in res, res)
4166 self.failIf("Not Healthy" in res, res)
4167 d.addCallback(_got_html_smalldir)
4168 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4169 def _got_json_smalldir(res):
4170 r = simplejson.loads(res)
4171 self.failUnlessEqual(r["storage-index"], "")
4172 self.failUnless(r["results"]["healthy"])
4173 d.addCallback(_got_json_smalldir)
4175 d.addCallback(self.CHECK, "sick", "t=check")
4176 def _got_html_sick(res):
4177 self.failUnless("Not Healthy" in res, res)
4178 d.addCallback(_got_html_sick)
4179 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4180 def _got_json_sick(res):
4181 r = simplejson.loads(res)
4182 self.failUnlessEqual(r["summary"],
4183 "Not Healthy: 9 shares (enc 3-of-10)")
4184 self.failIf(r["results"]["healthy"])
4185 self.failIf(r["results"]["needs-rebalancing"])
4186 self.failUnless(r["results"]["recoverable"])
4187 d.addCallback(_got_json_sick)
4189 d.addCallback(self.CHECK, "dead", "t=check")
4190 def _got_html_dead(res):
4191 self.failUnless("Not Healthy" in res, res)
4192 d.addCallback(_got_html_dead)
4193 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4194 def _got_json_dead(res):
4195 r = simplejson.loads(res)
4196 self.failUnlessEqual(r["summary"],
4197 "Not Healthy: 1 shares (enc 3-of-10)")
4198 self.failIf(r["results"]["healthy"])
4199 self.failIf(r["results"]["needs-rebalancing"])
4200 self.failIf(r["results"]["recoverable"])
4201 d.addCallback(_got_json_dead)
4203 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4204 def _got_html_corrupt(res):
4205 self.failUnless("Not Healthy! : Unhealthy" in res, res)
4206 d.addCallback(_got_html_corrupt)
4207 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4208 def _got_json_corrupt(res):
4209 r = simplejson.loads(res)
4210 self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4212 self.failIf(r["results"]["healthy"])
4213 self.failUnless(r["results"]["recoverable"])
4214 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4215 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4216 d.addCallback(_got_json_corrupt)
4218 d.addErrback(self.explain_web_error)
4221 def test_repair_html(self):
4222 self.basedir = "web/Grid/repair_html"
4224 c0 = self.g.clients[0]
4227 d = c0.upload(upload.Data(DATA, convergence=""))
4228 def _stash_uri(ur, which):
4229 self.uris[which] = ur.uri
4230 d.addCallback(_stash_uri, "good")
4231 d.addCallback(lambda ign:
4232 c0.upload(upload.Data(DATA+"1", convergence="")))
4233 d.addCallback(_stash_uri, "sick")
4234 d.addCallback(lambda ign:
4235 c0.upload(upload.Data(DATA+"2", convergence="")))
4236 d.addCallback(_stash_uri, "dead")
4237 def _stash_mutable_uri(n, which):
4238 self.uris[which] = n.get_uri()
4239 assert isinstance(self.uris[which], str)
4240 d.addCallback(lambda ign:
4241 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4242 d.addCallback(_stash_mutable_uri, "corrupt")
4244 def _compute_fileurls(ignored):
4246 for which in self.uris:
4247 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4248 d.addCallback(_compute_fileurls)
4250 def _clobber_shares(ignored):
4251 good_shares = self.find_uri_shares(self.uris["good"])
4252 self.failUnlessReallyEqual(len(good_shares), 10)
4253 sick_shares = self.find_uri_shares(self.uris["sick"])
4254 os.unlink(sick_shares[0][2])
4255 dead_shares = self.find_uri_shares(self.uris["dead"])
4256 for i in range(1, 10):
4257 os.unlink(dead_shares[i][2])
4258 c_shares = self.find_uri_shares(self.uris["corrupt"])
4259 cso = CorruptShareOptions()
4260 cso.stdout = StringIO()
4261 cso.parseOptions([c_shares[0][2]])
4263 d.addCallback(_clobber_shares)
4265 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4266 def _got_html_good(res):
4267 self.failUnless("Healthy" in res, res)
4268 self.failIf("Not Healthy" in res, res)
4269 self.failUnless("No repair necessary" in res, res)
4270 d.addCallback(_got_html_good)
4272 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4273 def _got_html_sick(res):
4274 self.failUnless("Healthy : healthy" in res, res)
4275 self.failIf("Not Healthy" in res, res)
4276 self.failUnless("Repair successful" in res, res)
4277 d.addCallback(_got_html_sick)
4279 # repair of a dead file will fail, of course, but it isn't yet
4280 # clear how this should be reported. Right now it shows up as
4283 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4284 #def _got_html_dead(res):
4286 # self.failUnless("Healthy : healthy" in res, res)
4287 # self.failIf("Not Healthy" in res, res)
4288 # self.failUnless("No repair necessary" in res, res)
4289 #d.addCallback(_got_html_dead)
4291 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4292 def _got_html_corrupt(res):
4293 self.failUnless("Healthy : Healthy" in res, res)
4294 self.failIf("Not Healthy" in res, res)
4295 self.failUnless("Repair successful" in res, res)
4296 d.addCallback(_got_html_corrupt)
4298 d.addErrback(self.explain_web_error)
4301 def test_repair_json(self):
4302 self.basedir = "web/Grid/repair_json"
4304 c0 = self.g.clients[0]
4307 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4308 def _stash_uri(ur, which):
4309 self.uris[which] = ur.uri
4310 d.addCallback(_stash_uri, "sick")
4312 def _compute_fileurls(ignored):
4314 for which in self.uris:
4315 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4316 d.addCallback(_compute_fileurls)
4318 def _clobber_shares(ignored):
4319 sick_shares = self.find_uri_shares(self.uris["sick"])
4320 os.unlink(sick_shares[0][2])
4321 d.addCallback(_clobber_shares)
4323 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4324 def _got_json_sick(res):
4325 r = simplejson.loads(res)
4326 self.failUnlessReallyEqual(r["repair-attempted"], True)
4327 self.failUnlessReallyEqual(r["repair-successful"], True)
4328 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4329 "Not Healthy: 9 shares (enc 3-of-10)")
4330 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4331 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4332 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4333 d.addCallback(_got_json_sick)
4335 d.addErrback(self.explain_web_error)
4338 def test_unknown(self, immutable=False):
4339 self.basedir = "web/Grid/unknown"
4341 self.basedir = "web/Grid/unknown-immutable"
4344 c0 = self.g.clients[0]
4348 # the future cap format may contain slashes, which must be tolerated
4349 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4353 name = u"future-imm"
4354 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4355 d = c0.create_immutable_dirnode({name: (future_node, {})})
4358 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4359 d = c0.create_dirnode()
4361 def _stash_root_and_create_file(n):
4363 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4364 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4366 return self.rootnode.set_node(name, future_node)
4367 d.addCallback(_stash_root_and_create_file)
4369 # make sure directory listing tolerates unknown nodes
4370 d.addCallback(lambda ign: self.GET(self.rooturl))
4371 def _check_directory_html(res, expected_type_suffix):
4372 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4373 '<td>%s</td>' % (expected_type_suffix, str(name)),
4375 self.failUnless(re.search(pattern, res), res)
4376 # find the More Info link for name, should be relative
4377 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4378 info_url = mo.group(1)
4379 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4381 d.addCallback(_check_directory_html, "-IMM")
4383 d.addCallback(_check_directory_html, "")
4385 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4386 def _check_directory_json(res, expect_rw_uri):
4387 data = simplejson.loads(res)
4388 self.failUnlessEqual(data[0], "dirnode")
4389 f = data[1]["children"][name]
4390 self.failUnlessEqual(f[0], "unknown")
4392 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4394 self.failIfIn("rw_uri", f[1])
4396 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4398 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4399 self.failUnless("metadata" in f[1])
4400 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4402 def _check_info(res, expect_rw_uri, expect_ro_uri):
4403 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4405 self.failUnlessIn(unknown_rwcap, res)
4408 self.failUnlessIn(unknown_immcap, res)
4410 self.failUnlessIn(unknown_rocap, res)
4412 self.failIfIn(unknown_rocap, res)
4413 self.failIfIn("Raw data as", res)
4414 self.failIfIn("Directory writecap", res)
4415 self.failIfIn("Checker Operations", res)
4416 self.failIfIn("Mutable File Operations", res)
4417 self.failIfIn("Directory Operations", res)
4419 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4420 # why they fail. Possibly related to ticket #922.
4422 d.addCallback(lambda ign: self.GET(expected_info_url))
4423 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4424 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4425 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4427 def _check_json(res, expect_rw_uri):
4428 data = simplejson.loads(res)
4429 self.failUnlessEqual(data[0], "unknown")
4431 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4433 self.failIfIn("rw_uri", data[1])
4436 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4437 self.failUnlessReallyEqual(data[1]["mutable"], False)
4439 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4440 self.failUnlessReallyEqual(data[1]["mutable"], True)
4442 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4443 self.failIf("mutable" in data[1], data[1])
4445 # TODO: check metadata contents
4446 self.failUnless("metadata" in data[1])
4448 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4449 d.addCallback(_check_json, expect_rw_uri=not immutable)
4451 # and make sure that a read-only version of the directory can be
4452 # rendered too. This version will not have unknown_rwcap, whether
4453 # or not future_node was immutable.
4454 d.addCallback(lambda ign: self.GET(self.rourl))
4456 d.addCallback(_check_directory_html, "-IMM")
4458 d.addCallback(_check_directory_html, "-RO")
4460 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4461 d.addCallback(_check_directory_json, expect_rw_uri=False)
4463 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4464 d.addCallback(_check_json, expect_rw_uri=False)
4466 # TODO: check that getting t=info from the Info link in the ro directory
4467 # works, and does not include the writecap URI.
4470 def test_immutable_unknown(self):
4471 return self.test_unknown(immutable=True)
4473 def test_mutant_dirnodes_are_omitted(self):
4474 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4477 c = self.g.clients[0]
4482 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4483 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4484 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4486 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4487 # test the dirnode and web layers separately.
4489 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4490 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4491 # When the directory is read, the mutants should be silently disposed of, leaving
4492 # their lonely sibling.
4493 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4494 # because immutable directories don't have a writecap and therefore that field
4495 # isn't (and can't be) decrypted.
4496 # TODO: The field still exists in the netstring. Technically we should check what
4497 # happens if something is put there (_unpack_contents should raise ValueError),
4498 # but that can wait.
4500 lonely_child = nm.create_from_cap(lonely_uri)
4501 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4502 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4504 def _by_hook_or_by_crook():
4506 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4507 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4509 mutant_write_in_ro_child.get_write_uri = lambda: None
4510 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4512 kids = {u"lonely": (lonely_child, {}),
4513 u"ro": (mutant_ro_child, {}),
4514 u"write-in-ro": (mutant_write_in_ro_child, {}),
4516 d = c.create_immutable_dirnode(kids)
4519 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4520 self.failIf(dn.is_mutable())
4521 self.failUnless(dn.is_readonly())
4522 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4523 self.failIf(hasattr(dn._node, 'get_writekey'))
4525 self.failUnless("RO-IMM" in rep)
4527 self.failUnlessIn("CHK", cap.to_string())
4530 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4531 return download_to_data(dn._node)
4532 d.addCallback(_created)
4534 def _check_data(data):
4535 # Decode the netstring representation of the directory to check that all children
4536 # are present. This is a bit of an abstraction violation, but there's not really
4537 # any other way to do it given that the real DirectoryNode._unpack_contents would
4538 # strip the mutant children out (which is what we're trying to test, later).
4541 while position < len(data):
4542 entries, position = split_netstring(data, 1, position)
4544 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4545 name = name_utf8.decode("utf-8")
4546 self.failUnless(rwcapdata == "")
4547 self.failUnless(name in kids)
4548 (expected_child, ign) = kids[name]
4549 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4552 self.failUnlessReallyEqual(numkids, 3)
4553 return self.rootnode.list()
4554 d.addCallback(_check_data)
4556 # Now when we use the real directory listing code, the mutants should be absent.
4557 def _check_kids(children):
4558 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4559 lonely_node, lonely_metadata = children[u"lonely"]
4561 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4562 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4563 d.addCallback(_check_kids)
4565 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4566 d.addCallback(lambda n: n.list())
4567 d.addCallback(_check_kids) # again with dirnode recreated from cap
4569 # Make sure the lonely child can be listed in HTML...
4570 d.addCallback(lambda ign: self.GET(self.rooturl))
4571 def _check_html(res):
4572 self.failIfIn("URI:SSK", res)
4573 get_lonely = "".join([r'<td>FILE</td>',
4575 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4577 r'\s+<td align="right">%d</td>' % len("one"),
4579 self.failUnless(re.search(get_lonely, res), res)
4581 # find the More Info link for name, should be relative
4582 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4583 info_url = mo.group(1)
4584 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4585 d.addCallback(_check_html)
4588 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4589 def _check_json(res):
4590 data = simplejson.loads(res)
4591 self.failUnlessEqual(data[0], "dirnode")
4592 listed_children = data[1]["children"]
4593 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4594 ll_type, ll_data = listed_children[u"lonely"]
4595 self.failUnlessEqual(ll_type, "filenode")
4596 self.failIf("rw_uri" in ll_data)
4597 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4598 d.addCallback(_check_json)
4601 def test_deep_check(self):
4602 self.basedir = "web/Grid/deep_check"
4604 c0 = self.g.clients[0]
4608 d = c0.create_dirnode()
4609 def _stash_root_and_create_file(n):
4611 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4612 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4613 d.addCallback(_stash_root_and_create_file)
4614 def _stash_uri(fn, which):
4615 self.uris[which] = fn.get_uri()
4617 d.addCallback(_stash_uri, "good")
4618 d.addCallback(lambda ign:
4619 self.rootnode.add_file(u"small",
4620 upload.Data("literal",
4622 d.addCallback(_stash_uri, "small")
4623 d.addCallback(lambda ign:
4624 self.rootnode.add_file(u"sick",
4625 upload.Data(DATA+"1",
4627 d.addCallback(_stash_uri, "sick")
4629 # this tests that deep-check and stream-manifest will ignore
4630 # UnknownNode instances. Hopefully this will also cover deep-stats.
4631 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4632 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4634 def _clobber_shares(ignored):
4635 self.delete_shares_numbered(self.uris["sick"], [0,1])
4636 d.addCallback(_clobber_shares)
4644 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4647 units = [simplejson.loads(line)
4648 for line in res.splitlines()
4651 print "response is:", res
4652 print "undecodeable line was '%s'" % line
4654 self.failUnlessReallyEqual(len(units), 5+1)
4655 # should be parent-first
4657 self.failUnlessEqual(u0["path"], [])
4658 self.failUnlessEqual(u0["type"], "directory")
4659 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4660 u0cr = u0["check-results"]
4661 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4663 ugood = [u for u in units
4664 if u["type"] == "file" and u["path"] == [u"good"]][0]
4665 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4666 ugoodcr = ugood["check-results"]
4667 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4670 self.failUnlessEqual(stats["type"], "stats")
4672 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4673 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4674 self.failUnlessReallyEqual(s["count-directories"], 1)
4675 self.failUnlessReallyEqual(s["count-unknown"], 1)
4676 d.addCallback(_done)
4678 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4679 def _check_manifest(res):
4680 self.failUnless(res.endswith("\n"))
4681 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4682 self.failUnlessReallyEqual(len(units), 5+1)
4683 self.failUnlessEqual(units[-1]["type"], "stats")
4685 self.failUnlessEqual(first["path"], [])
4686 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4687 self.failUnlessEqual(first["type"], "directory")
4688 stats = units[-1]["stats"]
4689 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4690 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4691 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4692 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4693 self.failUnlessReallyEqual(stats["count-unknown"], 1)
4694 d.addCallback(_check_manifest)
4696 # now add root/subdir and root/subdir/grandchild, then make subdir
4697 # unrecoverable, then see what happens
4699 d.addCallback(lambda ign:
4700 self.rootnode.create_subdirectory(u"subdir"))
4701 d.addCallback(_stash_uri, "subdir")
4702 d.addCallback(lambda subdir_node:
4703 subdir_node.add_file(u"grandchild",
4704 upload.Data(DATA+"2",
4706 d.addCallback(_stash_uri, "grandchild")
4708 d.addCallback(lambda ign:
4709 self.delete_shares_numbered(self.uris["subdir"],
4717 # root/subdir [unrecoverable]
4718 # root/subdir/grandchild
4720 # how should a streaming-JSON API indicate fatal error?
4721 # answer: emit ERROR: instead of a JSON string
4723 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4724 def _check_broken_manifest(res):
4725 lines = res.splitlines()
4727 for (i,line) in enumerate(lines)
4728 if line.startswith("ERROR:")]
4730 self.fail("no ERROR: in output: %s" % (res,))
4731 first_error = error_lines[0]
4732 error_line = lines[first_error]
4733 error_msg = lines[first_error+1:]
4734 error_msg_s = "\n".join(error_msg) + "\n"
4735 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4737 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4738 units = [simplejson.loads(line) for line in lines[:first_error]]
4739 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4740 last_unit = units[-1]
4741 self.failUnlessEqual(last_unit["path"], ["subdir"])
4742 d.addCallback(_check_broken_manifest)
4744 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4745 def _check_broken_deepcheck(res):
4746 lines = res.splitlines()
4748 for (i,line) in enumerate(lines)
4749 if line.startswith("ERROR:")]
4751 self.fail("no ERROR: in output: %s" % (res,))
4752 first_error = error_lines[0]
4753 error_line = lines[first_error]
4754 error_msg = lines[first_error+1:]
4755 error_msg_s = "\n".join(error_msg) + "\n"
4756 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4758 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4759 units = [simplejson.loads(line) for line in lines[:first_error]]
4760 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4761 last_unit = units[-1]
4762 self.failUnlessEqual(last_unit["path"], ["subdir"])
4763 r = last_unit["check-results"]["results"]
4764 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4765 self.failUnlessReallyEqual(r["count-shares-good"], 1)
4766 self.failUnlessReallyEqual(r["recoverable"], False)
4767 d.addCallback(_check_broken_deepcheck)
4769 d.addErrback(self.explain_web_error)
4772 def test_deep_check_and_repair(self):
4773 self.basedir = "web/Grid/deep_check_and_repair"
4775 c0 = self.g.clients[0]
4779 d = c0.create_dirnode()
4780 def _stash_root_and_create_file(n):
4782 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4783 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4784 d.addCallback(_stash_root_and_create_file)
4785 def _stash_uri(fn, which):
4786 self.uris[which] = fn.get_uri()
4787 d.addCallback(_stash_uri, "good")
4788 d.addCallback(lambda ign:
4789 self.rootnode.add_file(u"small",
4790 upload.Data("literal",
4792 d.addCallback(_stash_uri, "small")
4793 d.addCallback(lambda ign:
4794 self.rootnode.add_file(u"sick",
4795 upload.Data(DATA+"1",
4797 d.addCallback(_stash_uri, "sick")
4798 #d.addCallback(lambda ign:
4799 # self.rootnode.add_file(u"dead",
4800 # upload.Data(DATA+"2",
4802 #d.addCallback(_stash_uri, "dead")
4804 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4805 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4806 #d.addCallback(_stash_uri, "corrupt")
4808 def _clobber_shares(ignored):
4809 good_shares = self.find_uri_shares(self.uris["good"])
4810 self.failUnlessReallyEqual(len(good_shares), 10)
4811 sick_shares = self.find_uri_shares(self.uris["sick"])
4812 os.unlink(sick_shares[0][2])
4813 #dead_shares = self.find_uri_shares(self.uris["dead"])
4814 #for i in range(1, 10):
4815 # os.unlink(dead_shares[i][2])
4817 #c_shares = self.find_uri_shares(self.uris["corrupt"])
4818 #cso = CorruptShareOptions()
4819 #cso.stdout = StringIO()
4820 #cso.parseOptions([c_shares[0][2]])
4822 d.addCallback(_clobber_shares)
4825 # root/good CHK, 10 shares
4827 # root/sick CHK, 9 shares
4829 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4831 units = [simplejson.loads(line)
4832 for line in res.splitlines()
4834 self.failUnlessReallyEqual(len(units), 4+1)
4835 # should be parent-first
4837 self.failUnlessEqual(u0["path"], [])
4838 self.failUnlessEqual(u0["type"], "directory")
4839 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4840 u0crr = u0["check-and-repair-results"]
4841 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4842 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4844 ugood = [u for u in units
4845 if u["type"] == "file" and u["path"] == [u"good"]][0]
4846 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4847 ugoodcrr = ugood["check-and-repair-results"]
4848 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4849 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4851 usick = [u for u in units
4852 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4853 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4854 usickcrr = usick["check-and-repair-results"]
4855 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4856 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4857 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4858 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4861 self.failUnlessEqual(stats["type"], "stats")
4863 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4864 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4865 self.failUnlessReallyEqual(s["count-directories"], 1)
4866 d.addCallback(_done)
4868 d.addErrback(self.explain_web_error)
4871 def _count_leases(self, ignored, which):
4872 u = self.uris[which]
4873 shares = self.find_uri_shares(u)
4875 for shnum, serverid, fn in shares:
4876 sf = get_share_file(fn)
4877 num_leases = len(list(sf.get_leases()))
4878 lease_counts.append( (fn, num_leases) )
4881 def _assert_leasecount(self, lease_counts, expected):
4882 for (fn, num_leases) in lease_counts:
4883 if num_leases != expected:
4884 self.fail("expected %d leases, have %d, on %s" %
4885 (expected, num_leases, fn))
4887 def test_add_lease(self):
4888 self.basedir = "web/Grid/add_lease"
4889 self.set_up_grid(num_clients=2)
4890 c0 = self.g.clients[0]
4893 d = c0.upload(upload.Data(DATA, convergence=""))
4894 def _stash_uri(ur, which):
4895 self.uris[which] = ur.uri
4896 d.addCallback(_stash_uri, "one")
4897 d.addCallback(lambda ign:
4898 c0.upload(upload.Data(DATA+"1", convergence="")))
4899 d.addCallback(_stash_uri, "two")
4900 def _stash_mutable_uri(n, which):
4901 self.uris[which] = n.get_uri()
4902 assert isinstance(self.uris[which], str)
4903 d.addCallback(lambda ign:
4904 c0.create_mutable_file(publish.MutableData(DATA+"2")))
4905 d.addCallback(_stash_mutable_uri, "mutable")
4907 def _compute_fileurls(ignored):
4909 for which in self.uris:
4910 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4911 d.addCallback(_compute_fileurls)
4913 d.addCallback(self._count_leases, "one")
4914 d.addCallback(self._assert_leasecount, 1)
4915 d.addCallback(self._count_leases, "two")
4916 d.addCallback(self._assert_leasecount, 1)
4917 d.addCallback(self._count_leases, "mutable")
4918 d.addCallback(self._assert_leasecount, 1)
4920 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4921 def _got_html_good(res):
4922 self.failUnless("Healthy" in res, res)
4923 self.failIf("Not Healthy" in res, res)
4924 d.addCallback(_got_html_good)
4926 d.addCallback(self._count_leases, "one")
4927 d.addCallback(self._assert_leasecount, 1)
4928 d.addCallback(self._count_leases, "two")
4929 d.addCallback(self._assert_leasecount, 1)
4930 d.addCallback(self._count_leases, "mutable")
4931 d.addCallback(self._assert_leasecount, 1)
4933 # this CHECK uses the original client, which uses the same
4934 # lease-secrets, so it will just renew the original lease
4935 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4936 d.addCallback(_got_html_good)
4938 d.addCallback(self._count_leases, "one")
4939 d.addCallback(self._assert_leasecount, 1)
4940 d.addCallback(self._count_leases, "two")
4941 d.addCallback(self._assert_leasecount, 1)
4942 d.addCallback(self._count_leases, "mutable")
4943 d.addCallback(self._assert_leasecount, 1)
4945 # this CHECK uses an alternate client, which adds a second lease
4946 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4947 d.addCallback(_got_html_good)
4949 d.addCallback(self._count_leases, "one")
4950 d.addCallback(self._assert_leasecount, 2)
4951 d.addCallback(self._count_leases, "two")
4952 d.addCallback(self._assert_leasecount, 1)
4953 d.addCallback(self._count_leases, "mutable")
4954 d.addCallback(self._assert_leasecount, 1)
4956 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4957 d.addCallback(_got_html_good)
4959 d.addCallback(self._count_leases, "one")
4960 d.addCallback(self._assert_leasecount, 2)
4961 d.addCallback(self._count_leases, "two")
4962 d.addCallback(self._assert_leasecount, 1)
4963 d.addCallback(self._count_leases, "mutable")
4964 d.addCallback(self._assert_leasecount, 1)
4966 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4968 d.addCallback(_got_html_good)
4970 d.addCallback(self._count_leases, "one")
4971 d.addCallback(self._assert_leasecount, 2)
4972 d.addCallback(self._count_leases, "two")
4973 d.addCallback(self._assert_leasecount, 1)
4974 d.addCallback(self._count_leases, "mutable")
4975 d.addCallback(self._assert_leasecount, 2)
4977 d.addErrback(self.explain_web_error)
4980 def test_deep_add_lease(self):
4981 self.basedir = "web/Grid/deep_add_lease"
4982 self.set_up_grid(num_clients=2)
4983 c0 = self.g.clients[0]
4987 d = c0.create_dirnode()
4988 def _stash_root_and_create_file(n):
4990 self.uris["root"] = n.get_uri()
4991 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4992 return n.add_file(u"one", upload.Data(DATA, convergence=""))
4993 d.addCallback(_stash_root_and_create_file)
4994 def _stash_uri(fn, which):
4995 self.uris[which] = fn.get_uri()
4996 d.addCallback(_stash_uri, "one")
4997 d.addCallback(lambda ign:
4998 self.rootnode.add_file(u"small",
4999 upload.Data("literal",
5001 d.addCallback(_stash_uri, "small")
5003 d.addCallback(lambda ign:
5004 c0.create_mutable_file(publish.MutableData("mutable")))
5005 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5006 d.addCallback(_stash_uri, "mutable")
5008 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5010 units = [simplejson.loads(line)
5011 for line in res.splitlines()
5013 # root, one, small, mutable, stats
5014 self.failUnlessReallyEqual(len(units), 4+1)
5015 d.addCallback(_done)
5017 d.addCallback(self._count_leases, "root")
5018 d.addCallback(self._assert_leasecount, 1)
5019 d.addCallback(self._count_leases, "one")
5020 d.addCallback(self._assert_leasecount, 1)
5021 d.addCallback(self._count_leases, "mutable")
5022 d.addCallback(self._assert_leasecount, 1)
5024 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5025 d.addCallback(_done)
5027 d.addCallback(self._count_leases, "root")
5028 d.addCallback(self._assert_leasecount, 1)
5029 d.addCallback(self._count_leases, "one")
5030 d.addCallback(self._assert_leasecount, 1)
5031 d.addCallback(self._count_leases, "mutable")
5032 d.addCallback(self._assert_leasecount, 1)
5034 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5036 d.addCallback(_done)
5038 d.addCallback(self._count_leases, "root")
5039 d.addCallback(self._assert_leasecount, 2)
5040 d.addCallback(self._count_leases, "one")
5041 d.addCallback(self._assert_leasecount, 2)
5042 d.addCallback(self._count_leases, "mutable")
5043 d.addCallback(self._assert_leasecount, 2)
5045 d.addErrback(self.explain_web_error)
5049 def test_exceptions(self):
5050 self.basedir = "web/Grid/exceptions"
5051 self.set_up_grid(num_clients=1, num_servers=2)
5052 c0 = self.g.clients[0]
5053 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5056 d = c0.create_dirnode()
5058 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5059 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5061 d.addCallback(_stash_root)
5062 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5064 self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5065 self.delete_shares_numbered(ur.uri, range(1,10))
5067 u = uri.from_string(ur.uri)
5068 u.key = testutil.flip_bit(u.key, 0)
5069 baduri = u.to_string()
5070 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5071 d.addCallback(_stash_bad)
5072 d.addCallback(lambda ign: c0.create_dirnode())
5073 def _mangle_dirnode_1share(n):
5075 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5076 self.fileurls["dir-1share-json"] = url + "?t=json"
5077 self.delete_shares_numbered(u, range(1,10))
5078 d.addCallback(_mangle_dirnode_1share)
5079 d.addCallback(lambda ign: c0.create_dirnode())
5080 def _mangle_dirnode_0share(n):
5082 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5083 self.fileurls["dir-0share-json"] = url + "?t=json"
5084 self.delete_shares_numbered(u, range(0,10))
5085 d.addCallback(_mangle_dirnode_0share)
5087 # NotEnoughSharesError should be reported sensibly, with a
5088 # text/plain explanation of the problem, and perhaps some
5089 # information on which shares *could* be found.
5091 d.addCallback(lambda ignored:
5092 self.shouldHTTPError("GET unrecoverable",
5093 410, "Gone", "NoSharesError",
5094 self.GET, self.fileurls["0shares"]))
5095 def _check_zero_shares(body):
5096 self.failIf("<html>" in body, body)
5097 body = " ".join(body.strip().split())
5098 exp = ("NoSharesError: no shares could be found. "
5099 "Zero shares usually indicates a corrupt URI, or that "
5100 "no servers were connected, but it might also indicate "
5101 "severe corruption. You should perform a filecheck on "
5102 "this object to learn more. The full error message is: "
5103 "no shares (need 3). Last failure: None")
5104 self.failUnlessReallyEqual(exp, body)
5105 d.addCallback(_check_zero_shares)
5108 d.addCallback(lambda ignored:
5109 self.shouldHTTPError("GET 1share",
5110 410, "Gone", "NotEnoughSharesError",
5111 self.GET, self.fileurls["1share"]))
5112 def _check_one_share(body):
5113 self.failIf("<html>" in body, body)
5114 body = " ".join(body.strip().split())
5115 msgbase = ("NotEnoughSharesError: This indicates that some "
5116 "servers were unavailable, or that shares have been "
5117 "lost to server departure, hard drive failure, or disk "
5118 "corruption. You should perform a filecheck on "
5119 "this object to learn more. The full error message is:"
5121 msg1 = msgbase + (" ran out of shares:"
5124 " overdue= unused= need 3. Last failure: None")
5125 msg2 = msgbase + (" ran out of shares:"
5127 " pending=Share(sh0-on-xgru5)"
5128 " overdue= unused= need 3. Last failure: None")
5129 self.failUnless(body == msg1 or body == msg2, body)
5130 d.addCallback(_check_one_share)
5132 d.addCallback(lambda ignored:
5133 self.shouldHTTPError("GET imaginary",
5134 404, "Not Found", None,
5135 self.GET, self.fileurls["imaginary"]))
5136 def _missing_child(body):
5137 self.failUnless("No such child: imaginary" in body, body)
5138 d.addCallback(_missing_child)
5140 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5141 def _check_0shares_dir_html(body):
5142 self.failUnless("<html>" in body, body)
5143 # we should see the regular page, but without the child table or
5145 body = " ".join(body.strip().split())
5146 self.failUnlessIn('href="?t=info">More info on this directory',
5148 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5149 "could not be retrieved, because there were insufficient "
5150 "good shares. This might indicate that no servers were "
5151 "connected, insufficient servers were connected, the URI "
5152 "was corrupt, or that shares have been lost due to server "
5153 "departure, hard drive failure, or disk corruption. You "
5154 "should perform a filecheck on this object to learn more.")
5155 self.failUnlessIn(exp, body)
5156 self.failUnlessIn("No upload forms: directory is unreadable", body)
5157 d.addCallback(_check_0shares_dir_html)
5159 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5160 def _check_1shares_dir_html(body):
5161 # at some point, we'll split UnrecoverableFileError into 0-shares
5162 # and some-shares like we did for immutable files (since there
5163 # are different sorts of advice to offer in each case). For now,
5164 # they present the same way.
5165 self.failUnless("<html>" in body, body)
5166 body = " ".join(body.strip().split())
5167 self.failUnlessIn('href="?t=info">More info on this directory',
5169 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5170 "could not be retrieved, because there were insufficient "
5171 "good shares. This might indicate that no servers were "
5172 "connected, insufficient servers were connected, the URI "
5173 "was corrupt, or that shares have been lost due to server "
5174 "departure, hard drive failure, or disk corruption. You "
5175 "should perform a filecheck on this object to learn more.")
5176 self.failUnlessIn(exp, body)
5177 self.failUnlessIn("No upload forms: directory is unreadable", body)
5178 d.addCallback(_check_1shares_dir_html)
5180 d.addCallback(lambda ignored:
5181 self.shouldHTTPError("GET dir-0share-json",
5182 410, "Gone", "UnrecoverableFileError",
5184 self.fileurls["dir-0share-json"]))
5185 def _check_unrecoverable_file(body):
5186 self.failIf("<html>" in body, body)
5187 body = " ".join(body.strip().split())
5188 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5189 "could not be retrieved, because there were insufficient "
5190 "good shares. This might indicate that no servers were "
5191 "connected, insufficient servers were connected, the URI "
5192 "was corrupt, or that shares have been lost due to server "
5193 "departure, hard drive failure, or disk corruption. You "
5194 "should perform a filecheck on this object to learn more.")
5195 self.failUnlessReallyEqual(exp, body)
5196 d.addCallback(_check_unrecoverable_file)
5198 d.addCallback(lambda ignored:
5199 self.shouldHTTPError("GET dir-1share-json",
5200 410, "Gone", "UnrecoverableFileError",
5202 self.fileurls["dir-1share-json"]))
5203 d.addCallback(_check_unrecoverable_file)
5205 d.addCallback(lambda ignored:
5206 self.shouldHTTPError("GET imaginary",
5207 404, "Not Found", None,
5208 self.GET, self.fileurls["imaginary"]))
5210 # attach a webapi child that throws a random error, to test how it
5212 w = c0.getServiceNamed("webish")
5213 w.root.putChild("ERRORBOOM", ErrorBoom())
5215 # "Accept: */*" : should get a text/html stack trace
5216 # "Accept: text/plain" : should get a text/plain stack trace
5217 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5218 # no Accept header: should get a text/html stack trace
5220 d.addCallback(lambda ignored:
5221 self.shouldHTTPError("GET errorboom_html",
5222 500, "Internal Server Error", None,
5223 self.GET, "ERRORBOOM",
5224 headers={"accept": ["*/*"]}))
5225 def _internal_error_html1(body):
5226 self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5227 d.addCallback(_internal_error_html1)
5229 d.addCallback(lambda ignored:
5230 self.shouldHTTPError("GET errorboom_text",
5231 500, "Internal Server Error", None,
5232 self.GET, "ERRORBOOM",
5233 headers={"accept": ["text/plain"]}))
5234 def _internal_error_text2(body):
5235 self.failIf("<html>" in body, body)
5236 self.failUnless(body.startswith("Traceback "), body)
5237 d.addCallback(_internal_error_text2)
5239 CLI_accepts = "text/plain, application/octet-stream"
5240 d.addCallback(lambda ignored:
5241 self.shouldHTTPError("GET errorboom_text",
5242 500, "Internal Server Error", None,
5243 self.GET, "ERRORBOOM",
5244 headers={"accept": [CLI_accepts]}))
5245 def _internal_error_text3(body):
5246 self.failIf("<html>" in body, body)
5247 self.failUnless(body.startswith("Traceback "), body)
5248 d.addCallback(_internal_error_text3)
5250 d.addCallback(lambda ignored:
5251 self.shouldHTTPError("GET errorboom_text",
5252 500, "Internal Server Error", None,
5253 self.GET, "ERRORBOOM"))
5254 def _internal_error_html4(body):
5255 self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5256 d.addCallback(_internal_error_html4)
5258 def _flush_errors(res):
5259 # Trial: please ignore the CompletelyUnhandledError in the logs
5260 self.flushLoggedErrors(CompletelyUnhandledError)
5262 d.addBoth(_flush_errors)
5266 def test_blacklist(self):
5267 # download from a blacklisted URI, get an error
5268 self.basedir = "web/Grid/blacklist"
5270 c0 = self.g.clients[0]
5271 c0_basedir = c0.basedir
5272 fn = os.path.join(c0_basedir, "access.blacklist")
5274 DATA = "off-limits " * 50
5276 d = c0.upload(upload.Data(DATA, convergence=""))
5277 def _stash_uri_and_create_dir(ur):
5279 self.url = "uri/"+self.uri
5280 u = uri.from_string_filenode(self.uri)
5281 self.si = u.get_storage_index()
5282 childnode = c0.create_node_from_uri(self.uri, None)
5283 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5284 d.addCallback(_stash_uri_and_create_dir)
5285 def _stash_dir(node):
5286 self.dir_node = node
5287 self.dir_uri = node.get_uri()
5288 self.dir_url = "uri/"+self.dir_uri
5289 d.addCallback(_stash_dir)
5290 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5291 def _check_dir_html(body):
5292 self.failUnlessIn("<html>", body)
5293 self.failUnlessIn("blacklisted.txt</a>", body)
5294 d.addCallback(_check_dir_html)
5295 d.addCallback(lambda ign: self.GET(self.url))
5296 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5298 def _blacklist(ign):
5300 f.write(" # this is a comment\n")
5302 f.write("\n") # also exercise blank lines
5303 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5305 # clients should be checking the blacklist each time, so we don't
5306 # need to restart the client
5307 d.addCallback(_blacklist)
5308 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5310 "Access Prohibited: off-limits",
5311 self.GET, self.url))
5313 # We should still be able to list the parent directory, in HTML...
5314 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5315 def _check_dir_html2(body):
5316 self.failUnlessIn("<html>", body)
5317 self.failUnlessIn("blacklisted.txt</strike>", body)
5318 d.addCallback(_check_dir_html2)
5320 # ... and in JSON (used by CLI).
5321 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5322 def _check_dir_json(res):
5323 data = simplejson.loads(res)
5324 self.failUnless(isinstance(data, list), data)
5325 self.failUnlessEqual(data[0], "dirnode")
5326 self.failUnless(isinstance(data[1], dict), data)
5327 self.failUnlessIn("children", data[1])
5328 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5329 childdata = data[1]["children"]["blacklisted.txt"]
5330 self.failUnless(isinstance(childdata, list), data)
5331 self.failUnlessEqual(childdata[0], "filenode")
5332 self.failUnless(isinstance(childdata[1], dict), data)
5333 d.addCallback(_check_dir_json)
5335 def _unblacklist(ign):
5336 open(fn, "w").close()
5337 # the Blacklist object watches mtime to tell when the file has
5338 # changed, but on windows this test will run faster than the
5339 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5340 # to force a reload.
5341 self.g.clients[0].blacklist.last_mtime -= 2.0
5342 d.addCallback(_unblacklist)
5344 # now a read should work
5345 d.addCallback(lambda ign: self.GET(self.url))
5346 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5348 # read again to exercise the blacklist-is-unchanged logic
5349 d.addCallback(lambda ign: self.GET(self.url))
5350 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5352 # now add a blacklisted directory, and make sure files under it are
5355 childnode = c0.create_node_from_uri(self.uri, None)
5356 return c0.create_dirnode({u"child": (childnode,{}) })
5357 d.addCallback(_add_dir)
5358 def _get_dircap(dn):
5359 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5360 self.dir_url_base = "uri/"+dn.get_write_uri()
5361 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5362 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5363 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5364 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5365 d.addCallback(_get_dircap)
5366 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5367 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5368 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5369 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5370 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5371 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5372 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5373 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5374 d.addCallback(lambda ign: self.GET(self.child_url))
5375 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5377 def _block_dir(ign):
5379 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5381 self.g.clients[0].blacklist.last_mtime -= 2.0
5382 d.addCallback(_block_dir)
5383 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5385 "Access Prohibited: dir-off-limits",
5386 self.GET, self.dir_url_base))
5387 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5389 "Access Prohibited: dir-off-limits",
5390 self.GET, self.dir_url_json1))
5391 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5393 "Access Prohibited: dir-off-limits",
5394 self.GET, self.dir_url_json2))
5395 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5397 "Access Prohibited: dir-off-limits",
5398 self.GET, self.dir_url_json_ro))
5399 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5401 "Access Prohibited: dir-off-limits",
5402 self.GET, self.child_url))
5406 class CompletelyUnhandledError(Exception):
5408 class ErrorBoom(rend.Page):
5409 def beforeRender(self, ctx):
5410 raise CompletelyUnhandledError("whoops")