2 import os.path, re, urllib, time
4 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
11 from nevow import rend
12 from allmydata import interfaces, uri, webish, dirnode
13 from allmydata.storage.shares import get_share_file
14 from allmydata.storage_client import StorageFarmBroker
15 from allmydata.immutable import upload
16 from allmydata.immutable.downloader.status import DownloadStatus
17 from allmydata.dirnode import DirectoryNode
18 from allmydata.nodemaker import NodeMaker
19 from allmydata.unknown import UnknownNode
20 from allmydata.web import status, common
21 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
22 from allmydata.util import fileutil, base32, hashutil
23 from allmydata.util.consumer import download_to_data
24 from allmydata.util.netstring import split_netstring
25 from allmydata.util.encodingutil import to_str
26 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
27 create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
28 from allmydata.interfaces import IMutableFileNode
29 from allmydata.mutable import servermap, publish, retrieve
30 import allmydata.test.common_util as testutil
31 from allmydata.test.no_network import GridTestMixin
32 from allmydata.test.common_web import HTTPClientGETFactory, \
34 from allmydata.client import Client, SecretHolder
36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
37 # create a webserver that works against them
39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
45 class FakeStatsProvider:
47 stats = {'stats': {}, 'counters': {}}
50 class FakeNodeMaker(NodeMaker):
51 def _create_lit(self, cap):
52 return FakeCHKFileNode(cap)
53 def _create_immutable(self, cap):
54 return FakeCHKFileNode(cap)
55 def _create_mutable(self, cap):
56 return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
57 def create_mutable_file(self, contents="", keysize=None):
58 n = FakeMutableFileNode(None, None, None, None)
59 return n.create(contents)
61 class FakeUploader(service.Service):
63 def upload(self, uploadable, history=None):
64 d = uploadable.get_size()
65 d.addCallback(lambda size: uploadable.read(size))
68 n = create_chk_filenode(data)
69 results = upload.UploadResults()
70 results.uri = n.get_uri()
72 d.addCallback(_got_data)
74 def get_helper_info(self):
78 def __init__(self, binaryserverid):
79 self.binaryserverid = binaryserverid
80 def get_name(self): return "short"
81 def get_longname(self): return "long"
82 def get_serverid(self): return self.binaryserverid
85 ds = DownloadStatus("storage_index", 1234)
88 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
89 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
90 storage_index = hashutil.storage_index_hash("SI")
91 e0 = ds.add_segment_request(0, now)
93 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
94 e1 = ds.add_segment_request(1, now+2)
96 # two outstanding requests
97 e2 = ds.add_segment_request(2, now+4)
98 e3 = ds.add_segment_request(3, now+5)
99 del e2,e3 # hush pyflakes
101 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
102 e = ds.add_segment_request(4, now)
104 e.deliver(now, 0, 140, 0.5)
106 e = ds.add_dyhb_request(serverA, now)
107 e.finished([1,2], now+1)
108 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
110 e = ds.add_read_event(0, 120, now)
111 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
113 e = ds.add_read_event(120, 30, now+2) # left unfinished
115 e = ds.add_block_request(serverA, 1, 100, 20, now)
116 e.finished(20, now+1)
117 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
119 # make sure that add_read_event() can come first too
120 ds1 = DownloadStatus(storage_index, 1234)
121 e = ds1.add_read_event(0, 120, now)
122 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
128 _all_upload_status = [upload.UploadStatus()]
129 _all_download_status = [build_one_ds()]
130 _all_mapupdate_statuses = [servermap.UpdateStatus()]
131 _all_publish_statuses = [publish.PublishStatus()]
132 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
134 def list_all_upload_statuses(self):
135 return self._all_upload_status
136 def list_all_download_statuses(self):
137 return self._all_download_status
138 def list_all_mapupdate_statuses(self):
139 return self._all_mapupdate_statuses
140 def list_all_publish_statuses(self):
141 return self._all_publish_statuses
142 def list_all_retrieve_statuses(self):
143 return self._all_retrieve_statuses
144 def list_all_helper_statuses(self):
147 class FakeClient(Client):
149 # don't upcall to Client.__init__, since we only want to initialize a
151 service.MultiService.__init__(self)
152 self.nodeid = "fake_nodeid"
153 self.nickname = "fake_nickname"
154 self.introducer_furl = "None"
155 self.stats_provider = FakeStatsProvider()
156 self._secret_holder = SecretHolder("lease secret", "convergence secret")
158 self.convergence = "some random string"
159 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
160 self.introducer_client = None
161 self.history = FakeHistory()
162 self.uploader = FakeUploader()
163 self.uploader.setServiceParent(self)
164 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
168 def startService(self):
169 return service.MultiService.startService(self)
170 def stopService(self):
171 return service.MultiService.stopService(self)
173 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
175 class WebMixin(object):
177 self.s = FakeClient()
178 self.s.startService()
179 self.staticdir = self.mktemp()
181 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
183 self.ws.setServiceParent(self.s)
184 self.webish_port = self.ws.getPortnum()
185 self.webish_url = self.ws.getURL()
186 assert self.webish_url.endswith("/")
187 self.webish_url = self.webish_url[:-1] # these tests add their own /
189 l = [ self.s.create_dirnode() for x in range(6) ]
190 d = defer.DeferredList(l)
192 self.public_root = res[0][1]
193 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
194 self.public_url = "/uri/" + self.public_root.get_uri()
195 self.private_root = res[1][1]
199 self._foo_uri = foo.get_uri()
200 self._foo_readonly_uri = foo.get_readonly_uri()
201 self._foo_verifycap = foo.get_verify_cap().to_string()
202 # NOTE: we ignore the deferred on all set_uri() calls, because we
203 # know the fake nodes do these synchronously
204 self.public_root.set_uri(u"foo", foo.get_uri(),
205 foo.get_readonly_uri())
207 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
208 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
209 self._bar_txt_verifycap = n.get_verify_cap().to_string()
211 foo.set_uri(u"empty", res[3][1].get_uri(),
212 res[3][1].get_readonly_uri())
213 sub_uri = res[4][1].get_uri()
214 self._sub_uri = sub_uri
215 foo.set_uri(u"sub", sub_uri, sub_uri)
216 sub = self.s.create_node_from_uri(sub_uri)
218 _ign, n, blocking_uri = self.makefile(1)
219 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
221 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
222 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
223 # still think of it as an umlaut
224 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
226 _ign, n, baz_file = self.makefile(2)
227 self._baz_file_uri = baz_file
228 sub.set_uri(u"baz.txt", baz_file, baz_file)
230 _ign, n, self._bad_file_uri = self.makefile(3)
231 # this uri should not be downloadable
232 del FakeCHKFileNode.all_contents[self._bad_file_uri]
235 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
236 rodir.get_readonly_uri())
237 rodir.set_uri(u"nor", baz_file, baz_file)
242 # public/foo/blockingfile
245 # public/foo/sub/baz.txt
247 # public/reedownlee/nor
248 self.NEWFILE_CONTENTS = "newfile contents\n"
250 return foo.get_metadata_for(u"bar.txt")
252 def _got_metadata(metadata):
253 self._bar_txt_metadata = metadata
254 d.addCallback(_got_metadata)
257 def makefile(self, number):
258 contents = "contents of file %s\n" % number
259 n = create_chk_filenode(contents)
260 return contents, n, n.get_uri()
263 return self.s.stopService()
265 def failUnlessIsBarDotTxt(self, res):
266 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
268 def failUnlessIsBarJSON(self, res):
269 data = simplejson.loads(res)
270 self.failUnless(isinstance(data, list))
271 self.failUnlessEqual(data[0], "filenode")
272 self.failUnless(isinstance(data[1], dict))
273 self.failIf(data[1]["mutable"])
274 self.failIf("rw_uri" in data[1]) # immutable
275 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
276 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
277 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
279 def failUnlessIsFooJSON(self, res):
280 data = simplejson.loads(res)
281 self.failUnless(isinstance(data, list))
282 self.failUnlessEqual(data[0], "dirnode", res)
283 self.failUnless(isinstance(data[1], dict))
284 self.failUnless(data[1]["mutable"])
285 self.failUnless("rw_uri" in data[1]) # mutable
286 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
287 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
288 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
290 kidnames = sorted([unicode(n) for n in data[1]["children"]])
291 self.failUnlessEqual(kidnames,
292 [u"bar.txt", u"blockingfile", u"empty",
293 u"n\u00fc.txt", u"sub"])
294 kids = dict( [(unicode(name),value)
296 in data[1]["children"].iteritems()] )
297 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
298 self.failUnlessIn("metadata", kids[u"sub"][1])
299 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
300 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
301 self.failUnlessIn("linkcrtime", tahoe_md)
302 self.failUnlessIn("linkmotime", tahoe_md)
303 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
304 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
305 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
306 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
307 self._bar_txt_verifycap)
308 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
309 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
310 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
311 self._bar_txt_metadata["tahoe"]["linkcrtime"])
312 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
315 def GET(self, urlpath, followRedirect=False, return_response=False,
317 # if return_response=True, this fires with (data, statuscode,
318 # respheaders) instead of just data.
319 assert not isinstance(urlpath, unicode)
320 url = self.webish_url + urlpath
321 factory = HTTPClientGETFactory(url, method="GET",
322 followRedirect=followRedirect, **kwargs)
323 reactor.connectTCP("localhost", self.webish_port, factory)
326 return (data, factory.status, factory.response_headers)
328 d.addCallback(_got_data)
329 return factory.deferred
331 def HEAD(self, urlpath, return_response=False, **kwargs):
332 # this requires some surgery, because twisted.web.client doesn't want
333 # to give us back the response headers.
334 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
335 reactor.connectTCP("localhost", self.webish_port, factory)
338 return (data, factory.status, factory.response_headers)
340 d.addCallback(_got_data)
341 return factory.deferred
343 def PUT(self, urlpath, data, **kwargs):
344 url = self.webish_url + urlpath
345 return client.getPage(url, method="PUT", postdata=data, **kwargs)
347 def DELETE(self, urlpath):
348 url = self.webish_url + urlpath
349 return client.getPage(url, method="DELETE")
351 def POST(self, urlpath, followRedirect=False, **fields):
352 sepbase = "boogabooga"
356 form.append('Content-Disposition: form-data; name="_charset"')
360 for name, value in fields.iteritems():
361 if isinstance(value, tuple):
362 filename, value = value
363 form.append('Content-Disposition: form-data; name="%s"; '
364 'filename="%s"' % (name, filename.encode("utf-8")))
366 form.append('Content-Disposition: form-data; name="%s"' % name)
368 if isinstance(value, unicode):
369 value = value.encode("utf-8")
372 assert isinstance(value, str)
379 body = "\r\n".join(form) + "\r\n"
380 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
381 return self.POST2(urlpath, body, headers, followRedirect)
383 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
384 url = self.webish_url + urlpath
385 return client.getPage(url, method="POST", postdata=body,
386 headers=headers, followRedirect=followRedirect)
388 def shouldFail(self, res, expected_failure, which,
389 substring=None, response_substring=None):
390 if isinstance(res, failure.Failure):
391 res.trap(expected_failure)
393 self.failUnless(substring in str(res),
394 "substring '%s' not in '%s'"
395 % (substring, str(res)))
396 if response_substring:
397 self.failUnless(response_substring in res.value.response,
398 "response substring '%s' not in '%s'"
399 % (response_substring, res.value.response))
401 self.fail("%s was supposed to raise %s, not get '%s'" %
402 (which, expected_failure, res))
404 def shouldFail2(self, expected_failure, which, substring,
406 callable, *args, **kwargs):
407 assert substring is None or isinstance(substring, str)
408 assert response_substring is None or isinstance(response_substring, str)
409 d = defer.maybeDeferred(callable, *args, **kwargs)
411 if isinstance(res, failure.Failure):
412 res.trap(expected_failure)
414 self.failUnless(substring in str(res),
415 "%s: substring '%s' not in '%s'"
416 % (which, substring, str(res)))
417 if response_substring:
418 self.failUnless(response_substring in res.value.response,
419 "%s: response substring '%s' not in '%s'"
421 response_substring, res.value.response))
423 self.fail("%s was supposed to raise %s, not get '%s'" %
424 (which, expected_failure, res))
428 def should404(self, res, which):
429 if isinstance(res, failure.Failure):
430 res.trap(error.Error)
431 self.failUnlessReallyEqual(res.value.status, "404")
433 self.fail("%s was supposed to Error(404), not get '%s'" %
436 def should302(self, res, which):
437 if isinstance(res, failure.Failure):
438 res.trap(error.Error)
439 self.failUnlessReallyEqual(res.value.status, "302")
441 self.fail("%s was supposed to Error(302), not get '%s'" %
445 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
446 def test_create(self):
449 def test_welcome(self):
452 self.failUnless('Welcome To Tahoe-LAFS' in res, res)
454 self.s.basedir = 'web/test_welcome'
455 fileutil.make_dirs("web/test_welcome")
456 fileutil.make_dirs("web/test_welcome/private")
458 d.addCallback(_check)
461 def test_provisioning(self):
462 d = self.GET("/provisioning/")
464 self.failUnless('Provisioning Tool' in res)
465 fields = {'filled': True,
466 "num_users": int(50e3),
467 "files_per_user": 1000,
468 "space_per_user": int(1e9),
469 "sharing_ratio": 1.0,
470 "encoding_parameters": "3-of-10-5",
472 "ownership_mode": "A",
473 "download_rate": 100,
478 return self.POST("/provisioning/", **fields)
480 d.addCallback(_check)
482 self.failUnless('Provisioning Tool' in res)
483 self.failUnless("Share space consumed: 167.01TB" in res)
485 fields = {'filled': True,
486 "num_users": int(50e6),
487 "files_per_user": 1000,
488 "space_per_user": int(5e9),
489 "sharing_ratio": 1.0,
490 "encoding_parameters": "25-of-100-50",
491 "num_servers": 30000,
492 "ownership_mode": "E",
493 "drive_failure_model": "U",
495 "download_rate": 1000,
500 return self.POST("/provisioning/", **fields)
501 d.addCallback(_check2)
503 self.failUnless("Share space consumed: huge!" in res)
504 fields = {'filled': True}
505 return self.POST("/provisioning/", **fields)
506 d.addCallback(_check3)
508 self.failUnless("Share space consumed:" in res)
509 d.addCallback(_check4)
512 def test_reliability_tool(self):
514 from allmydata import reliability
515 _hush_pyflakes = reliability
518 raise unittest.SkipTest("reliability tool requires NumPy")
520 d = self.GET("/reliability/")
522 self.failUnless('Reliability Tool' in res)
523 fields = {'drive_lifetime': "8Y",
528 "check_period": "1M",
529 "report_period": "3M",
532 return self.POST("/reliability/", **fields)
534 d.addCallback(_check)
536 self.failUnless('Reliability Tool' in res)
537 r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
538 self.failUnless(re.search(r, res), res)
539 d.addCallback(_check2)
542 def test_status(self):
543 h = self.s.get_history()
544 dl_num = h.list_all_download_statuses()[0].get_counter()
545 ul_num = h.list_all_upload_statuses()[0].get_counter()
546 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
547 pub_num = h.list_all_publish_statuses()[0].get_counter()
548 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
549 d = self.GET("/status", followRedirect=True)
551 self.failUnless('Upload and Download Status' in res, res)
552 self.failUnless('"down-%d"' % dl_num in res, res)
553 self.failUnless('"up-%d"' % ul_num in res, res)
554 self.failUnless('"mapupdate-%d"' % mu_num in res, res)
555 self.failUnless('"publish-%d"' % pub_num in res, res)
556 self.failUnless('"retrieve-%d"' % ret_num in res, res)
557 d.addCallback(_check)
558 d.addCallback(lambda res: self.GET("/status/?t=json"))
559 def _check_json(res):
560 data = simplejson.loads(res)
561 self.failUnless(isinstance(data, dict))
562 #active = data["active"]
563 # TODO: test more. We need a way to fake an active operation
565 d.addCallback(_check_json)
567 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
569 self.failUnless("File Download Status" in res, res)
570 d.addCallback(_check_dl)
571 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
572 def _check_dl_json(res):
573 data = simplejson.loads(res)
574 self.failUnless(isinstance(data, dict))
575 self.failUnless("read" in data)
576 self.failUnlessEqual(data["read"][0]["length"], 120)
577 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
578 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
579 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
580 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
581 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
582 # serverids[] keys are strings, since that's what JSON does, but
583 # we'd really like them to be ints
584 self.failUnlessEqual(data["serverids"]["0"], "phwr")
585 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
586 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
587 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
588 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
589 self.failUnless("dyhb" in data)
590 d.addCallback(_check_dl_json)
591 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
593 self.failUnless("File Upload Status" in res, res)
594 d.addCallback(_check_ul)
595 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
596 def _check_mapupdate(res):
597 self.failUnless("Mutable File Servermap Update Status" in res, res)
598 d.addCallback(_check_mapupdate)
599 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
600 def _check_publish(res):
601 self.failUnless("Mutable File Publish Status" in res, res)
602 d.addCallback(_check_publish)
603 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
604 def _check_retrieve(res):
605 self.failUnless("Mutable File Retrieve Status" in res, res)
606 d.addCallback(_check_retrieve)
610 def test_status_numbers(self):
611 drrm = status.DownloadResultsRendererMixin()
612 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
613 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
614 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
615 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
616 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
617 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
618 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
619 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
620 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
622 urrm = status.UploadResultsRendererMixin()
623 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
624 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
625 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
626 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
627 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
628 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
629 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
630 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
631 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
633 def test_GET_FILEURL(self):
634 d = self.GET(self.public_url + "/foo/bar.txt")
635 d.addCallback(self.failUnlessIsBarDotTxt)
638 def test_GET_FILEURL_range(self):
639 headers = {"range": "bytes=1-10"}
640 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
641 return_response=True)
642 def _got((res, status, headers)):
643 self.failUnlessReallyEqual(int(status), 206)
644 self.failUnless(headers.has_key("content-range"))
645 self.failUnlessReallyEqual(headers["content-range"][0],
646 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
647 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
651 def test_GET_FILEURL_partial_range(self):
652 headers = {"range": "bytes=5-"}
653 length = len(self.BAR_CONTENTS)
654 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
655 return_response=True)
656 def _got((res, status, headers)):
657 self.failUnlessReallyEqual(int(status), 206)
658 self.failUnless(headers.has_key("content-range"))
659 self.failUnlessReallyEqual(headers["content-range"][0],
660 "bytes 5-%d/%d" % (length-1, length))
661 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
665 def test_GET_FILEURL_partial_end_range(self):
666 headers = {"range": "bytes=-5"}
667 length = len(self.BAR_CONTENTS)
668 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
669 return_response=True)
670 def _got((res, status, headers)):
671 self.failUnlessReallyEqual(int(status), 206)
672 self.failUnless(headers.has_key("content-range"))
673 self.failUnlessReallyEqual(headers["content-range"][0],
674 "bytes %d-%d/%d" % (length-5, length-1, length))
675 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
679 def test_GET_FILEURL_partial_range_overrun(self):
680 headers = {"range": "bytes=100-200"}
681 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
682 "416 Requested Range not satisfiable",
683 "First beyond end of file",
684 self.GET, self.public_url + "/foo/bar.txt",
688 def test_HEAD_FILEURL_range(self):
689 headers = {"range": "bytes=1-10"}
690 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
691 return_response=True)
692 def _got((res, status, headers)):
693 self.failUnlessReallyEqual(res, "")
694 self.failUnlessReallyEqual(int(status), 206)
695 self.failUnless(headers.has_key("content-range"))
696 self.failUnlessReallyEqual(headers["content-range"][0],
697 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
701 def test_HEAD_FILEURL_partial_range(self):
702 headers = {"range": "bytes=5-"}
703 length = len(self.BAR_CONTENTS)
704 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
705 return_response=True)
706 def _got((res, status, headers)):
707 self.failUnlessReallyEqual(int(status), 206)
708 self.failUnless(headers.has_key("content-range"))
709 self.failUnlessReallyEqual(headers["content-range"][0],
710 "bytes 5-%d/%d" % (length-1, length))
714 def test_HEAD_FILEURL_partial_end_range(self):
715 headers = {"range": "bytes=-5"}
716 length = len(self.BAR_CONTENTS)
717 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
718 return_response=True)
719 def _got((res, status, headers)):
720 self.failUnlessReallyEqual(int(status), 206)
721 self.failUnless(headers.has_key("content-range"))
722 self.failUnlessReallyEqual(headers["content-range"][0],
723 "bytes %d-%d/%d" % (length-5, length-1, length))
727 def test_HEAD_FILEURL_partial_range_overrun(self):
728 headers = {"range": "bytes=100-200"}
729 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
730 "416 Requested Range not satisfiable",
732 self.HEAD, self.public_url + "/foo/bar.txt",
736 def test_GET_FILEURL_range_bad(self):
737 headers = {"range": "BOGUS=fizbop-quarnak"}
738 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
739 return_response=True)
740 def _got((res, status, headers)):
741 self.failUnlessReallyEqual(int(status), 200)
742 self.failUnless(not headers.has_key("content-range"))
743 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
747 def test_HEAD_FILEURL(self):
748 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
749 def _got((res, status, headers)):
750 self.failUnlessReallyEqual(res, "")
751 self.failUnlessReallyEqual(headers["content-length"][0],
752 str(len(self.BAR_CONTENTS)))
753 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
757 def test_GET_FILEURL_named(self):
758 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
759 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
760 d = self.GET(base + "/@@name=/blah.txt")
761 d.addCallback(self.failUnlessIsBarDotTxt)
762 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
763 d.addCallback(self.failUnlessIsBarDotTxt)
764 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
765 d.addCallback(self.failUnlessIsBarDotTxt)
766 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
767 d.addCallback(self.failUnlessIsBarDotTxt)
768 save_url = base + "?save=true&filename=blah.txt"
769 d.addCallback(lambda res: self.GET(save_url))
770 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
771 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
772 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
773 u_url = base + "?save=true&filename=" + u_fn_e
774 d.addCallback(lambda res: self.GET(u_url))
775 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
778 def test_PUT_FILEURL_named_bad(self):
779 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
780 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
782 "/file can only be used with GET or HEAD",
783 self.PUT, base + "/@@name=/blah.txt", "")
786 def test_GET_DIRURL_named_bad(self):
787 base = "/file/%s" % urllib.quote(self._foo_uri)
788 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
791 self.GET, base + "/@@name=/blah.txt")
794 def test_GET_slash_file_bad(self):
795 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
797 "/file must be followed by a file-cap and a name",
801 def test_GET_unhandled_URI_named(self):
802 contents, n, newuri = self.makefile(12)
803 verifier_cap = n.get_verify_cap().to_string()
804 base = "/file/%s" % urllib.quote(verifier_cap)
805 # client.create_node_from_uri() can't handle verify-caps
806 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
807 "400 Bad Request", "is not a file-cap",
811 def test_GET_unhandled_URI(self):
812 contents, n, newuri = self.makefile(12)
813 verifier_cap = n.get_verify_cap().to_string()
814 base = "/uri/%s" % urllib.quote(verifier_cap)
815 # client.create_node_from_uri() can't handle verify-caps
816 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
818 "GET unknown URI type: can only do t=info",
822 def test_GET_FILE_URI(self):
823 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
825 d.addCallback(self.failUnlessIsBarDotTxt)
828 def test_GET_FILE_URI_badchild(self):
829 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
830 errmsg = "Files have no children, certainly not named 'boguschild'"
831 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
832 "400 Bad Request", errmsg,
836 def test_PUT_FILE_URI_badchild(self):
837 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
838 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
839 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
840 "400 Bad Request", errmsg,
844 # TODO: version of this with a Unicode filename
845 def test_GET_FILEURL_save(self):
846 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
847 return_response=True)
848 def _got((res, statuscode, headers)):
849 content_disposition = headers["content-disposition"][0]
850 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
851 self.failUnlessIsBarDotTxt(res)
855 def test_GET_FILEURL_missing(self):
856 d = self.GET(self.public_url + "/foo/missing")
857 d.addBoth(self.should404, "test_GET_FILEURL_missing")
860 def test_PUT_overwrite_only_files(self):
861 # create a directory, put a file in that directory.
862 contents, n, filecap = self.makefile(8)
863 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
864 d.addCallback(lambda res:
865 self.PUT(self.public_url + "/foo/dir/file1.txt",
866 self.NEWFILE_CONTENTS))
867 # try to overwrite the file with replace=only-files
869 d.addCallback(lambda res:
870 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
872 d.addCallback(lambda res:
873 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
874 "There was already a child by that name, and you asked me "
876 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
880 def test_PUT_NEWFILEURL(self):
881 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
882 # TODO: we lose the response code, so we can't check this
883 #self.failUnlessReallyEqual(responsecode, 201)
884 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
885 d.addCallback(lambda res:
886 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
887 self.NEWFILE_CONTENTS))
890 def test_PUT_NEWFILEURL_not_mutable(self):
891 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
892 self.NEWFILE_CONTENTS)
893 # TODO: we lose the response code, so we can't check this
894 #self.failUnlessReallyEqual(responsecode, 201)
895 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
896 d.addCallback(lambda res:
897 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
898 self.NEWFILE_CONTENTS))
901 def test_PUT_NEWFILEURL_range_bad(self):
902 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
903 target = self.public_url + "/foo/new.txt"
904 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
905 "501 Not Implemented",
906 "Content-Range in PUT not yet supported",
907 # (and certainly not for immutable files)
908 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
910 d.addCallback(lambda res:
911 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
914 def test_PUT_NEWFILEURL_mutable(self):
915 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
916 self.NEWFILE_CONTENTS)
917 # TODO: we lose the response code, so we can't check this
918 #self.failUnlessReallyEqual(responsecode, 201)
920 u = uri.from_string_mutable_filenode(res)
921 self.failUnless(u.is_mutable())
922 self.failIf(u.is_readonly())
924 d.addCallback(_check_uri)
925 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
926 d.addCallback(lambda res:
927 self.failUnlessMutableChildContentsAre(self._foo_node,
929 self.NEWFILE_CONTENTS))
932 def test_PUT_NEWFILEURL_mutable_toobig(self):
933 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
934 "413 Request Entity Too Large",
935 "SDMF is limited to one segment, and 10001 > 10000",
937 self.public_url + "/foo/new.txt?mutable=true",
938 "b" * (self.s.MUTABLE_SIZELIMIT+1))
941 def test_PUT_NEWFILEURL_replace(self):
942 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
943 # TODO: we lose the response code, so we can't check this
944 #self.failUnlessReallyEqual(responsecode, 200)
945 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
946 d.addCallback(lambda res:
947 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
948 self.NEWFILE_CONTENTS))
951 def test_PUT_NEWFILEURL_bad_t(self):
952 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
953 "PUT to a file: bad t=bogus",
954 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
958 def test_PUT_NEWFILEURL_no_replace(self):
959 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
960 self.NEWFILE_CONTENTS)
961 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
963 "There was already a child by that name, and you asked me "
967 def test_PUT_NEWFILEURL_mkdirs(self):
968 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
970 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
971 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
972 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
973 d.addCallback(lambda res:
974 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
975 self.NEWFILE_CONTENTS))
978 def test_PUT_NEWFILEURL_blocked(self):
979 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
980 self.NEWFILE_CONTENTS)
981 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
983 "Unable to create directory 'blockingfile': a file was in the way")
986 def test_PUT_NEWFILEURL_emptyname(self):
987 # an empty pathname component (i.e. a double-slash) is disallowed
988 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
990 "The webapi does not allow empty pathname components",
991 self.PUT, self.public_url + "/foo//new.txt", "")
994 def test_DELETE_FILEURL(self):
995 d = self.DELETE(self.public_url + "/foo/bar.txt")
996 d.addCallback(lambda res:
997 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1000 def test_DELETE_FILEURL_missing(self):
1001 d = self.DELETE(self.public_url + "/foo/missing")
1002 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1005 def test_DELETE_FILEURL_missing2(self):
1006 d = self.DELETE(self.public_url + "/missing/missing")
1007 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1010 def failUnlessHasBarDotTxtMetadata(self, res):
1011 data = simplejson.loads(res)
1012 self.failUnless(isinstance(data, list))
1013 self.failUnlessIn("metadata", data[1])
1014 self.failUnlessIn("tahoe", data[1]["metadata"])
1015 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1016 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1017 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1018 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1020 def test_GET_FILEURL_json(self):
1021 # twisted.web.http.parse_qs ignores any query args without an '=', so
1022 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1023 # instead. This may make it tricky to emulate the S3 interface
1025 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1027 self.failUnlessIsBarJSON(data)
1028 self.failUnlessHasBarDotTxtMetadata(data)
1030 d.addCallback(_check1)
1033 def test_GET_FILEURL_json_missing(self):
1034 d = self.GET(self.public_url + "/foo/missing?json")
1035 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1038 def test_GET_FILEURL_uri(self):
1039 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1041 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1042 d.addCallback(_check)
1043 d.addCallback(lambda res:
1044 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1046 # for now, for files, uris and readonly-uris are the same
1047 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1048 d.addCallback(_check2)
1051 def test_GET_FILEURL_badtype(self):
1052 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1055 self.public_url + "/foo/bar.txt?t=bogus")
1058 def test_CSS_FILE(self):
1059 d = self.GET("/tahoe_css", followRedirect=True)
1061 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1062 self.failUnless(CSS_STYLE.search(res), res)
1063 d.addCallback(_check)
1066 def test_GET_FILEURL_uri_missing(self):
1067 d = self.GET(self.public_url + "/foo/missing?t=uri")
1068 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1071 def test_GET_DIRECTORY_html_banner(self):
1072 d = self.GET(self.public_url + "/foo", followRedirect=True)
1074 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1075 d.addCallback(_check)
1078 def test_GET_DIRURL(self):
1079 # the addSlash means we get a redirect here
1080 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1082 d = self.GET(self.public_url + "/foo", followRedirect=True)
1084 self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1086 # the FILE reference points to a URI, but it should end in bar.txt
1087 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1088 (ROOT, urllib.quote(self._bar_txt_uri)))
1089 get_bar = "".join([r'<td>FILE</td>',
1091 r'<a href="%s">bar.txt</a>' % bar_url,
1093 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1095 self.failUnless(re.search(get_bar, res), res)
1096 for line in res.split("\n"):
1097 # find the line that contains the delete button for bar.txt
1098 if ("form action" in line and
1099 'value="delete"' in line and
1100 'value="bar.txt"' in line):
1101 # the form target should use a relative URL
1102 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1103 self.failUnless(('action="%s"' % foo_url) in line, line)
1104 # and the when_done= should too
1105 #done_url = urllib.quote(???)
1106 #self.failUnless(('name="when_done" value="%s"' % done_url)
1110 self.fail("unable to find delete-bar.txt line", res)
1112 # the DIR reference just points to a URI
1113 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1114 get_sub = ((r'<td>DIR</td>')
1115 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1116 self.failUnless(re.search(get_sub, res), res)
1117 d.addCallback(_check)
1119 # look at a readonly directory
1120 d.addCallback(lambda res:
1121 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1123 self.failUnless("(read-only)" in res, res)
1124 self.failIf("Upload a file" in res, res)
1125 d.addCallback(_check2)
1127 # and at a directory that contains a readonly directory
1128 d.addCallback(lambda res:
1129 self.GET(self.public_url, followRedirect=True))
1131 self.failUnless(re.search('<td>DIR-RO</td>'
1132 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1133 d.addCallback(_check3)
1135 # and an empty directory
1136 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1138 self.failUnless("directory is empty" in res, res)
1139 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1140 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1141 d.addCallback(_check4)
1143 # and at a literal directory
1144 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1145 d.addCallback(lambda res:
1146 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1148 self.failUnless('(immutable)' in res, res)
1149 self.failUnless(re.search('<td>FILE</td>'
1150 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1151 d.addCallback(_check5)
1154 def test_GET_DIRURL_badtype(self):
1155 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1159 self.public_url + "/foo?t=bogus")
1162 def test_GET_DIRURL_json(self):
1163 d = self.GET(self.public_url + "/foo?t=json")
1164 d.addCallback(self.failUnlessIsFooJSON)
1168 def test_POST_DIRURL_manifest_no_ophandle(self):
1169 d = self.shouldFail2(error.Error,
1170 "test_POST_DIRURL_manifest_no_ophandle",
1172 "slow operation requires ophandle=",
1173 self.POST, self.public_url, t="start-manifest")
1176 def test_POST_DIRURL_manifest(self):
1177 d = defer.succeed(None)
1178 def getman(ignored, output):
1179 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1180 followRedirect=True)
1181 d.addCallback(self.wait_for_operation, "125")
1182 d.addCallback(self.get_operation_results, "125", output)
1184 d.addCallback(getman, None)
1185 def _got_html(manifest):
1186 self.failUnless("Manifest of SI=" in manifest)
1187 self.failUnless("<td>sub</td>" in manifest)
1188 self.failUnless(self._sub_uri in manifest)
1189 self.failUnless("<td>sub/baz.txt</td>" in manifest)
1190 d.addCallback(_got_html)
1192 # both t=status and unadorned GET should be identical
1193 d.addCallback(lambda res: self.GET("/operations/125"))
1194 d.addCallback(_got_html)
1196 d.addCallback(getman, "html")
1197 d.addCallback(_got_html)
1198 d.addCallback(getman, "text")
1199 def _got_text(manifest):
1200 self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1201 self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1202 d.addCallback(_got_text)
1203 d.addCallback(getman, "JSON")
1205 data = res["manifest"]
1207 for (path_list, cap) in data:
1208 got[tuple(path_list)] = cap
1209 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1210 self.failUnless((u"sub",u"baz.txt") in got)
1211 self.failUnless("finished" in res)
1212 self.failUnless("origin" in res)
1213 self.failUnless("storage-index" in res)
1214 self.failUnless("verifycaps" in res)
1215 self.failUnless("stats" in res)
1216 d.addCallback(_got_json)
1219 def test_POST_DIRURL_deepsize_no_ophandle(self):
1220 d = self.shouldFail2(error.Error,
1221 "test_POST_DIRURL_deepsize_no_ophandle",
1223 "slow operation requires ophandle=",
1224 self.POST, self.public_url, t="start-deep-size")
1227 def test_POST_DIRURL_deepsize(self):
1228 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1229 followRedirect=True)
1230 d.addCallback(self.wait_for_operation, "126")
1231 d.addCallback(self.get_operation_results, "126", "json")
1232 def _got_json(data):
1233 self.failUnlessReallyEqual(data["finished"], True)
1235 self.failUnless(size > 1000)
1236 d.addCallback(_got_json)
1237 d.addCallback(self.get_operation_results, "126", "text")
1239 mo = re.search(r'^size: (\d+)$', res, re.M)
1240 self.failUnless(mo, res)
1241 size = int(mo.group(1))
1242 # with directories, the size varies.
1243 self.failUnless(size > 1000)
1244 d.addCallback(_got_text)
1247 def test_POST_DIRURL_deepstats_no_ophandle(self):
1248 d = self.shouldFail2(error.Error,
1249 "test_POST_DIRURL_deepstats_no_ophandle",
1251 "slow operation requires ophandle=",
1252 self.POST, self.public_url, t="start-deep-stats")
1255 def test_POST_DIRURL_deepstats(self):
1256 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1257 followRedirect=True)
1258 d.addCallback(self.wait_for_operation, "127")
1259 d.addCallback(self.get_operation_results, "127", "json")
1260 def _got_json(stats):
1261 expected = {"count-immutable-files": 3,
1262 "count-mutable-files": 0,
1263 "count-literal-files": 0,
1265 "count-directories": 3,
1266 "size-immutable-files": 57,
1267 "size-literal-files": 0,
1268 #"size-directories": 1912, # varies
1269 #"largest-directory": 1590,
1270 "largest-directory-children": 5,
1271 "largest-immutable-file": 19,
1273 for k,v in expected.iteritems():
1274 self.failUnlessReallyEqual(stats[k], v,
1275 "stats[%s] was %s, not %s" %
1277 self.failUnlessReallyEqual(stats["size-files-histogram"],
1279 d.addCallback(_got_json)
1282 def test_POST_DIRURL_stream_manifest(self):
1283 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1285 self.failUnless(res.endswith("\n"))
1286 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1287 self.failUnlessReallyEqual(len(units), 7)
1288 self.failUnlessEqual(units[-1]["type"], "stats")
1290 self.failUnlessEqual(first["path"], [])
1291 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1292 self.failUnlessEqual(first["type"], "directory")
1293 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1294 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1295 self.failIfEqual(baz["storage-index"], None)
1296 self.failIfEqual(baz["verifycap"], None)
1297 self.failIfEqual(baz["repaircap"], None)
1299 d.addCallback(_check)
1302 def test_GET_DIRURL_uri(self):
1303 d = self.GET(self.public_url + "/foo?t=uri")
1305 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1306 d.addCallback(_check)
1309 def test_GET_DIRURL_readonly_uri(self):
1310 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1312 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1313 d.addCallback(_check)
1316 def test_PUT_NEWDIRURL(self):
1317 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1318 d.addCallback(lambda res:
1319 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1320 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1321 d.addCallback(self.failUnlessNodeKeysAre, [])
1324 def test_POST_NEWDIRURL(self):
1325 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1326 d.addCallback(lambda res:
1327 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1328 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1329 d.addCallback(self.failUnlessNodeKeysAre, [])
1332 def test_POST_NEWDIRURL_emptyname(self):
1333 # an empty pathname component (i.e. a double-slash) is disallowed
1334 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1336 "The webapi does not allow empty pathname components, i.e. a double slash",
1337 self.POST, self.public_url + "//?t=mkdir")
1340 def test_POST_NEWDIRURL_initial_children(self):
1341 (newkids, caps) = self._create_initial_children()
1342 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1343 simplejson.dumps(newkids))
1345 n = self.s.create_node_from_uri(uri.strip())
1346 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1347 d2.addCallback(lambda ign:
1348 self.failUnlessROChildURIIs(n, u"child-imm",
1350 d2.addCallback(lambda ign:
1351 self.failUnlessRWChildURIIs(n, u"child-mutable",
1353 d2.addCallback(lambda ign:
1354 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1356 d2.addCallback(lambda ign:
1357 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1358 caps['unknown_rocap']))
1359 d2.addCallback(lambda ign:
1360 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1361 caps['unknown_rwcap']))
1362 d2.addCallback(lambda ign:
1363 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1364 caps['unknown_immcap']))
1365 d2.addCallback(lambda ign:
1366 self.failUnlessRWChildURIIs(n, u"dirchild",
1368 d2.addCallback(lambda ign:
1369 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1371 d2.addCallback(lambda ign:
1372 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1373 caps['emptydircap']))
1375 d.addCallback(_check)
1376 d.addCallback(lambda res:
1377 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1378 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1379 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1380 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1381 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1384 def test_POST_NEWDIRURL_immutable(self):
1385 (newkids, caps) = self._create_immutable_children()
1386 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1387 simplejson.dumps(newkids))
1389 n = self.s.create_node_from_uri(uri.strip())
1390 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1391 d2.addCallback(lambda ign:
1392 self.failUnlessROChildURIIs(n, u"child-imm",
1394 d2.addCallback(lambda ign:
1395 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1396 caps['unknown_immcap']))
1397 d2.addCallback(lambda ign:
1398 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1400 d2.addCallback(lambda ign:
1401 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1403 d2.addCallback(lambda ign:
1404 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1405 caps['emptydircap']))
1407 d.addCallback(_check)
1408 d.addCallback(lambda res:
1409 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1410 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1411 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1412 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1413 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1414 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1415 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1416 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1417 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1418 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1419 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1420 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1421 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1422 d.addErrback(self.explain_web_error)
1425 def test_POST_NEWDIRURL_immutable_bad(self):
1426 (newkids, caps) = self._create_initial_children()
1427 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1429 "needed to be immutable but was not",
1431 self.public_url + "/foo/newdir?t=mkdir-immutable",
1432 simplejson.dumps(newkids))
1435 def test_PUT_NEWDIRURL_exists(self):
1436 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1437 d.addCallback(lambda res:
1438 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1439 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1440 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1443 def test_PUT_NEWDIRURL_blocked(self):
1444 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1445 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1447 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1448 d.addCallback(lambda res:
1449 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1450 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1451 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1454 def test_PUT_NEWDIRURL_mkdir_p(self):
1455 d = defer.succeed(None)
1456 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1457 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1458 d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1459 def mkdir_p(mkpnode):
1460 url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1462 def made_subsub(ssuri):
1463 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1464 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1466 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1468 d.addCallback(made_subsub)
1470 d.addCallback(mkdir_p)
1473 def test_PUT_NEWDIRURL_mkdirs(self):
1474 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1475 d.addCallback(lambda res:
1476 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1477 d.addCallback(lambda res:
1478 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1479 d.addCallback(lambda res:
1480 self._foo_node.get_child_at_path(u"subdir/newdir"))
1481 d.addCallback(self.failUnlessNodeKeysAre, [])
1484 def test_DELETE_DIRURL(self):
1485 d = self.DELETE(self.public_url + "/foo")
1486 d.addCallback(lambda res:
1487 self.failIfNodeHasChild(self.public_root, u"foo"))
1490 def test_DELETE_DIRURL_missing(self):
1491 d = self.DELETE(self.public_url + "/foo/missing")
1492 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1493 d.addCallback(lambda res:
1494 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1497 def test_DELETE_DIRURL_missing2(self):
1498 d = self.DELETE(self.public_url + "/missing")
1499 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1502 def dump_root(self):
1504 w = webish.DirnodeWalkerMixin()
1505 def visitor(childpath, childnode, metadata):
1507 d = w.walk(self.public_root, visitor)
1510 def failUnlessNodeKeysAre(self, node, expected_keys):
1511 for k in expected_keys:
1512 assert isinstance(k, unicode)
1514 def _check(children):
1515 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1516 d.addCallback(_check)
1518 def failUnlessNodeHasChild(self, node, name):
1519 assert isinstance(name, unicode)
1521 def _check(children):
1522 self.failUnless(name in children)
1523 d.addCallback(_check)
1525 def failIfNodeHasChild(self, node, name):
1526 assert isinstance(name, unicode)
1528 def _check(children):
1529 self.failIf(name in children)
1530 d.addCallback(_check)
1533 def failUnlessChildContentsAre(self, node, name, expected_contents):
1534 assert isinstance(name, unicode)
1535 d = node.get_child_at_path(name)
1536 d.addCallback(lambda node: download_to_data(node))
1537 def _check(contents):
1538 self.failUnlessReallyEqual(contents, expected_contents)
1539 d.addCallback(_check)
1542 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1543 assert isinstance(name, unicode)
1544 d = node.get_child_at_path(name)
1545 d.addCallback(lambda node: node.download_best_version())
1546 def _check(contents):
1547 self.failUnlessReallyEqual(contents, expected_contents)
1548 d.addCallback(_check)
1551 def failUnlessRWChildURIIs(self, node, name, expected_uri):
1552 assert isinstance(name, unicode)
1553 d = node.get_child_at_path(name)
1555 self.failUnless(child.is_unknown() or not child.is_readonly())
1556 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1557 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1558 expected_ro_uri = self._make_readonly(expected_uri)
1560 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1561 d.addCallback(_check)
1564 def failUnlessROChildURIIs(self, node, name, expected_uri):
1565 assert isinstance(name, unicode)
1566 d = node.get_child_at_path(name)
1568 self.failUnless(child.is_unknown() or child.is_readonly())
1569 self.failUnlessReallyEqual(child.get_write_uri(), None)
1570 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1571 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1572 d.addCallback(_check)
1575 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1576 assert isinstance(name, unicode)
1577 d = node.get_child_at_path(name)
1579 self.failUnless(child.is_unknown() or not child.is_readonly())
1580 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1581 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1582 expected_ro_uri = self._make_readonly(got_uri)
1584 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1585 d.addCallback(_check)
1588 def failUnlessURIMatchesROChild(self, got_uri, node, name):
1589 assert isinstance(name, unicode)
1590 d = node.get_child_at_path(name)
1592 self.failUnless(child.is_unknown() or child.is_readonly())
1593 self.failUnlessReallyEqual(child.get_write_uri(), None)
1594 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1595 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1596 d.addCallback(_check)
1599 def failUnlessCHKURIHasContents(self, got_uri, contents):
1600 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1602 def test_POST_upload(self):
1603 d = self.POST(self.public_url + "/foo", t="upload",
1604 file=("new.txt", self.NEWFILE_CONTENTS))
1606 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1607 d.addCallback(lambda res:
1608 self.failUnlessChildContentsAre(fn, u"new.txt",
1609 self.NEWFILE_CONTENTS))
1612 def test_POST_upload_unicode(self):
1613 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1614 d = self.POST(self.public_url + "/foo", t="upload",
1615 file=(filename, self.NEWFILE_CONTENTS))
1617 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1618 d.addCallback(lambda res:
1619 self.failUnlessChildContentsAre(fn, filename,
1620 self.NEWFILE_CONTENTS))
1621 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1622 d.addCallback(lambda res: self.GET(target_url))
1623 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1624 self.NEWFILE_CONTENTS,
1628 def test_POST_upload_unicode_named(self):
1629 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1630 d = self.POST(self.public_url + "/foo", t="upload",
1632 file=("overridden", self.NEWFILE_CONTENTS))
1634 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1635 d.addCallback(lambda res:
1636 self.failUnlessChildContentsAre(fn, filename,
1637 self.NEWFILE_CONTENTS))
1638 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1639 d.addCallback(lambda res: self.GET(target_url))
1640 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1641 self.NEWFILE_CONTENTS,
1645 def test_POST_upload_no_link(self):
1646 d = self.POST("/uri", t="upload",
1647 file=("new.txt", self.NEWFILE_CONTENTS))
1648 def _check_upload_results(page):
1649 # this should be a page which describes the results of the upload
1650 # that just finished.
1651 self.failUnless("Upload Results:" in page)
1652 self.failUnless("URI:" in page)
1653 uri_re = re.compile("URI: <tt><span>(.*)</span>")
1654 mo = uri_re.search(page)
1655 self.failUnless(mo, page)
1656 new_uri = mo.group(1)
1658 d.addCallback(_check_upload_results)
1659 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1662 def test_POST_upload_no_link_whendone(self):
1663 d = self.POST("/uri", t="upload", when_done="/",
1664 file=("new.txt", self.NEWFILE_CONTENTS))
1665 d.addBoth(self.shouldRedirect, "/")
1668 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1669 d = defer.maybeDeferred(callable, *args, **kwargs)
1671 if isinstance(res, failure.Failure):
1672 res.trap(error.PageRedirect)
1673 statuscode = res.value.status
1674 target = res.value.location
1675 return checker(statuscode, target)
1676 self.fail("%s: callable was supposed to redirect, not return '%s'"
1681 def test_POST_upload_no_link_whendone_results(self):
1682 def check(statuscode, target):
1683 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1684 self.failUnless(target.startswith(self.webish_url), target)
1685 return client.getPage(target, method="GET")
1686 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1688 self.POST, "/uri", t="upload",
1689 when_done="/uri/%(uri)s",
1690 file=("new.txt", self.NEWFILE_CONTENTS))
1691 d.addCallback(lambda res:
1692 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
1695 def test_POST_upload_no_link_mutable(self):
1696 d = self.POST("/uri", t="upload", mutable="true",
1697 file=("new.txt", self.NEWFILE_CONTENTS))
1698 def _check(filecap):
1699 filecap = filecap.strip()
1700 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1701 self.filecap = filecap
1702 u = uri.WriteableSSKFileURI.init_from_string(filecap)
1703 self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1704 n = self.s.create_node_from_uri(filecap)
1705 return n.download_best_version()
1706 d.addCallback(_check)
1708 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1709 return self.GET("/uri/%s" % urllib.quote(self.filecap))
1710 d.addCallback(_check2)
1712 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1713 return self.GET("/file/%s" % urllib.quote(self.filecap))
1714 d.addCallback(_check3)
1716 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1717 d.addCallback(_check4)
1720 def test_POST_upload_no_link_mutable_toobig(self):
1721 d = self.shouldFail2(error.Error,
1722 "test_POST_upload_no_link_mutable_toobig",
1723 "413 Request Entity Too Large",
1724 "SDMF is limited to one segment, and 10001 > 10000",
1726 "/uri", t="upload", mutable="true",
1728 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1731 def test_POST_upload_mutable(self):
1732 # this creates a mutable file
1733 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1734 file=("new.txt", self.NEWFILE_CONTENTS))
1736 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1737 d.addCallback(lambda res:
1738 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1739 self.NEWFILE_CONTENTS))
1740 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1742 self.failUnless(IMutableFileNode.providedBy(newnode))
1743 self.failUnless(newnode.is_mutable())
1744 self.failIf(newnode.is_readonly())
1745 self._mutable_node = newnode
1746 self._mutable_uri = newnode.get_uri()
1749 # now upload it again and make sure that the URI doesn't change
1750 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1751 d.addCallback(lambda res:
1752 self.POST(self.public_url + "/foo", t="upload",
1754 file=("new.txt", NEWER_CONTENTS)))
1755 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1756 d.addCallback(lambda res:
1757 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1759 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1761 self.failUnless(IMutableFileNode.providedBy(newnode))
1762 self.failUnless(newnode.is_mutable())
1763 self.failIf(newnode.is_readonly())
1764 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1765 d.addCallback(_got2)
1767 # upload a second time, using PUT instead of POST
1768 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1769 d.addCallback(lambda res:
1770 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1771 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1772 d.addCallback(lambda res:
1773 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1776 # finally list the directory, since mutable files are displayed
1777 # slightly differently
1779 d.addCallback(lambda res:
1780 self.GET(self.public_url + "/foo/",
1781 followRedirect=True))
1782 def _check_page(res):
1783 # TODO: assert more about the contents
1784 self.failUnless("SSK" in res)
1786 d.addCallback(_check_page)
1788 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1790 self.failUnless(IMutableFileNode.providedBy(newnode))
1791 self.failUnless(newnode.is_mutable())
1792 self.failIf(newnode.is_readonly())
1793 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1794 d.addCallback(_got3)
1796 # look at the JSON form of the enclosing directory
1797 d.addCallback(lambda res:
1798 self.GET(self.public_url + "/foo/?t=json",
1799 followRedirect=True))
1800 def _check_page_json(res):
1801 parsed = simplejson.loads(res)
1802 self.failUnlessEqual(parsed[0], "dirnode")
1803 children = dict( [(unicode(name),value)
1805 in parsed[1]["children"].iteritems()] )
1806 self.failUnless(u"new.txt" in children)
1807 new_json = children[u"new.txt"]
1808 self.failUnlessEqual(new_json[0], "filenode")
1809 self.failUnless(new_json[1]["mutable"])
1810 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
1811 ro_uri = self._mutable_node.get_readonly().to_string()
1812 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
1813 d.addCallback(_check_page_json)
1815 # and the JSON form of the file
1816 d.addCallback(lambda res:
1817 self.GET(self.public_url + "/foo/new.txt?t=json"))
1818 def _check_file_json(res):
1819 parsed = simplejson.loads(res)
1820 self.failUnlessEqual(parsed[0], "filenode")
1821 self.failUnless(parsed[1]["mutable"])
1822 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
1823 ro_uri = self._mutable_node.get_readonly().to_string()
1824 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
1825 d.addCallback(_check_file_json)
1827 # and look at t=uri and t=readonly-uri
1828 d.addCallback(lambda res:
1829 self.GET(self.public_url + "/foo/new.txt?t=uri"))
1830 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
1831 d.addCallback(lambda res:
1832 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1833 def _check_ro_uri(res):
1834 ro_uri = self._mutable_node.get_readonly().to_string()
1835 self.failUnlessReallyEqual(res, ro_uri)
1836 d.addCallback(_check_ro_uri)
1838 # make sure we can get to it from /uri/URI
1839 d.addCallback(lambda res:
1840 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1841 d.addCallback(lambda res:
1842 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
1844 # and that HEAD computes the size correctly
1845 d.addCallback(lambda res:
1846 self.HEAD(self.public_url + "/foo/new.txt",
1847 return_response=True))
1848 def _got_headers((res, status, headers)):
1849 self.failUnlessReallyEqual(res, "")
1850 self.failUnlessReallyEqual(headers["content-length"][0],
1851 str(len(NEW2_CONTENTS)))
1852 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
1853 d.addCallback(_got_headers)
1855 # make sure that size errors are displayed correctly for overwrite
1856 d.addCallback(lambda res:
1857 self.shouldFail2(error.Error,
1858 "test_POST_upload_mutable-toobig",
1859 "413 Request Entity Too Large",
1860 "SDMF is limited to one segment, and 10001 > 10000",
1862 self.public_url + "/foo", t="upload",
1865 "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1868 d.addErrback(self.dump_error)
1871 def test_POST_upload_mutable_toobig(self):
1872 d = self.shouldFail2(error.Error,
1873 "test_POST_upload_mutable_toobig",
1874 "413 Request Entity Too Large",
1875 "SDMF is limited to one segment, and 10001 > 10000",
1877 self.public_url + "/foo",
1878 t="upload", mutable="true",
1880 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1883 def dump_error(self, f):
1884 # if the web server returns an error code (like 400 Bad Request),
1885 # web.client.getPage puts the HTTP response body into the .response
1886 # attribute of the exception object that it gives back. It does not
1887 # appear in the Failure's repr(), so the ERROR that trial displays
1888 # will be rather terse and unhelpful. addErrback this method to the
1889 # end of your chain to get more information out of these errors.
1890 if f.check(error.Error):
1891 print "web.error.Error:"
1893 print f.value.response
1896 def test_POST_upload_replace(self):
1897 d = self.POST(self.public_url + "/foo", t="upload",
1898 file=("bar.txt", self.NEWFILE_CONTENTS))
1900 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1901 d.addCallback(lambda res:
1902 self.failUnlessChildContentsAre(fn, u"bar.txt",
1903 self.NEWFILE_CONTENTS))
1906 def test_POST_upload_no_replace_ok(self):
1907 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1908 file=("new.txt", self.NEWFILE_CONTENTS))
1909 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1910 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
1911 self.NEWFILE_CONTENTS))
1914 def test_POST_upload_no_replace_queryarg(self):
1915 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1916 file=("bar.txt", self.NEWFILE_CONTENTS))
1917 d.addBoth(self.shouldFail, error.Error,
1918 "POST_upload_no_replace_queryarg",
1920 "There was already a child by that name, and you asked me "
1921 "to not replace it")
1922 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1923 d.addCallback(self.failUnlessIsBarDotTxt)
1926 def test_POST_upload_no_replace_field(self):
1927 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1928 file=("bar.txt", self.NEWFILE_CONTENTS))
1929 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1931 "There was already a child by that name, and you asked me "
1932 "to not replace it")
1933 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1934 d.addCallback(self.failUnlessIsBarDotTxt)
1937 def test_POST_upload_whendone(self):
1938 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1939 file=("new.txt", self.NEWFILE_CONTENTS))
1940 d.addBoth(self.shouldRedirect, "/THERE")
1942 d.addCallback(lambda res:
1943 self.failUnlessChildContentsAre(fn, u"new.txt",
1944 self.NEWFILE_CONTENTS))
1947 def test_POST_upload_named(self):
1949 d = self.POST(self.public_url + "/foo", t="upload",
1950 name="new.txt", file=self.NEWFILE_CONTENTS)
1951 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1952 d.addCallback(lambda res:
1953 self.failUnlessChildContentsAre(fn, u"new.txt",
1954 self.NEWFILE_CONTENTS))
1957 def test_POST_upload_named_badfilename(self):
1958 d = self.POST(self.public_url + "/foo", t="upload",
1959 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1960 d.addBoth(self.shouldFail, error.Error,
1961 "test_POST_upload_named_badfilename",
1963 "name= may not contain a slash",
1965 # make sure that nothing was added
1966 d.addCallback(lambda res:
1967 self.failUnlessNodeKeysAre(self._foo_node,
1968 [u"bar.txt", u"blockingfile",
1969 u"empty", u"n\u00fc.txt",
1973 def test_POST_FILEURL_check(self):
1974 bar_url = self.public_url + "/foo/bar.txt"
1975 d = self.POST(bar_url, t="check")
1977 self.failUnless("Healthy :" in res)
1978 d.addCallback(_check)
1979 redir_url = "http://allmydata.org/TARGET"
1980 def _check2(statuscode, target):
1981 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1982 self.failUnlessReallyEqual(target, redir_url)
1983 d.addCallback(lambda res:
1984 self.shouldRedirect2("test_POST_FILEURL_check",
1988 when_done=redir_url))
1989 d.addCallback(lambda res:
1990 self.POST(bar_url, t="check", return_to=redir_url))
1992 self.failUnless("Healthy :" in res)
1993 self.failUnless("Return to file" in res)
1994 self.failUnless(redir_url in res)
1995 d.addCallback(_check3)
1997 d.addCallback(lambda res:
1998 self.POST(bar_url, t="check", output="JSON"))
1999 def _check_json(res):
2000 data = simplejson.loads(res)
2001 self.failUnless("storage-index" in data)
2002 self.failUnless(data["results"]["healthy"])
2003 d.addCallback(_check_json)
2007 def test_POST_FILEURL_check_and_repair(self):
2008 bar_url = self.public_url + "/foo/bar.txt"
2009 d = self.POST(bar_url, t="check", repair="true")
2011 self.failUnless("Healthy :" in res)
2012 d.addCallback(_check)
2013 redir_url = "http://allmydata.org/TARGET"
2014 def _check2(statuscode, target):
2015 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2016 self.failUnlessReallyEqual(target, redir_url)
2017 d.addCallback(lambda res:
2018 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2021 t="check", repair="true",
2022 when_done=redir_url))
2023 d.addCallback(lambda res:
2024 self.POST(bar_url, t="check", return_to=redir_url))
2026 self.failUnless("Healthy :" in res)
2027 self.failUnless("Return to file" in res)
2028 self.failUnless(redir_url in res)
2029 d.addCallback(_check3)
2032 def test_POST_DIRURL_check(self):
2033 foo_url = self.public_url + "/foo/"
2034 d = self.POST(foo_url, t="check")
2036 self.failUnless("Healthy :" in res, res)
2037 d.addCallback(_check)
2038 redir_url = "http://allmydata.org/TARGET"
2039 def _check2(statuscode, target):
2040 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2041 self.failUnlessReallyEqual(target, redir_url)
2042 d.addCallback(lambda res:
2043 self.shouldRedirect2("test_POST_DIRURL_check",
2047 when_done=redir_url))
2048 d.addCallback(lambda res:
2049 self.POST(foo_url, t="check", return_to=redir_url))
2051 self.failUnless("Healthy :" in res, res)
2052 self.failUnless("Return to file/directory" in res)
2053 self.failUnless(redir_url in res)
2054 d.addCallback(_check3)
2056 d.addCallback(lambda res:
2057 self.POST(foo_url, t="check", output="JSON"))
2058 def _check_json(res):
2059 data = simplejson.loads(res)
2060 self.failUnless("storage-index" in data)
2061 self.failUnless(data["results"]["healthy"])
2062 d.addCallback(_check_json)
2066 def test_POST_DIRURL_check_and_repair(self):
2067 foo_url = self.public_url + "/foo/"
2068 d = self.POST(foo_url, t="check", repair="true")
2070 self.failUnless("Healthy :" in res, res)
2071 d.addCallback(_check)
2072 redir_url = "http://allmydata.org/TARGET"
2073 def _check2(statuscode, target):
2074 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2075 self.failUnlessReallyEqual(target, redir_url)
2076 d.addCallback(lambda res:
2077 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2080 t="check", repair="true",
2081 when_done=redir_url))
2082 d.addCallback(lambda res:
2083 self.POST(foo_url, t="check", return_to=redir_url))
2085 self.failUnless("Healthy :" in res)
2086 self.failUnless("Return to file/directory" in res)
2087 self.failUnless(redir_url in res)
2088 d.addCallback(_check3)
2091 def wait_for_operation(self, ignored, ophandle):
2092 url = "/operations/" + ophandle
2093 url += "?t=status&output=JSON"
2096 data = simplejson.loads(res)
2097 if not data["finished"]:
2098 d = self.stall(delay=1.0)
2099 d.addCallback(self.wait_for_operation, ophandle)
2105 def get_operation_results(self, ignored, ophandle, output=None):
2106 url = "/operations/" + ophandle
2109 url += "&output=" + output
2112 if output and output.lower() == "json":
2113 return simplejson.loads(res)
2118 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2119 d = self.shouldFail2(error.Error,
2120 "test_POST_DIRURL_deepcheck_no_ophandle",
2122 "slow operation requires ophandle=",
2123 self.POST, self.public_url, t="start-deep-check")
2126 def test_POST_DIRURL_deepcheck(self):
2127 def _check_redirect(statuscode, target):
2128 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2129 self.failUnless(target.endswith("/operations/123"))
2130 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2131 self.POST, self.public_url,
2132 t="start-deep-check", ophandle="123")
2133 d.addCallback(self.wait_for_operation, "123")
2134 def _check_json(data):
2135 self.failUnlessReallyEqual(data["finished"], True)
2136 self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2137 self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
2138 d.addCallback(_check_json)
2139 d.addCallback(self.get_operation_results, "123", "html")
2140 def _check_html(res):
2141 self.failUnless("Objects Checked: <span>8</span>" in res)
2142 self.failUnless("Objects Healthy: <span>8</span>" in res)
2143 d.addCallback(_check_html)
2145 d.addCallback(lambda res:
2146 self.GET("/operations/123/"))
2147 d.addCallback(_check_html) # should be the same as without the slash
2149 d.addCallback(lambda res:
2150 self.shouldFail2(error.Error, "one", "404 Not Found",
2151 "No detailed results for SI bogus",
2152 self.GET, "/operations/123/bogus"))
2154 foo_si = self._foo_node.get_storage_index()
2155 foo_si_s = base32.b2a(foo_si)
2156 d.addCallback(lambda res:
2157 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2158 def _check_foo_json(res):
2159 data = simplejson.loads(res)
2160 self.failUnlessEqual(data["storage-index"], foo_si_s)
2161 self.failUnless(data["results"]["healthy"])
2162 d.addCallback(_check_foo_json)
2165 def test_POST_DIRURL_deepcheck_and_repair(self):
2166 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2167 ophandle="124", output="json", followRedirect=True)
2168 d.addCallback(self.wait_for_operation, "124")
2169 def _check_json(data):
2170 self.failUnlessReallyEqual(data["finished"], True)
2171 self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2172 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
2173 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2174 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2175 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2176 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2177 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2178 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
2179 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2180 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2181 d.addCallback(_check_json)
2182 d.addCallback(self.get_operation_results, "124", "html")
2183 def _check_html(res):
2184 self.failUnless("Objects Checked: <span>8</span>" in res)
2186 self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2187 self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2188 self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2190 self.failUnless("Repairs Attempted: <span>0</span>" in res)
2191 self.failUnless("Repairs Successful: <span>0</span>" in res)
2192 self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2194 self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2195 self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2196 self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2197 d.addCallback(_check_html)
2200 def test_POST_FILEURL_bad_t(self):
2201 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2202 "POST to file: bad t=bogus",
2203 self.POST, self.public_url + "/foo/bar.txt",
2207 def test_POST_mkdir(self): # return value?
2208 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2209 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2210 d.addCallback(self.failUnlessNodeKeysAre, [])
2213 def test_POST_mkdir_initial_children(self):
2214 (newkids, caps) = self._create_initial_children()
2215 d = self.POST2(self.public_url +
2216 "/foo?t=mkdir-with-children&name=newdir",
2217 simplejson.dumps(newkids))
2218 d.addCallback(lambda res:
2219 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2220 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2221 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2222 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2223 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2226 def test_POST_mkdir_immutable(self):
2227 (newkids, caps) = self._create_immutable_children()
2228 d = self.POST2(self.public_url +
2229 "/foo?t=mkdir-immutable&name=newdir",
2230 simplejson.dumps(newkids))
2231 d.addCallback(lambda res:
2232 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2233 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2234 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2235 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2236 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2237 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2238 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2239 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2240 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2241 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2242 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2243 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2244 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2247 def test_POST_mkdir_immutable_bad(self):
2248 (newkids, caps) = self._create_initial_children()
2249 d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2251 "needed to be immutable but was not",
2254 "/foo?t=mkdir-immutable&name=newdir",
2255 simplejson.dumps(newkids))
2258 def test_POST_mkdir_2(self):
2259 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2260 d.addCallback(lambda res:
2261 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2262 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2263 d.addCallback(self.failUnlessNodeKeysAre, [])
2266 def test_POST_mkdirs_2(self):
2267 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2268 d.addCallback(lambda res:
2269 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2270 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2271 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2272 d.addCallback(self.failUnlessNodeKeysAre, [])
2275 def test_POST_mkdir_no_parentdir_noredirect(self):
2276 d = self.POST("/uri?t=mkdir")
2277 def _after_mkdir(res):
2278 uri.DirectoryURI.init_from_string(res)
2279 d.addCallback(_after_mkdir)
2282 def test_POST_mkdir_no_parentdir_noredirect2(self):
2283 # make sure form-based arguments (as on the welcome page) still work
2284 d = self.POST("/uri", t="mkdir")
2285 def _after_mkdir(res):
2286 uri.DirectoryURI.init_from_string(res)
2287 d.addCallback(_after_mkdir)
2288 d.addErrback(self.explain_web_error)
2291 def test_POST_mkdir_no_parentdir_redirect(self):
2292 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2293 d.addBoth(self.shouldRedirect, None, statuscode='303')
2294 def _check_target(target):
2295 target = urllib.unquote(target)
2296 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2297 d.addCallback(_check_target)
2300 def test_POST_mkdir_no_parentdir_redirect2(self):
2301 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2302 d.addBoth(self.shouldRedirect, None, statuscode='303')
2303 def _check_target(target):
2304 target = urllib.unquote(target)
2305 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2306 d.addCallback(_check_target)
2307 d.addErrback(self.explain_web_error)
2310 def _make_readonly(self, u):
2311 ro_uri = uri.from_string(u).get_readonly()
2314 return ro_uri.to_string()
2316 def _create_initial_children(self):
2317 contents, n, filecap1 = self.makefile(12)
2318 md1 = {"metakey1": "metavalue1"}
2319 filecap2 = make_mutable_file_uri()
2320 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2321 filecap3 = node3.get_readonly_uri()
2322 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2323 dircap = DirectoryNode(node4, None, None).get_uri()
2324 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2325 emptydircap = "URI:DIR2-LIT:"
2326 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
2327 "ro_uri": self._make_readonly(filecap1),
2328 "metadata": md1, }],
2329 u"child-mutable": ["filenode", {"rw_uri": filecap2,
2330 "ro_uri": self._make_readonly(filecap2)}],
2331 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2332 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
2333 "ro_uri": unknown_rocap}],
2334 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
2335 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2336 u"dirchild": ["dirnode", {"rw_uri": dircap,
2337 "ro_uri": self._make_readonly(dircap)}],
2338 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2339 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2341 return newkids, {'filecap1': filecap1,
2342 'filecap2': filecap2,
2343 'filecap3': filecap3,
2344 'unknown_rwcap': unknown_rwcap,
2345 'unknown_rocap': unknown_rocap,
2346 'unknown_immcap': unknown_immcap,
2348 'litdircap': litdircap,
2349 'emptydircap': emptydircap}
2351 def _create_immutable_children(self):
2352 contents, n, filecap1 = self.makefile(12)
2353 md1 = {"metakey1": "metavalue1"}
2354 tnode = create_chk_filenode("immutable directory contents\n"*10)
2355 dnode = DirectoryNode(tnode, None, None)
2356 assert not dnode.is_mutable()
2357 immdircap = dnode.get_uri()
2358 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2359 emptydircap = "URI:DIR2-LIT:"
2360 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2361 "metadata": md1, }],
2362 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2363 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2364 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2365 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2367 return newkids, {'filecap1': filecap1,
2368 'unknown_immcap': unknown_immcap,
2369 'immdircap': immdircap,
2370 'litdircap': litdircap,
2371 'emptydircap': emptydircap}
2373 def test_POST_mkdir_no_parentdir_initial_children(self):
2374 (newkids, caps) = self._create_initial_children()
2375 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2376 def _after_mkdir(res):
2377 self.failUnless(res.startswith("URI:DIR"), res)
2378 n = self.s.create_node_from_uri(res)
2379 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2380 d2.addCallback(lambda ign:
2381 self.failUnlessROChildURIIs(n, u"child-imm",
2383 d2.addCallback(lambda ign:
2384 self.failUnlessRWChildURIIs(n, u"child-mutable",
2386 d2.addCallback(lambda ign:
2387 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2389 d2.addCallback(lambda ign:
2390 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2391 caps['unknown_rwcap']))
2392 d2.addCallback(lambda ign:
2393 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2394 caps['unknown_rocap']))
2395 d2.addCallback(lambda ign:
2396 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2397 caps['unknown_immcap']))
2398 d2.addCallback(lambda ign:
2399 self.failUnlessRWChildURIIs(n, u"dirchild",
2402 d.addCallback(_after_mkdir)
2405 def test_POST_mkdir_no_parentdir_unexpected_children(self):
2406 # the regular /uri?t=mkdir operation is specified to ignore its body.
2407 # Only t=mkdir-with-children pays attention to it.
2408 (newkids, caps) = self._create_initial_children()
2409 d = self.shouldHTTPError("POST t=mkdir unexpected children",
2411 "t=mkdir does not accept children=, "
2412 "try t=mkdir-with-children instead",
2413 self.POST2, "/uri?t=mkdir", # without children
2414 simplejson.dumps(newkids))
2417 def test_POST_noparent_bad(self):
2418 d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2419 "/uri accepts only PUT, PUT?t=mkdir, "
2420 "POST?t=upload, and POST?t=mkdir",
2421 self.POST, "/uri?t=bogus")
2424 def test_POST_mkdir_no_parentdir_immutable(self):
2425 (newkids, caps) = self._create_immutable_children()
2426 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2427 def _after_mkdir(res):
2428 self.failUnless(res.startswith("URI:DIR"), res)
2429 n = self.s.create_node_from_uri(res)
2430 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2431 d2.addCallback(lambda ign:
2432 self.failUnlessROChildURIIs(n, u"child-imm",
2434 d2.addCallback(lambda ign:
2435 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2436 caps['unknown_immcap']))
2437 d2.addCallback(lambda ign:
2438 self.failUnlessROChildURIIs(n, u"dirchild-imm",
2440 d2.addCallback(lambda ign:
2441 self.failUnlessROChildURIIs(n, u"dirchild-lit",
2443 d2.addCallback(lambda ign:
2444 self.failUnlessROChildURIIs(n, u"dirchild-empty",
2445 caps['emptydircap']))
2447 d.addCallback(_after_mkdir)
2450 def test_POST_mkdir_no_parentdir_immutable_bad(self):
2451 (newkids, caps) = self._create_initial_children()
2452 d = self.shouldFail2(error.Error,
2453 "test_POST_mkdir_no_parentdir_immutable_bad",
2455 "needed to be immutable but was not",
2457 "/uri?t=mkdir-immutable",
2458 simplejson.dumps(newkids))
2461 def test_welcome_page_mkdir_button(self):
2462 # Fetch the welcome page.
2464 def _after_get_welcome_page(res):
2465 MKDIR_BUTTON_RE = re.compile(
2466 '<form action="([^"]*)" method="post".*?'
2467 '<input type="hidden" name="t" value="([^"]*)" />'
2468 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2469 '<input type="submit" value="Create a directory" />',
2471 mo = MKDIR_BUTTON_RE.search(res)
2472 formaction = mo.group(1)
2474 formaname = mo.group(3)
2475 formavalue = mo.group(4)
2476 return (formaction, formt, formaname, formavalue)
2477 d.addCallback(_after_get_welcome_page)
2478 def _after_parse_form(res):
2479 (formaction, formt, formaname, formavalue) = res
2480 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2481 d.addCallback(_after_parse_form)
2482 d.addBoth(self.shouldRedirect, None, statuscode='303')
2485 def test_POST_mkdir_replace(self): # return value?
2486 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2487 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2488 d.addCallback(self.failUnlessNodeKeysAre, [])
2491 def test_POST_mkdir_no_replace_queryarg(self): # return value?
2492 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2493 d.addBoth(self.shouldFail, error.Error,
2494 "POST_mkdir_no_replace_queryarg",
2496 "There was already a child by that name, and you asked me "
2497 "to not replace it")
2498 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2499 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2502 def test_POST_mkdir_no_replace_field(self): # return value?
2503 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2505 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2507 "There was already a child by that name, and you asked me "
2508 "to not replace it")
2509 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2510 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2513 def test_POST_mkdir_whendone_field(self):
2514 d = self.POST(self.public_url + "/foo",
2515 t="mkdir", name="newdir", when_done="/THERE")
2516 d.addBoth(self.shouldRedirect, "/THERE")
2517 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2518 d.addCallback(self.failUnlessNodeKeysAre, [])
2521 def test_POST_mkdir_whendone_queryarg(self):
2522 d = self.POST(self.public_url + "/foo?when_done=/THERE",
2523 t="mkdir", name="newdir")
2524 d.addBoth(self.shouldRedirect, "/THERE")
2525 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2526 d.addCallback(self.failUnlessNodeKeysAre, [])
2529 def test_POST_bad_t(self):
2530 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2531 "POST to a directory with bad t=BOGUS",
2532 self.POST, self.public_url + "/foo", t="BOGUS")
2535 def test_POST_set_children(self, command_name="set_children"):
2536 contents9, n9, newuri9 = self.makefile(9)
2537 contents10, n10, newuri10 = self.makefile(10)
2538 contents11, n11, newuri11 = self.makefile(11)
2541 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2544 "ctime": 1002777696.7564139,
2545 "mtime": 1002777696.7564139
2548 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2551 "ctime": 1002777696.7564139,
2552 "mtime": 1002777696.7564139
2555 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2558 "ctime": 1002777696.7564139,
2559 "mtime": 1002777696.7564139
2562 }""" % (newuri9, newuri10, newuri11)
2564 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2566 d = client.getPage(url, method="POST", postdata=reqbody)
2568 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2569 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2570 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2572 d.addCallback(_then)
2573 d.addErrback(self.dump_error)
2576 def test_POST_set_children_with_hyphen(self):
2577 return self.test_POST_set_children(command_name="set-children")
2579 def test_POST_link_uri(self):
2580 contents, n, newuri = self.makefile(8)
2581 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2582 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2583 d.addCallback(lambda res:
2584 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2588 def test_POST_link_uri_replace(self):
2589 contents, n, newuri = self.makefile(8)
2590 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2591 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2592 d.addCallback(lambda res:
2593 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2597 def test_POST_link_uri_unknown_bad(self):
2598 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2599 d.addBoth(self.shouldFail, error.Error,
2600 "POST_link_uri_unknown_bad",
2602 "unknown cap in a write slot")
2605 def test_POST_link_uri_unknown_ro_good(self):
2606 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2607 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2610 def test_POST_link_uri_unknown_imm_good(self):
2611 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2612 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2615 def test_POST_link_uri_no_replace_queryarg(self):
2616 contents, n, newuri = self.makefile(8)
2617 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2618 name="bar.txt", uri=newuri)
2619 d.addBoth(self.shouldFail, error.Error,
2620 "POST_link_uri_no_replace_queryarg",
2622 "There was already a child by that name, and you asked me "
2623 "to not replace it")
2624 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2625 d.addCallback(self.failUnlessIsBarDotTxt)
2628 def test_POST_link_uri_no_replace_field(self):
2629 contents, n, newuri = self.makefile(8)
2630 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2631 name="bar.txt", uri=newuri)
2632 d.addBoth(self.shouldFail, error.Error,
2633 "POST_link_uri_no_replace_field",
2635 "There was already a child by that name, and you asked me "
2636 "to not replace it")
2637 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2638 d.addCallback(self.failUnlessIsBarDotTxt)
2641 def test_POST_delete(self):
2642 d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2643 d.addCallback(lambda res: self._foo_node.list())
2644 def _check(children):
2645 self.failIf(u"bar.txt" in children)
2646 d.addCallback(_check)
2649 def test_POST_rename_file(self):
2650 d = self.POST(self.public_url + "/foo", t="rename",
2651 from_name="bar.txt", to_name='wibble.txt')
2652 d.addCallback(lambda res:
2653 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2654 d.addCallback(lambda res:
2655 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2656 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2657 d.addCallback(self.failUnlessIsBarDotTxt)
2658 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2659 d.addCallback(self.failUnlessIsBarJSON)
2662 def test_POST_rename_file_redundant(self):
2663 d = self.POST(self.public_url + "/foo", t="rename",
2664 from_name="bar.txt", to_name='bar.txt')
2665 d.addCallback(lambda res:
2666 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2667 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2668 d.addCallback(self.failUnlessIsBarDotTxt)
2669 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2670 d.addCallback(self.failUnlessIsBarJSON)
2673 def test_POST_rename_file_replace(self):
2674 # rename a file and replace a directory with it
2675 d = self.POST(self.public_url + "/foo", t="rename",
2676 from_name="bar.txt", to_name='empty')
2677 d.addCallback(lambda res:
2678 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2679 d.addCallback(lambda res:
2680 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2681 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2682 d.addCallback(self.failUnlessIsBarDotTxt)
2683 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2684 d.addCallback(self.failUnlessIsBarJSON)
2687 def test_POST_rename_file_no_replace_queryarg(self):
2688 # rename a file and replace a directory with it
2689 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2690 from_name="bar.txt", to_name='empty')
2691 d.addBoth(self.shouldFail, error.Error,
2692 "POST_rename_file_no_replace_queryarg",
2694 "There was already a child by that name, and you asked me "
2695 "to not replace it")
2696 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2697 d.addCallback(self.failUnlessIsEmptyJSON)
2700 def test_POST_rename_file_no_replace_field(self):
2701 # rename a file and replace a directory with it
2702 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2703 from_name="bar.txt", to_name='empty')
2704 d.addBoth(self.shouldFail, error.Error,
2705 "POST_rename_file_no_replace_field",
2707 "There was already a child by that name, and you asked me "
2708 "to not replace it")
2709 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2710 d.addCallback(self.failUnlessIsEmptyJSON)
2713 def failUnlessIsEmptyJSON(self, res):
2714 data = simplejson.loads(res)
2715 self.failUnlessEqual(data[0], "dirnode", data)
2716 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
2718 def test_POST_rename_file_slash_fail(self):
2719 d = self.POST(self.public_url + "/foo", t="rename",
2720 from_name="bar.txt", to_name='kirk/spock.txt')
2721 d.addBoth(self.shouldFail, error.Error,
2722 "test_POST_rename_file_slash_fail",
2724 "to_name= may not contain a slash",
2726 d.addCallback(lambda res:
2727 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2730 def test_POST_rename_dir(self):
2731 d = self.POST(self.public_url, t="rename",
2732 from_name="foo", to_name='plunk')
2733 d.addCallback(lambda res:
2734 self.failIfNodeHasChild(self.public_root, u"foo"))
2735 d.addCallback(lambda res:
2736 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2737 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2738 d.addCallback(self.failUnlessIsFooJSON)
2741 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2742 """ If target is not None then the redirection has to go to target. If
2743 statuscode is not None then the redirection has to be accomplished with
2744 that HTTP status code."""
2745 if not isinstance(res, failure.Failure):
2746 to_where = (target is None) and "somewhere" or ("to " + target)
2747 self.fail("%s: we were expecting to get redirected %s, not get an"
2748 " actual page: %s" % (which, to_where, res))
2749 res.trap(error.PageRedirect)
2750 if statuscode is not None:
2751 self.failUnlessReallyEqual(res.value.status, statuscode,
2752 "%s: not a redirect" % which)
2753 if target is not None:
2754 # the PageRedirect does not seem to capture the uri= query arg
2755 # properly, so we can't check for it.
2756 realtarget = self.webish_url + target
2757 self.failUnlessReallyEqual(res.value.location, realtarget,
2758 "%s: wrong target" % which)
2759 return res.value.location
2761 def test_GET_URI_form(self):
2762 base = "/uri?uri=%s" % self._bar_txt_uri
2763 # this is supposed to give us a redirect to /uri/$URI, plus arguments
2764 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2766 d.addBoth(self.shouldRedirect, targetbase)
2767 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2768 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2769 d.addCallback(lambda res: self.GET(base+"&t=json"))
2770 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2771 d.addCallback(self.log, "about to get file by uri")
2772 d.addCallback(lambda res: self.GET(base, followRedirect=True))
2773 d.addCallback(self.failUnlessIsBarDotTxt)
2774 d.addCallback(self.log, "got file by uri, about to get dir by uri")
2775 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2776 followRedirect=True))
2777 d.addCallback(self.failUnlessIsFooJSON)
2778 d.addCallback(self.log, "got dir by uri")
2782 def test_GET_URI_form_bad(self):
2783 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2784 "400 Bad Request", "GET /uri requires uri=",
2788 def test_GET_rename_form(self):
2789 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2790 followRedirect=True)
2792 self.failUnless('name="when_done" value="."' in res, res)
2793 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2794 d.addCallback(_check)
2797 def log(self, res, msg):
2798 #print "MSG: %s RES: %s" % (msg, res)
2802 def test_GET_URI_URL(self):
2803 base = "/uri/%s" % self._bar_txt_uri
2805 d.addCallback(self.failUnlessIsBarDotTxt)
2806 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2807 d.addCallback(self.failUnlessIsBarDotTxt)
2808 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2809 d.addCallback(self.failUnlessIsBarDotTxt)
2812 def test_GET_URI_URL_dir(self):
2813 base = "/uri/%s?t=json" % self._foo_uri
2815 d.addCallback(self.failUnlessIsFooJSON)
2818 def test_GET_URI_URL_missing(self):
2819 base = "/uri/%s" % self._bad_file_uri
2820 d = self.shouldHTTPError("test_GET_URI_URL_missing",
2821 http.GONE, None, "NotEnoughSharesError",
2823 # TODO: how can we exercise both sides of WebDownloadTarget.fail
2824 # here? we must arrange for a download to fail after target.open()
2825 # has been called, and then inspect the response to see that it is
2826 # shorter than we expected.
2829 def test_PUT_DIRURL_uri(self):
2830 d = self.s.create_dirnode()
2832 new_uri = dn.get_uri()
2833 # replace /foo with a new (empty) directory
2834 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2835 d.addCallback(lambda res:
2836 self.failUnlessReallyEqual(res.strip(), new_uri))
2837 d.addCallback(lambda res:
2838 self.failUnlessRWChildURIIs(self.public_root,
2842 d.addCallback(_made_dir)
2845 def test_PUT_DIRURL_uri_noreplace(self):
2846 d = self.s.create_dirnode()
2848 new_uri = dn.get_uri()
2849 # replace /foo with a new (empty) directory, but ask that
2850 # replace=false, so it should fail
2851 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2852 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2854 self.public_url + "/foo?t=uri&replace=false",
2856 d.addCallback(lambda res:
2857 self.failUnlessRWChildURIIs(self.public_root,
2861 d.addCallback(_made_dir)
2864 def test_PUT_DIRURL_bad_t(self):
2865 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2866 "400 Bad Request", "PUT to a directory",
2867 self.PUT, self.public_url + "/foo?t=BOGUS", "")
2868 d.addCallback(lambda res:
2869 self.failUnlessRWChildURIIs(self.public_root,
2874 def test_PUT_NEWFILEURL_uri(self):
2875 contents, n, new_uri = self.makefile(8)
2876 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2877 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2878 d.addCallback(lambda res:
2879 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2883 def test_PUT_NEWFILEURL_uri_replace(self):
2884 contents, n, new_uri = self.makefile(8)
2885 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2886 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2887 d.addCallback(lambda res:
2888 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2892 def test_PUT_NEWFILEURL_uri_no_replace(self):
2893 contents, n, new_uri = self.makefile(8)
2894 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2895 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2897 "There was already a child by that name, and you asked me "
2898 "to not replace it")
2901 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2902 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2903 d.addBoth(self.shouldFail, error.Error,
2904 "POST_put_uri_unknown_bad",
2906 "unknown cap in a write slot")
2909 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2910 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2911 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2912 u"put-future-ro.txt")
2915 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2916 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2917 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2918 u"put-future-imm.txt")
2921 def test_PUT_NEWFILE_URI(self):
2922 file_contents = "New file contents here\n"
2923 d = self.PUT("/uri", file_contents)
2925 assert isinstance(uri, str), uri
2926 self.failUnless(uri in FakeCHKFileNode.all_contents)
2927 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2929 return self.GET("/uri/%s" % uri)
2930 d.addCallback(_check)
2932 self.failUnlessReallyEqual(res, file_contents)
2933 d.addCallback(_check2)
2936 def test_PUT_NEWFILE_URI_not_mutable(self):
2937 file_contents = "New file contents here\n"
2938 d = self.PUT("/uri?mutable=false", file_contents)
2940 assert isinstance(uri, str), uri
2941 self.failUnless(uri in FakeCHKFileNode.all_contents)
2942 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2944 return self.GET("/uri/%s" % uri)
2945 d.addCallback(_check)
2947 self.failUnlessReallyEqual(res, file_contents)
2948 d.addCallback(_check2)
2951 def test_PUT_NEWFILE_URI_only_PUT(self):
2952 d = self.PUT("/uri?t=bogus", "")
2953 d.addBoth(self.shouldFail, error.Error,
2954 "PUT_NEWFILE_URI_only_PUT",
2956 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2959 def test_PUT_NEWFILE_URI_mutable(self):
2960 file_contents = "New file contents here\n"
2961 d = self.PUT("/uri?mutable=true", file_contents)
2962 def _check1(filecap):
2963 filecap = filecap.strip()
2964 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2965 self.filecap = filecap
2966 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2967 self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2968 n = self.s.create_node_from_uri(filecap)
2969 return n.download_best_version()
2970 d.addCallback(_check1)
2972 self.failUnlessReallyEqual(data, file_contents)
2973 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2974 d.addCallback(_check2)
2976 self.failUnlessReallyEqual(res, file_contents)
2977 d.addCallback(_check3)
2980 def test_PUT_mkdir(self):
2981 d = self.PUT("/uri?t=mkdir", "")
2983 n = self.s.create_node_from_uri(uri.strip())
2984 d2 = self.failUnlessNodeKeysAre(n, [])
2985 d2.addCallback(lambda res:
2986 self.GET("/uri/%s?t=json" % uri))
2988 d.addCallback(_check)
2989 d.addCallback(self.failUnlessIsEmptyJSON)
2992 def test_POST_check(self):
2993 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2995 # this returns a string form of the results, which are probably
2996 # None since we're using fake filenodes.
2997 # TODO: verify that the check actually happened, by changing
2998 # FakeCHKFileNode to count how many times .check() has been
3001 d.addCallback(_done)
3004 def test_bad_method(self):
3005 url = self.webish_url + self.public_url + "/foo/bar.txt"
3006 d = self.shouldHTTPError("test_bad_method",
3007 501, "Not Implemented",
3008 "I don't know how to treat a BOGUS request.",
3009 client.getPage, url, method="BOGUS")
3012 def test_short_url(self):
3013 url = self.webish_url + "/uri"
3014 d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3015 "I don't know how to treat a DELETE request.",
3016 client.getPage, url, method="DELETE")
3019 def test_ophandle_bad(self):
3020 url = self.webish_url + "/operations/bogus?t=status"
3021 d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3022 "unknown/expired handle 'bogus'",
3023 client.getPage, url)
3026 def test_ophandle_cancel(self):
3027 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3028 followRedirect=True)
3029 d.addCallback(lambda ignored:
3030 self.GET("/operations/128?t=status&output=JSON"))
3032 data = simplejson.loads(res)
3033 self.failUnless("finished" in data, res)
3034 monitor = self.ws.root.child_operations.handles["128"][0]
3035 d = self.POST("/operations/128?t=cancel&output=JSON")
3037 data = simplejson.loads(res)
3038 self.failUnless("finished" in data, res)
3039 # t=cancel causes the handle to be forgotten
3040 self.failUnless(monitor.is_cancelled())
3041 d.addCallback(_check2)
3043 d.addCallback(_check1)
3044 d.addCallback(lambda ignored:
3045 self.shouldHTTPError("test_ophandle_cancel",
3046 404, "404 Not Found",
3047 "unknown/expired handle '128'",
3049 "/operations/128?t=status&output=JSON"))
3052 def test_ophandle_retainfor(self):
3053 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3054 followRedirect=True)
3055 d.addCallback(lambda ignored:
3056 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3058 data = simplejson.loads(res)
3059 self.failUnless("finished" in data, res)
3060 d.addCallback(_check1)
3061 # the retain-for=0 will cause the handle to be expired very soon
3062 d.addCallback(lambda ign:
3063 self.clock.advance(2.0))
3064 d.addCallback(lambda ignored:
3065 self.shouldHTTPError("test_ophandle_retainfor",
3066 404, "404 Not Found",
3067 "unknown/expired handle '129'",
3069 "/operations/129?t=status&output=JSON"))
3072 def test_ophandle_release_after_complete(self):
3073 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3074 followRedirect=True)
3075 d.addCallback(self.wait_for_operation, "130")
3076 d.addCallback(lambda ignored:
3077 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3078 # the release-after-complete=true will cause the handle to be expired
3079 d.addCallback(lambda ignored:
3080 self.shouldHTTPError("test_ophandle_release_after_complete",
3081 404, "404 Not Found",
3082 "unknown/expired handle '130'",
3084 "/operations/130?t=status&output=JSON"))
3087 def test_uncollected_ophandle_expiration(self):
3088 # uncollected ophandles should expire after 4 days
3089 def _make_uncollected_ophandle(ophandle):
3090 d = self.POST(self.public_url +
3091 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3092 followRedirect=False)
3093 # When we start the operation, the webapi server will want
3094 # to redirect us to the page for the ophandle, so we get
3095 # confirmation that the operation has started. If the
3096 # manifest operation has finished by the time we get there,
3097 # following that redirect (by setting followRedirect=True
3098 # above) has the side effect of collecting the ophandle that
3099 # we've just created, which means that we can't use the
3100 # ophandle to test the uncollected timeout anymore. So,
3101 # instead, catch the 302 here and don't follow it.
3102 d.addBoth(self.should302, "uncollected_ophandle_creation")
3104 # Create an ophandle, don't collect it, then advance the clock by
3105 # 4 days - 1 second and make sure that the ophandle is still there.
3106 d = _make_uncollected_ophandle(131)
3107 d.addCallback(lambda ign:
3108 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3109 d.addCallback(lambda ign:
3110 self.GET("/operations/131?t=status&output=JSON"))
3112 data = simplejson.loads(res)
3113 self.failUnless("finished" in data, res)
3114 d.addCallback(_check1)
3115 # Create an ophandle, don't collect it, then try to collect it
3116 # after 4 days. It should be gone.
3117 d.addCallback(lambda ign:
3118 _make_uncollected_ophandle(132))
3119 d.addCallback(lambda ign:
3120 self.clock.advance(96*60*60))
3121 d.addCallback(lambda ign:
3122 self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3123 404, "404 Not Found",
3124 "unknown/expired handle '132'",
3126 "/operations/132?t=status&output=JSON"))
3129 def test_collected_ophandle_expiration(self):
3130 # collected ophandles should expire after 1 day
3131 def _make_collected_ophandle(ophandle):
3132 d = self.POST(self.public_url +
3133 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3134 followRedirect=True)
3135 # By following the initial redirect, we collect the ophandle
3136 # we've just created.
3138 # Create a collected ophandle, then collect it after 23 hours
3139 # and 59 seconds to make sure that it is still there.
3140 d = _make_collected_ophandle(133)
3141 d.addCallback(lambda ign:
3142 self.clock.advance((24*60*60) - 1))
3143 d.addCallback(lambda ign:
3144 self.GET("/operations/133?t=status&output=JSON"))
3146 data = simplejson.loads(res)
3147 self.failUnless("finished" in data, res)
3148 d.addCallback(_check1)
3149 # Create another uncollected ophandle, then try to collect it
3150 # after 24 hours to make sure that it is gone.
3151 d.addCallback(lambda ign:
3152 _make_collected_ophandle(134))
3153 d.addCallback(lambda ign:
3154 self.clock.advance(24*60*60))
3155 d.addCallback(lambda ign:
3156 self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3157 404, "404 Not Found",
3158 "unknown/expired handle '134'",
3160 "/operations/134?t=status&output=JSON"))
3163 def test_incident(self):
3164 d = self.POST("/report_incident", details="eek")
3166 self.failUnless("Thank you for your report!" in res, res)
3167 d.addCallback(_done)
3170 def test_static(self):
3171 webdir = os.path.join(self.staticdir, "subdir")
3172 fileutil.make_dirs(webdir)
3173 f = open(os.path.join(webdir, "hello.txt"), "wb")
3177 d = self.GET("/static/subdir/hello.txt")
3179 self.failUnlessReallyEqual(res, "hello")
3180 d.addCallback(_check)
3184 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3185 def test_load_file(self):
3186 # This will raise an exception unless a well-formed XML file is found under that name.
3187 common.getxmlfile('directory.xhtml').load()
3189 def test_parse_replace_arg(self):
3190 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3191 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3192 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3194 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3195 common.parse_replace_arg, "only_fles")
3197 def test_abbreviate_time(self):
3198 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3199 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3200 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3201 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3202 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3203 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3205 def test_compute_rate(self):
3206 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3207 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3208 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3209 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3210 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3211 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3212 self.shouldFail(AssertionError, "test_compute_rate", "",
3213 common.compute_rate, -100, 10)
3214 self.shouldFail(AssertionError, "test_compute_rate", "",
3215 common.compute_rate, 100, -10)
3218 rate = common.compute_rate(10*1000*1000, 1)
3219 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3221 def test_abbreviate_rate(self):
3222 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3223 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3224 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3225 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3227 def test_abbreviate_size(self):
3228 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3229 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3230 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3231 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3232 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3234 def test_plural(self):
3236 return "%d second%s" % (s, status.plural(s))
3237 self.failUnlessReallyEqual(convert(0), "0 seconds")
3238 self.failUnlessReallyEqual(convert(1), "1 second")
3239 self.failUnlessReallyEqual(convert(2), "2 seconds")
3241 return "has share%s: %s" % (status.plural(s), ",".join(s))
3242 self.failUnlessReallyEqual(convert2([]), "has shares: ")
3243 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3244 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3247 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3249 def CHECK(self, ign, which, args, clientnum=0):
3250 fileurl = self.fileurls[which]
3251 url = fileurl + "?" + args
3252 return self.GET(url, method="POST", clientnum=clientnum)
3254 def test_filecheck(self):
3255 self.basedir = "web/Grid/filecheck"
3257 c0 = self.g.clients[0]
3260 d = c0.upload(upload.Data(DATA, convergence=""))
3261 def _stash_uri(ur, which):
3262 self.uris[which] = ur.uri
3263 d.addCallback(_stash_uri, "good")
3264 d.addCallback(lambda ign:
3265 c0.upload(upload.Data(DATA+"1", convergence="")))
3266 d.addCallback(_stash_uri, "sick")
3267 d.addCallback(lambda ign:
3268 c0.upload(upload.Data(DATA+"2", convergence="")))
3269 d.addCallback(_stash_uri, "dead")
3270 def _stash_mutable_uri(n, which):
3271 self.uris[which] = n.get_uri()
3272 assert isinstance(self.uris[which], str)
3273 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3274 d.addCallback(_stash_mutable_uri, "corrupt")
3275 d.addCallback(lambda ign:
3276 c0.upload(upload.Data("literal", convergence="")))
3277 d.addCallback(_stash_uri, "small")
3278 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3279 d.addCallback(_stash_mutable_uri, "smalldir")
3281 def _compute_fileurls(ignored):
3283 for which in self.uris:
3284 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3285 d.addCallback(_compute_fileurls)
3287 def _clobber_shares(ignored):
3288 good_shares = self.find_uri_shares(self.uris["good"])
3289 self.failUnlessReallyEqual(len(good_shares), 10)
3290 sick_shares = self.find_uri_shares(self.uris["sick"])
3291 os.unlink(sick_shares[0][2])
3292 dead_shares = self.find_uri_shares(self.uris["dead"])
3293 for i in range(1, 10):
3294 os.unlink(dead_shares[i][2])
3295 c_shares = self.find_uri_shares(self.uris["corrupt"])
3296 cso = CorruptShareOptions()
3297 cso.stdout = StringIO()
3298 cso.parseOptions([c_shares[0][2]])
3300 d.addCallback(_clobber_shares)
3302 d.addCallback(self.CHECK, "good", "t=check")
3303 def _got_html_good(res):
3304 self.failUnless("Healthy" in res, res)
3305 self.failIf("Not Healthy" in res, res)
3306 d.addCallback(_got_html_good)
3307 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3308 def _got_html_good_return_to(res):
3309 self.failUnless("Healthy" in res, res)
3310 self.failIf("Not Healthy" in res, res)
3311 self.failUnless('<a href="somewhere">Return to file'
3313 d.addCallback(_got_html_good_return_to)
3314 d.addCallback(self.CHECK, "good", "t=check&output=json")
3315 def _got_json_good(res):
3316 r = simplejson.loads(res)
3317 self.failUnlessEqual(r["summary"], "Healthy")
3318 self.failUnless(r["results"]["healthy"])
3319 self.failIf(r["results"]["needs-rebalancing"])
3320 self.failUnless(r["results"]["recoverable"])
3321 d.addCallback(_got_json_good)
3323 d.addCallback(self.CHECK, "small", "t=check")
3324 def _got_html_small(res):
3325 self.failUnless("Literal files are always healthy" in res, res)
3326 self.failIf("Not Healthy" in res, res)
3327 d.addCallback(_got_html_small)
3328 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3329 def _got_html_small_return_to(res):
3330 self.failUnless("Literal files are always healthy" in res, res)
3331 self.failIf("Not Healthy" in res, res)
3332 self.failUnless('<a href="somewhere">Return to file'
3334 d.addCallback(_got_html_small_return_to)
3335 d.addCallback(self.CHECK, "small", "t=check&output=json")
3336 def _got_json_small(res):
3337 r = simplejson.loads(res)
3338 self.failUnlessEqual(r["storage-index"], "")
3339 self.failUnless(r["results"]["healthy"])
3340 d.addCallback(_got_json_small)
3342 d.addCallback(self.CHECK, "smalldir", "t=check")
3343 def _got_html_smalldir(res):
3344 self.failUnless("Literal files are always healthy" in res, res)
3345 self.failIf("Not Healthy" in res, res)
3346 d.addCallback(_got_html_smalldir)
3347 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3348 def _got_json_smalldir(res):
3349 r = simplejson.loads(res)
3350 self.failUnlessEqual(r["storage-index"], "")
3351 self.failUnless(r["results"]["healthy"])
3352 d.addCallback(_got_json_smalldir)
3354 d.addCallback(self.CHECK, "sick", "t=check")
3355 def _got_html_sick(res):
3356 self.failUnless("Not Healthy" in res, res)
3357 d.addCallback(_got_html_sick)
3358 d.addCallback(self.CHECK, "sick", "t=check&output=json")
3359 def _got_json_sick(res):
3360 r = simplejson.loads(res)
3361 self.failUnlessEqual(r["summary"],
3362 "Not Healthy: 9 shares (enc 3-of-10)")
3363 self.failIf(r["results"]["healthy"])
3364 self.failIf(r["results"]["needs-rebalancing"])
3365 self.failUnless(r["results"]["recoverable"])
3366 d.addCallback(_got_json_sick)
3368 d.addCallback(self.CHECK, "dead", "t=check")
3369 def _got_html_dead(res):
3370 self.failUnless("Not Healthy" in res, res)
3371 d.addCallback(_got_html_dead)
3372 d.addCallback(self.CHECK, "dead", "t=check&output=json")
3373 def _got_json_dead(res):
3374 r = simplejson.loads(res)
3375 self.failUnlessEqual(r["summary"],
3376 "Not Healthy: 1 shares (enc 3-of-10)")
3377 self.failIf(r["results"]["healthy"])
3378 self.failIf(r["results"]["needs-rebalancing"])
3379 self.failIf(r["results"]["recoverable"])
3380 d.addCallback(_got_json_dead)
3382 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3383 def _got_html_corrupt(res):
3384 self.failUnless("Not Healthy! : Unhealthy" in res, res)
3385 d.addCallback(_got_html_corrupt)
3386 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3387 def _got_json_corrupt(res):
3388 r = simplejson.loads(res)
3389 self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3391 self.failIf(r["results"]["healthy"])
3392 self.failUnless(r["results"]["recoverable"])
3393 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
3394 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
3395 d.addCallback(_got_json_corrupt)
3397 d.addErrback(self.explain_web_error)
3400 def test_repair_html(self):
3401 self.basedir = "web/Grid/repair_html"
3403 c0 = self.g.clients[0]
3406 d = c0.upload(upload.Data(DATA, convergence=""))
3407 def _stash_uri(ur, which):
3408 self.uris[which] = ur.uri
3409 d.addCallback(_stash_uri, "good")
3410 d.addCallback(lambda ign:
3411 c0.upload(upload.Data(DATA+"1", convergence="")))
3412 d.addCallback(_stash_uri, "sick")
3413 d.addCallback(lambda ign:
3414 c0.upload(upload.Data(DATA+"2", convergence="")))
3415 d.addCallback(_stash_uri, "dead")
3416 def _stash_mutable_uri(n, which):
3417 self.uris[which] = n.get_uri()
3418 assert isinstance(self.uris[which], str)
3419 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3420 d.addCallback(_stash_mutable_uri, "corrupt")
3422 def _compute_fileurls(ignored):
3424 for which in self.uris:
3425 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3426 d.addCallback(_compute_fileurls)
3428 def _clobber_shares(ignored):
3429 good_shares = self.find_uri_shares(self.uris["good"])
3430 self.failUnlessReallyEqual(len(good_shares), 10)
3431 sick_shares = self.find_uri_shares(self.uris["sick"])
3432 os.unlink(sick_shares[0][2])
3433 dead_shares = self.find_uri_shares(self.uris["dead"])
3434 for i in range(1, 10):
3435 os.unlink(dead_shares[i][2])
3436 c_shares = self.find_uri_shares(self.uris["corrupt"])
3437 cso = CorruptShareOptions()
3438 cso.stdout = StringIO()
3439 cso.parseOptions([c_shares[0][2]])
3441 d.addCallback(_clobber_shares)
3443 d.addCallback(self.CHECK, "good", "t=check&repair=true")
3444 def _got_html_good(res):
3445 self.failUnless("Healthy" in res, res)
3446 self.failIf("Not Healthy" in res, res)
3447 self.failUnless("No repair necessary" in res, res)
3448 d.addCallback(_got_html_good)
3450 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3451 def _got_html_sick(res):
3452 self.failUnless("Healthy : healthy" in res, res)
3453 self.failIf("Not Healthy" in res, res)
3454 self.failUnless("Repair successful" in res, res)
3455 d.addCallback(_got_html_sick)
3457 # repair of a dead file will fail, of course, but it isn't yet
3458 # clear how this should be reported. Right now it shows up as
3461 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3462 #def _got_html_dead(res):
3464 # self.failUnless("Healthy : healthy" in res, res)
3465 # self.failIf("Not Healthy" in res, res)
3466 # self.failUnless("No repair necessary" in res, res)
3467 #d.addCallback(_got_html_dead)
3469 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3470 def _got_html_corrupt(res):
3471 self.failUnless("Healthy : Healthy" in res, res)
3472 self.failIf("Not Healthy" in res, res)
3473 self.failUnless("Repair successful" in res, res)
3474 d.addCallback(_got_html_corrupt)
3476 d.addErrback(self.explain_web_error)
3479 def test_repair_json(self):
3480 self.basedir = "web/Grid/repair_json"
3482 c0 = self.g.clients[0]
3485 d = c0.upload(upload.Data(DATA+"1", convergence=""))
3486 def _stash_uri(ur, which):
3487 self.uris[which] = ur.uri
3488 d.addCallback(_stash_uri, "sick")
3490 def _compute_fileurls(ignored):
3492 for which in self.uris:
3493 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3494 d.addCallback(_compute_fileurls)
3496 def _clobber_shares(ignored):
3497 sick_shares = self.find_uri_shares(self.uris["sick"])
3498 os.unlink(sick_shares[0][2])
3499 d.addCallback(_clobber_shares)
3501 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3502 def _got_json_sick(res):
3503 r = simplejson.loads(res)
3504 self.failUnlessReallyEqual(r["repair-attempted"], True)
3505 self.failUnlessReallyEqual(r["repair-successful"], True)
3506 self.failUnlessEqual(r["pre-repair-results"]["summary"],
3507 "Not Healthy: 9 shares (enc 3-of-10)")
3508 self.failIf(r["pre-repair-results"]["results"]["healthy"])
3509 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3510 self.failUnless(r["post-repair-results"]["results"]["healthy"])
3511 d.addCallback(_got_json_sick)
3513 d.addErrback(self.explain_web_error)
3516 def test_unknown(self, immutable=False):
3517 self.basedir = "web/Grid/unknown"
3519 self.basedir = "web/Grid/unknown-immutable"
3522 c0 = self.g.clients[0]
3526 # the future cap format may contain slashes, which must be tolerated
3527 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3531 name = u"future-imm"
3532 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3533 d = c0.create_immutable_dirnode({name: (future_node, {})})
3536 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3537 d = c0.create_dirnode()
3539 def _stash_root_and_create_file(n):
3541 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3542 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3544 return self.rootnode.set_node(name, future_node)
3545 d.addCallback(_stash_root_and_create_file)
3547 # make sure directory listing tolerates unknown nodes
3548 d.addCallback(lambda ign: self.GET(self.rooturl))
3549 def _check_directory_html(res, expected_type_suffix):
3550 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3551 '<td>%s</td>' % (expected_type_suffix, str(name)),
3553 self.failUnless(re.search(pattern, res), res)
3554 # find the More Info link for name, should be relative
3555 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3556 info_url = mo.group(1)
3557 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
3559 d.addCallback(_check_directory_html, "-IMM")
3561 d.addCallback(_check_directory_html, "")
3563 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3564 def _check_directory_json(res, expect_rw_uri):
3565 data = simplejson.loads(res)
3566 self.failUnlessEqual(data[0], "dirnode")
3567 f = data[1]["children"][name]
3568 self.failUnlessEqual(f[0], "unknown")
3570 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
3572 self.failIfIn("rw_uri", f[1])
3574 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
3576 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
3577 self.failUnless("metadata" in f[1])
3578 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3580 def _check_info(res, expect_rw_uri, expect_ro_uri):
3581 self.failUnlessIn("Object Type: <span>unknown</span>", res)
3583 self.failUnlessIn(unknown_rwcap, res)
3586 self.failUnlessIn(unknown_immcap, res)
3588 self.failUnlessIn(unknown_rocap, res)
3590 self.failIfIn(unknown_rocap, res)
3591 self.failIfIn("Raw data as", res)
3592 self.failIfIn("Directory writecap", res)
3593 self.failIfIn("Checker Operations", res)
3594 self.failIfIn("Mutable File Operations", res)
3595 self.failIfIn("Directory Operations", res)
3597 # FIXME: these should have expect_rw_uri=not immutable; I don't know
3598 # why they fail. Possibly related to ticket #922.
3600 d.addCallback(lambda ign: self.GET(expected_info_url))
3601 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3602 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3603 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3605 def _check_json(res, expect_rw_uri):
3606 data = simplejson.loads(res)
3607 self.failUnlessEqual(data[0], "unknown")
3609 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
3611 self.failIfIn("rw_uri", data[1])
3614 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
3615 self.failUnlessReallyEqual(data[1]["mutable"], False)
3617 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3618 self.failUnlessReallyEqual(data[1]["mutable"], True)
3620 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3621 self.failIf("mutable" in data[1], data[1])
3623 # TODO: check metadata contents
3624 self.failUnless("metadata" in data[1])
3626 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3627 d.addCallback(_check_json, expect_rw_uri=not immutable)
3629 # and make sure that a read-only version of the directory can be
3630 # rendered too. This version will not have unknown_rwcap, whether
3631 # or not future_node was immutable.
3632 d.addCallback(lambda ign: self.GET(self.rourl))
3634 d.addCallback(_check_directory_html, "-IMM")
3636 d.addCallback(_check_directory_html, "-RO")
3638 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3639 d.addCallback(_check_directory_json, expect_rw_uri=False)
3641 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3642 d.addCallback(_check_json, expect_rw_uri=False)
3644 # TODO: check that getting t=info from the Info link in the ro directory
3645 # works, and does not include the writecap URI.
3648 def test_immutable_unknown(self):
3649 return self.test_unknown(immutable=True)
3651 def test_mutant_dirnodes_are_omitted(self):
3652 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3655 c = self.g.clients[0]
3660 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3661 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3662 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3664 # This method tests mainly dirnode, but we'd have to duplicate code in order to
3665 # test the dirnode and web layers separately.
3667 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3668 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3669 # When the directory is read, the mutants should be silently disposed of, leaving
3670 # their lonely sibling.
3671 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3672 # because immutable directories don't have a writecap and therefore that field
3673 # isn't (and can't be) decrypted.
3674 # TODO: The field still exists in the netstring. Technically we should check what
3675 # happens if something is put there (_unpack_contents should raise ValueError),
3676 # but that can wait.
3678 lonely_child = nm.create_from_cap(lonely_uri)
3679 mutant_ro_child = nm.create_from_cap(mut_read_uri)
3680 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3682 def _by_hook_or_by_crook():
3684 for n in [mutant_ro_child, mutant_write_in_ro_child]:
3685 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3687 mutant_write_in_ro_child.get_write_uri = lambda: None
3688 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3690 kids = {u"lonely": (lonely_child, {}),
3691 u"ro": (mutant_ro_child, {}),
3692 u"write-in-ro": (mutant_write_in_ro_child, {}),
3694 d = c.create_immutable_dirnode(kids)
3697 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3698 self.failIf(dn.is_mutable())
3699 self.failUnless(dn.is_readonly())
3700 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3701 self.failIf(hasattr(dn._node, 'get_writekey'))
3703 self.failUnless("RO-IMM" in rep)
3705 self.failUnlessIn("CHK", cap.to_string())
3708 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3709 return download_to_data(dn._node)
3710 d.addCallback(_created)
3712 def _check_data(data):
3713 # Decode the netstring representation of the directory to check that all children
3714 # are present. This is a bit of an abstraction violation, but there's not really
3715 # any other way to do it given that the real DirectoryNode._unpack_contents would
3716 # strip the mutant children out (which is what we're trying to test, later).
3719 while position < len(data):
3720 entries, position = split_netstring(data, 1, position)
3722 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3723 name = name_utf8.decode("utf-8")
3724 self.failUnless(rwcapdata == "")
3725 self.failUnless(name in kids)
3726 (expected_child, ign) = kids[name]
3727 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
3730 self.failUnlessReallyEqual(numkids, 3)
3731 return self.rootnode.list()
3732 d.addCallback(_check_data)
3734 # Now when we use the real directory listing code, the mutants should be absent.
3735 def _check_kids(children):
3736 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
3737 lonely_node, lonely_metadata = children[u"lonely"]
3739 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
3740 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
3741 d.addCallback(_check_kids)
3743 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3744 d.addCallback(lambda n: n.list())
3745 d.addCallback(_check_kids) # again with dirnode recreated from cap
3747 # Make sure the lonely child can be listed in HTML...
3748 d.addCallback(lambda ign: self.GET(self.rooturl))
3749 def _check_html(res):
3750 self.failIfIn("URI:SSK", res)
3751 get_lonely = "".join([r'<td>FILE</td>',
3753 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3755 r'\s+<td align="right">%d</td>' % len("one"),
3757 self.failUnless(re.search(get_lonely, res), res)
3759 # find the More Info link for name, should be relative
3760 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3761 info_url = mo.group(1)
3762 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3763 d.addCallback(_check_html)
3766 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3767 def _check_json(res):
3768 data = simplejson.loads(res)
3769 self.failUnlessEqual(data[0], "dirnode")
3770 listed_children = data[1]["children"]
3771 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
3772 ll_type, ll_data = listed_children[u"lonely"]
3773 self.failUnlessEqual(ll_type, "filenode")
3774 self.failIf("rw_uri" in ll_data)
3775 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
3776 d.addCallback(_check_json)
3779 def test_deep_check(self):
3780 self.basedir = "web/Grid/deep_check"
3782 c0 = self.g.clients[0]
3786 d = c0.create_dirnode()
3787 def _stash_root_and_create_file(n):
3789 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3790 return n.add_file(u"good", upload.Data(DATA, convergence=""))
3791 d.addCallback(_stash_root_and_create_file)
3792 def _stash_uri(fn, which):
3793 self.uris[which] = fn.get_uri()
3795 d.addCallback(_stash_uri, "good")
3796 d.addCallback(lambda ign:
3797 self.rootnode.add_file(u"small",
3798 upload.Data("literal",
3800 d.addCallback(_stash_uri, "small")
3801 d.addCallback(lambda ign:
3802 self.rootnode.add_file(u"sick",
3803 upload.Data(DATA+"1",
3805 d.addCallback(_stash_uri, "sick")
3807 # this tests that deep-check and stream-manifest will ignore
3808 # UnknownNode instances. Hopefully this will also cover deep-stats.
3809 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3810 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3812 def _clobber_shares(ignored):
3813 self.delete_shares_numbered(self.uris["sick"], [0,1])
3814 d.addCallback(_clobber_shares)
3822 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3825 units = [simplejson.loads(line)
3826 for line in res.splitlines()
3829 print "response is:", res
3830 print "undecodeable line was '%s'" % line
3832 self.failUnlessReallyEqual(len(units), 5+1)
3833 # should be parent-first
3835 self.failUnlessEqual(u0["path"], [])
3836 self.failUnlessEqual(u0["type"], "directory")
3837 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
3838 u0cr = u0["check-results"]
3839 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
3841 ugood = [u for u in units
3842 if u["type"] == "file" and u["path"] == [u"good"]][0]
3843 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
3844 ugoodcr = ugood["check-results"]
3845 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
3848 self.failUnlessEqual(stats["type"], "stats")
3850 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3851 self.failUnlessReallyEqual(s["count-literal-files"], 1)
3852 self.failUnlessReallyEqual(s["count-directories"], 1)
3853 self.failUnlessReallyEqual(s["count-unknown"], 1)
3854 d.addCallback(_done)
3856 d.addCallback(self.CHECK, "root", "t=stream-manifest")
3857 def _check_manifest(res):
3858 self.failUnless(res.endswith("\n"))
3859 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3860 self.failUnlessReallyEqual(len(units), 5+1)
3861 self.failUnlessEqual(units[-1]["type"], "stats")
3863 self.failUnlessEqual(first["path"], [])
3864 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
3865 self.failUnlessEqual(first["type"], "directory")
3866 stats = units[-1]["stats"]
3867 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3868 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
3869 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
3870 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3871 self.failUnlessReallyEqual(stats["count-unknown"], 1)
3872 d.addCallback(_check_manifest)
3874 # now add root/subdir and root/subdir/grandchild, then make subdir
3875 # unrecoverable, then see what happens
3877 d.addCallback(lambda ign:
3878 self.rootnode.create_subdirectory(u"subdir"))
3879 d.addCallback(_stash_uri, "subdir")
3880 d.addCallback(lambda subdir_node:
3881 subdir_node.add_file(u"grandchild",
3882 upload.Data(DATA+"2",
3884 d.addCallback(_stash_uri, "grandchild")
3886 d.addCallback(lambda ign:
3887 self.delete_shares_numbered(self.uris["subdir"],
3895 # root/subdir [unrecoverable]
3896 # root/subdir/grandchild
3898 # how should a streaming-JSON API indicate fatal error?
3899 # answer: emit ERROR: instead of a JSON string
3901 d.addCallback(self.CHECK, "root", "t=stream-manifest")
3902 def _check_broken_manifest(res):
3903 lines = res.splitlines()
3905 for (i,line) in enumerate(lines)
3906 if line.startswith("ERROR:")]
3908 self.fail("no ERROR: in output: %s" % (res,))
3909 first_error = error_lines[0]
3910 error_line = lines[first_error]
3911 error_msg = lines[first_error+1:]
3912 error_msg_s = "\n".join(error_msg) + "\n"
3913 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3915 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3916 units = [simplejson.loads(line) for line in lines[:first_error]]
3917 self.failUnlessReallyEqual(len(units), 6) # includes subdir
3918 last_unit = units[-1]
3919 self.failUnlessEqual(last_unit["path"], ["subdir"])
3920 d.addCallback(_check_broken_manifest)
3922 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3923 def _check_broken_deepcheck(res):
3924 lines = res.splitlines()
3926 for (i,line) in enumerate(lines)
3927 if line.startswith("ERROR:")]
3929 self.fail("no ERROR: in output: %s" % (res,))
3930 first_error = error_lines[0]
3931 error_line = lines[first_error]
3932 error_msg = lines[first_error+1:]
3933 error_msg_s = "\n".join(error_msg) + "\n"
3934 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3936 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3937 units = [simplejson.loads(line) for line in lines[:first_error]]
3938 self.failUnlessReallyEqual(len(units), 6) # includes subdir
3939 last_unit = units[-1]
3940 self.failUnlessEqual(last_unit["path"], ["subdir"])
3941 r = last_unit["check-results"]["results"]
3942 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
3943 self.failUnlessReallyEqual(r["count-shares-good"], 1)
3944 self.failUnlessReallyEqual(r["recoverable"], False)
3945 d.addCallback(_check_broken_deepcheck)
3947 d.addErrback(self.explain_web_error)
3950 def test_deep_check_and_repair(self):
3951 self.basedir = "web/Grid/deep_check_and_repair"
3953 c0 = self.g.clients[0]
3957 d = c0.create_dirnode()
3958 def _stash_root_and_create_file(n):
3960 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3961 return n.add_file(u"good", upload.Data(DATA, convergence=""))
3962 d.addCallback(_stash_root_and_create_file)
3963 def _stash_uri(fn, which):
3964 self.uris[which] = fn.get_uri()
3965 d.addCallback(_stash_uri, "good")
3966 d.addCallback(lambda ign:
3967 self.rootnode.add_file(u"small",
3968 upload.Data("literal",
3970 d.addCallback(_stash_uri, "small")
3971 d.addCallback(lambda ign:
3972 self.rootnode.add_file(u"sick",
3973 upload.Data(DATA+"1",
3975 d.addCallback(_stash_uri, "sick")
3976 #d.addCallback(lambda ign:
3977 # self.rootnode.add_file(u"dead",
3978 # upload.Data(DATA+"2",
3980 #d.addCallback(_stash_uri, "dead")
3982 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3983 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3984 #d.addCallback(_stash_uri, "corrupt")
3986 def _clobber_shares(ignored):
3987 good_shares = self.find_uri_shares(self.uris["good"])
3988 self.failUnlessReallyEqual(len(good_shares), 10)
3989 sick_shares = self.find_uri_shares(self.uris["sick"])
3990 os.unlink(sick_shares[0][2])
3991 #dead_shares = self.find_uri_shares(self.uris["dead"])
3992 #for i in range(1, 10):
3993 # os.unlink(dead_shares[i][2])
3995 #c_shares = self.find_uri_shares(self.uris["corrupt"])
3996 #cso = CorruptShareOptions()
3997 #cso.stdout = StringIO()
3998 #cso.parseOptions([c_shares[0][2]])
4000 d.addCallback(_clobber_shares)
4003 # root/good CHK, 10 shares
4005 # root/sick CHK, 9 shares
4007 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4009 units = [simplejson.loads(line)
4010 for line in res.splitlines()
4012 self.failUnlessReallyEqual(len(units), 4+1)
4013 # should be parent-first
4015 self.failUnlessEqual(u0["path"], [])
4016 self.failUnlessEqual(u0["type"], "directory")
4017 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4018 u0crr = u0["check-and-repair-results"]
4019 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4020 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4022 ugood = [u for u in units
4023 if u["type"] == "file" and u["path"] == [u"good"]][0]
4024 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4025 ugoodcrr = ugood["check-and-repair-results"]
4026 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4027 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4029 usick = [u for u in units
4030 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4031 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4032 usickcrr = usick["check-and-repair-results"]
4033 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4034 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4035 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4036 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4039 self.failUnlessEqual(stats["type"], "stats")
4041 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4042 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4043 self.failUnlessReallyEqual(s["count-directories"], 1)
4044 d.addCallback(_done)
4046 d.addErrback(self.explain_web_error)
4049 def _count_leases(self, ignored, which):
4050 u = self.uris[which]
4051 shares = self.find_uri_shares(u)
4053 for shnum, serverid, fn in shares:
4054 sf = get_share_file(fn)
4055 num_leases = len(list(sf.get_leases()))
4056 lease_counts.append( (fn, num_leases) )
4059 def _assert_leasecount(self, lease_counts, expected):
4060 for (fn, num_leases) in lease_counts:
4061 if num_leases != expected:
4062 self.fail("expected %d leases, have %d, on %s" %
4063 (expected, num_leases, fn))
4065 def test_add_lease(self):
4066 self.basedir = "web/Grid/add_lease"
4067 self.set_up_grid(num_clients=2)
4068 c0 = self.g.clients[0]
4071 d = c0.upload(upload.Data(DATA, convergence=""))
4072 def _stash_uri(ur, which):
4073 self.uris[which] = ur.uri
4074 d.addCallback(_stash_uri, "one")
4075 d.addCallback(lambda ign:
4076 c0.upload(upload.Data(DATA+"1", convergence="")))
4077 d.addCallback(_stash_uri, "two")
4078 def _stash_mutable_uri(n, which):
4079 self.uris[which] = n.get_uri()
4080 assert isinstance(self.uris[which], str)
4081 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
4082 d.addCallback(_stash_mutable_uri, "mutable")
4084 def _compute_fileurls(ignored):
4086 for which in self.uris:
4087 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4088 d.addCallback(_compute_fileurls)
4090 d.addCallback(self._count_leases, "one")
4091 d.addCallback(self._assert_leasecount, 1)
4092 d.addCallback(self._count_leases, "two")
4093 d.addCallback(self._assert_leasecount, 1)
4094 d.addCallback(self._count_leases, "mutable")
4095 d.addCallback(self._assert_leasecount, 1)
4097 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4098 def _got_html_good(res):
4099 self.failUnless("Healthy" in res, res)
4100 self.failIf("Not Healthy" in res, res)
4101 d.addCallback(_got_html_good)
4103 d.addCallback(self._count_leases, "one")
4104 d.addCallback(self._assert_leasecount, 1)
4105 d.addCallback(self._count_leases, "two")
4106 d.addCallback(self._assert_leasecount, 1)
4107 d.addCallback(self._count_leases, "mutable")
4108 d.addCallback(self._assert_leasecount, 1)
4110 # this CHECK uses the original client, which uses the same
4111 # lease-secrets, so it will just renew the original lease
4112 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4113 d.addCallback(_got_html_good)
4115 d.addCallback(self._count_leases, "one")
4116 d.addCallback(self._assert_leasecount, 1)
4117 d.addCallback(self._count_leases, "two")
4118 d.addCallback(self._assert_leasecount, 1)
4119 d.addCallback(self._count_leases, "mutable")
4120 d.addCallback(self._assert_leasecount, 1)
4122 # this CHECK uses an alternate client, which adds a second lease
4123 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4124 d.addCallback(_got_html_good)
4126 d.addCallback(self._count_leases, "one")
4127 d.addCallback(self._assert_leasecount, 2)
4128 d.addCallback(self._count_leases, "two")
4129 d.addCallback(self._assert_leasecount, 1)
4130 d.addCallback(self._count_leases, "mutable")
4131 d.addCallback(self._assert_leasecount, 1)
4133 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4134 d.addCallback(_got_html_good)
4136 d.addCallback(self._count_leases, "one")
4137 d.addCallback(self._assert_leasecount, 2)
4138 d.addCallback(self._count_leases, "two")
4139 d.addCallback(self._assert_leasecount, 1)
4140 d.addCallback(self._count_leases, "mutable")
4141 d.addCallback(self._assert_leasecount, 1)
4143 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4145 d.addCallback(_got_html_good)
4147 d.addCallback(self._count_leases, "one")
4148 d.addCallback(self._assert_leasecount, 2)
4149 d.addCallback(self._count_leases, "two")
4150 d.addCallback(self._assert_leasecount, 1)
4151 d.addCallback(self._count_leases, "mutable")
4152 d.addCallback(self._assert_leasecount, 2)
4154 d.addErrback(self.explain_web_error)
4157 def test_deep_add_lease(self):
4158 self.basedir = "web/Grid/deep_add_lease"
4159 self.set_up_grid(num_clients=2)
4160 c0 = self.g.clients[0]
4164 d = c0.create_dirnode()
4165 def _stash_root_and_create_file(n):
4167 self.uris["root"] = n.get_uri()
4168 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4169 return n.add_file(u"one", upload.Data(DATA, convergence=""))
4170 d.addCallback(_stash_root_and_create_file)
4171 def _stash_uri(fn, which):
4172 self.uris[which] = fn.get_uri()
4173 d.addCallback(_stash_uri, "one")
4174 d.addCallback(lambda ign:
4175 self.rootnode.add_file(u"small",
4176 upload.Data("literal",
4178 d.addCallback(_stash_uri, "small")
4180 d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4181 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4182 d.addCallback(_stash_uri, "mutable")
4184 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4186 units = [simplejson.loads(line)
4187 for line in res.splitlines()
4189 # root, one, small, mutable, stats
4190 self.failUnlessReallyEqual(len(units), 4+1)
4191 d.addCallback(_done)
4193 d.addCallback(self._count_leases, "root")
4194 d.addCallback(self._assert_leasecount, 1)
4195 d.addCallback(self._count_leases, "one")
4196 d.addCallback(self._assert_leasecount, 1)
4197 d.addCallback(self._count_leases, "mutable")
4198 d.addCallback(self._assert_leasecount, 1)
4200 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4201 d.addCallback(_done)
4203 d.addCallback(self._count_leases, "root")
4204 d.addCallback(self._assert_leasecount, 1)
4205 d.addCallback(self._count_leases, "one")
4206 d.addCallback(self._assert_leasecount, 1)
4207 d.addCallback(self._count_leases, "mutable")
4208 d.addCallback(self._assert_leasecount, 1)
4210 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4212 d.addCallback(_done)
4214 d.addCallback(self._count_leases, "root")
4215 d.addCallback(self._assert_leasecount, 2)
4216 d.addCallback(self._count_leases, "one")
4217 d.addCallback(self._assert_leasecount, 2)
4218 d.addCallback(self._count_leases, "mutable")
4219 d.addCallback(self._assert_leasecount, 2)
4221 d.addErrback(self.explain_web_error)
4225 def test_exceptions(self):
4226 self.basedir = "web/Grid/exceptions"
4227 self.set_up_grid(num_clients=1, num_servers=2)
4228 c0 = self.g.clients[0]
4229 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4232 d = c0.create_dirnode()
4234 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4235 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4237 d.addCallback(_stash_root)
4238 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4240 self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4241 self.delete_shares_numbered(ur.uri, range(1,10))
4243 u = uri.from_string(ur.uri)
4244 u.key = testutil.flip_bit(u.key, 0)
4245 baduri = u.to_string()
4246 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4247 d.addCallback(_stash_bad)
4248 d.addCallback(lambda ign: c0.create_dirnode())
4249 def _mangle_dirnode_1share(n):
4251 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4252 self.fileurls["dir-1share-json"] = url + "?t=json"
4253 self.delete_shares_numbered(u, range(1,10))
4254 d.addCallback(_mangle_dirnode_1share)
4255 d.addCallback(lambda ign: c0.create_dirnode())
4256 def _mangle_dirnode_0share(n):
4258 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4259 self.fileurls["dir-0share-json"] = url + "?t=json"
4260 self.delete_shares_numbered(u, range(0,10))
4261 d.addCallback(_mangle_dirnode_0share)
4263 # NotEnoughSharesError should be reported sensibly, with a
4264 # text/plain explanation of the problem, and perhaps some
4265 # information on which shares *could* be found.
4267 d.addCallback(lambda ignored:
4268 self.shouldHTTPError("GET unrecoverable",
4269 410, "Gone", "NoSharesError",
4270 self.GET, self.fileurls["0shares"]))
4271 def _check_zero_shares(body):
4272 self.failIf("<html>" in body, body)
4273 body = " ".join(body.strip().split())
4274 exp = ("NoSharesError: no shares could be found. "
4275 "Zero shares usually indicates a corrupt URI, or that "
4276 "no servers were connected, but it might also indicate "
4277 "severe corruption. You should perform a filecheck on "
4278 "this object to learn more. The full error message is: "
4279 "no shares (need 3). Last failure: None")
4280 self.failUnlessReallyEqual(exp, body)
4281 d.addCallback(_check_zero_shares)
4284 d.addCallback(lambda ignored:
4285 self.shouldHTTPError("GET 1share",
4286 410, "Gone", "NotEnoughSharesError",
4287 self.GET, self.fileurls["1share"]))
4288 def _check_one_share(body):
4289 self.failIf("<html>" in body, body)
4290 body = " ".join(body.strip().split())
4291 msgbase = ("NotEnoughSharesError: This indicates that some "
4292 "servers were unavailable, or that shares have been "
4293 "lost to server departure, hard drive failure, or disk "
4294 "corruption. You should perform a filecheck on "
4295 "this object to learn more. The full error message is:"
4297 msg1 = msgbase + (" ran out of shares:"
4300 " overdue= unused= need 3. Last failure: None")
4301 msg2 = msgbase + (" ran out of shares:"
4303 " pending=Share(sh0-on-xgru5)"
4304 " overdue= unused= need 3. Last failure: None")
4305 self.failUnless(body == msg1 or body == msg2, body)
4306 d.addCallback(_check_one_share)
4308 d.addCallback(lambda ignored:
4309 self.shouldHTTPError("GET imaginary",
4310 404, "Not Found", None,
4311 self.GET, self.fileurls["imaginary"]))
4312 def _missing_child(body):
4313 self.failUnless("No such child: imaginary" in body, body)
4314 d.addCallback(_missing_child)
4316 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4317 def _check_0shares_dir_html(body):
4318 self.failUnless("<html>" in body, body)
4319 # we should see the regular page, but without the child table or
4321 body = " ".join(body.strip().split())
4322 self.failUnlessIn('href="?t=info">More info on this directory',
4324 exp = ("UnrecoverableFileError: the directory (or mutable file) "
4325 "could not be retrieved, because there were insufficient "
4326 "good shares. This might indicate that no servers were "
4327 "connected, insufficient servers were connected, the URI "
4328 "was corrupt, or that shares have been lost due to server "
4329 "departure, hard drive failure, or disk corruption. You "
4330 "should perform a filecheck on this object to learn more.")
4331 self.failUnlessIn(exp, body)
4332 self.failUnlessIn("No upload forms: directory is unreadable", body)
4333 d.addCallback(_check_0shares_dir_html)
4335 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4336 def _check_1shares_dir_html(body):
4337 # at some point, we'll split UnrecoverableFileError into 0-shares
4338 # and some-shares like we did for immutable files (since there
4339 # are different sorts of advice to offer in each case). For now,
4340 # they present the same way.
4341 self.failUnless("<html>" in body, body)
4342 body = " ".join(body.strip().split())
4343 self.failUnlessIn('href="?t=info">More info on this directory',
4345 exp = ("UnrecoverableFileError: the directory (or mutable file) "
4346 "could not be retrieved, because there were insufficient "
4347 "good shares. This might indicate that no servers were "
4348 "connected, insufficient servers were connected, the URI "
4349 "was corrupt, or that shares have been lost due to server "
4350 "departure, hard drive failure, or disk corruption. You "
4351 "should perform a filecheck on this object to learn more.")
4352 self.failUnlessIn(exp, body)
4353 self.failUnlessIn("No upload forms: directory is unreadable", body)
4354 d.addCallback(_check_1shares_dir_html)
4356 d.addCallback(lambda ignored:
4357 self.shouldHTTPError("GET dir-0share-json",
4358 410, "Gone", "UnrecoverableFileError",
4360 self.fileurls["dir-0share-json"]))
4361 def _check_unrecoverable_file(body):
4362 self.failIf("<html>" in body, body)
4363 body = " ".join(body.strip().split())
4364 exp = ("UnrecoverableFileError: the directory (or mutable file) "
4365 "could not be retrieved, because there were insufficient "
4366 "good shares. This might indicate that no servers were "
4367 "connected, insufficient servers were connected, the URI "
4368 "was corrupt, or that shares have been lost due to server "
4369 "departure, hard drive failure, or disk corruption. You "
4370 "should perform a filecheck on this object to learn more.")
4371 self.failUnlessReallyEqual(exp, body)
4372 d.addCallback(_check_unrecoverable_file)
4374 d.addCallback(lambda ignored:
4375 self.shouldHTTPError("GET dir-1share-json",
4376 410, "Gone", "UnrecoverableFileError",
4378 self.fileurls["dir-1share-json"]))
4379 d.addCallback(_check_unrecoverable_file)
4381 d.addCallback(lambda ignored:
4382 self.shouldHTTPError("GET imaginary",
4383 404, "Not Found", None,
4384 self.GET, self.fileurls["imaginary"]))
4386 # attach a webapi child that throws a random error, to test how it
4388 w = c0.getServiceNamed("webish")
4389 w.root.putChild("ERRORBOOM", ErrorBoom())
4391 # "Accept: */*" : should get a text/html stack trace
4392 # "Accept: text/plain" : should get a text/plain stack trace
4393 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4394 # no Accept header: should get a text/html stack trace
4396 d.addCallback(lambda ignored:
4397 self.shouldHTTPError("GET errorboom_html",
4398 500, "Internal Server Error", None,
4399 self.GET, "ERRORBOOM",
4400 headers={"accept": ["*/*"]}))
4401 def _internal_error_html1(body):
4402 self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4403 d.addCallback(_internal_error_html1)
4405 d.addCallback(lambda ignored:
4406 self.shouldHTTPError("GET errorboom_text",
4407 500, "Internal Server Error", None,
4408 self.GET, "ERRORBOOM",
4409 headers={"accept": ["text/plain"]}))
4410 def _internal_error_text2(body):
4411 self.failIf("<html>" in body, body)
4412 self.failUnless(body.startswith("Traceback "), body)
4413 d.addCallback(_internal_error_text2)
4415 CLI_accepts = "text/plain, application/octet-stream"
4416 d.addCallback(lambda ignored:
4417 self.shouldHTTPError("GET errorboom_text",
4418 500, "Internal Server Error", None,
4419 self.GET, "ERRORBOOM",
4420 headers={"accept": [CLI_accepts]}))
4421 def _internal_error_text3(body):
4422 self.failIf("<html>" in body, body)
4423 self.failUnless(body.startswith("Traceback "), body)
4424 d.addCallback(_internal_error_text3)
4426 d.addCallback(lambda ignored:
4427 self.shouldHTTPError("GET errorboom_text",
4428 500, "Internal Server Error", None,
4429 self.GET, "ERRORBOOM"))
4430 def _internal_error_html4(body):
4431 self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4432 d.addCallback(_internal_error_html4)
4434 def _flush_errors(res):
4435 # Trial: please ignore the CompletelyUnhandledError in the logs
4436 self.flushLoggedErrors(CompletelyUnhandledError)
4438 d.addBoth(_flush_errors)
4442 class CompletelyUnhandledError(Exception):
4444 class ErrorBoom(rend.Page):
4445 def beforeRender(self, ctx):
4446 raise CompletelyUnhandledError("whoops")