2 import os.path, re, urllib, time
4 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
11 from nevow import rend
12 from allmydata import interfaces, uri, webish, dirnode
13 from allmydata.storage.shares import get_share_file
14 from allmydata.storage_client import StorageFarmBroker
15 from allmydata.immutable import upload
16 from allmydata.immutable.downloader.status import DownloadStatus
17 from allmydata.dirnode import DirectoryNode
18 from allmydata.nodemaker import NodeMaker
19 from allmydata.unknown import UnknownNode
20 from allmydata.web import status, common
21 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
22 from allmydata.util import fileutil, base32, hashutil
23 from allmydata.util.consumer import download_to_data
24 from allmydata.util.netstring import split_netstring
25 from allmydata.util.encodingutil import to_str
26 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
27 create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
28 from allmydata.interfaces import IMutableFileNode
29 from allmydata.mutable import servermap, publish, retrieve
30 import allmydata.test.common_util as testutil
31 from allmydata.test.no_network import GridTestMixin
32 from allmydata.test.common_web import HTTPClientGETFactory, \
34 from allmydata.client import Client, SecretHolder
36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
37 # create a webserver that works against them
39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
45 class FakeStatsProvider:
47 stats = {'stats': {}, 'counters': {}}
50 class FakeNodeMaker(NodeMaker):
51 def _create_lit(self, cap):
52 return FakeCHKFileNode(cap)
53 def _create_immutable(self, cap):
54 return FakeCHKFileNode(cap)
55 def _create_mutable(self, cap):
56 return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
57 def create_mutable_file(self, contents="", keysize=None):
58 n = FakeMutableFileNode(None, None, None, None)
59 return n.create(contents)
61 class FakeUploader(service.Service):
63 def upload(self, uploadable, history=None):
64 d = uploadable.get_size()
65 d.addCallback(lambda size: uploadable.read(size))
68 n = create_chk_filenode(data)
69 results = upload.UploadResults()
70 results.uri = n.get_uri()
72 d.addCallback(_got_data)
74 def get_helper_info(self):
78 def __init__(self, binaryserverid):
79 self.binaryserverid = binaryserverid
80 def get_name(self): return "short"
81 def get_longname(self): return "long"
82 def get_serverid(self): return self.binaryserverid
85 ds = DownloadStatus("storage_index", 1234)
88 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
89 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
90 storage_index = hashutil.storage_index_hash("SI")
91 e0 = ds.add_segment_request(0, now)
93 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
94 e1 = ds.add_segment_request(1, now+2)
96 # two outstanding requests
97 e2 = ds.add_segment_request(2, now+4)
98 e3 = ds.add_segment_request(3, now+5)
99 del e2,e3 # hush pyflakes
101 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
102 e = ds.add_segment_request(4, now)
104 e.deliver(now, 0, 140, 0.5)
106 e = ds.add_dyhb_request(serverA, now)
107 e.finished([1,2], now+1)
108 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
110 e = ds.add_read_event(0, 120, now)
111 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
113 e = ds.add_read_event(120, 30, now+2) # left unfinished
115 e = ds.add_block_request(serverA, 1, 100, 20, now)
116 e.finished(20, now+1)
117 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
119 # make sure that add_read_event() can come first too
120 ds1 = DownloadStatus(storage_index, 1234)
121 e = ds1.add_read_event(0, 120, now)
122 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
128 _all_upload_status = [upload.UploadStatus()]
129 _all_download_status = [build_one_ds()]
130 _all_mapupdate_statuses = [servermap.UpdateStatus()]
131 _all_publish_statuses = [publish.PublishStatus()]
132 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
134 def list_all_upload_statuses(self):
135 return self._all_upload_status
136 def list_all_download_statuses(self):
137 return self._all_download_status
138 def list_all_mapupdate_statuses(self):
139 return self._all_mapupdate_statuses
140 def list_all_publish_statuses(self):
141 return self._all_publish_statuses
142 def list_all_retrieve_statuses(self):
143 return self._all_retrieve_statuses
144 def list_all_helper_statuses(self):
147 class FakeClient(Client):
149 # don't upcall to Client.__init__, since we only want to initialize a
151 service.MultiService.__init__(self)
152 self.nodeid = "fake_nodeid"
153 self.nickname = "fake_nickname"
154 self.introducer_furl = "None"
155 self.stats_provider = FakeStatsProvider()
156 self._secret_holder = SecretHolder("lease secret", "convergence secret")
158 self.convergence = "some random string"
159 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
160 self.introducer_client = None
161 self.history = FakeHistory()
162 self.uploader = FakeUploader()
163 self.uploader.setServiceParent(self)
164 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
168 def startService(self):
169 return service.MultiService.startService(self)
170 def stopService(self):
171 return service.MultiService.stopService(self)
173 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
175 class WebMixin(object):
177 self.s = FakeClient()
178 self.s.startService()
179 self.staticdir = self.mktemp()
181 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
183 self.ws.setServiceParent(self.s)
184 self.webish_port = self.ws.getPortnum()
185 self.webish_url = self.ws.getURL()
186 assert self.webish_url.endswith("/")
187 self.webish_url = self.webish_url[:-1] # these tests add their own /
189 l = [ self.s.create_dirnode() for x in range(6) ]
190 d = defer.DeferredList(l)
192 self.public_root = res[0][1]
193 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
194 self.public_url = "/uri/" + self.public_root.get_uri()
195 self.private_root = res[1][1]
199 self._foo_uri = foo.get_uri()
200 self._foo_readonly_uri = foo.get_readonly_uri()
201 self._foo_verifycap = foo.get_verify_cap().to_string()
202 # NOTE: we ignore the deferred on all set_uri() calls, because we
203 # know the fake nodes do these synchronously
204 self.public_root.set_uri(u"foo", foo.get_uri(),
205 foo.get_readonly_uri())
207 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
208 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
209 self._bar_txt_verifycap = n.get_verify_cap().to_string()
211 foo.set_uri(u"empty", res[3][1].get_uri(),
212 res[3][1].get_readonly_uri())
213 sub_uri = res[4][1].get_uri()
214 self._sub_uri = sub_uri
215 foo.set_uri(u"sub", sub_uri, sub_uri)
216 sub = self.s.create_node_from_uri(sub_uri)
218 _ign, n, blocking_uri = self.makefile(1)
219 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
221 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
222 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
223 # still think of it as an umlaut
224 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
226 _ign, n, baz_file = self.makefile(2)
227 self._baz_file_uri = baz_file
228 sub.set_uri(u"baz.txt", baz_file, baz_file)
230 _ign, n, self._bad_file_uri = self.makefile(3)
231 # this uri should not be downloadable
232 del FakeCHKFileNode.all_contents[self._bad_file_uri]
235 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
236 rodir.get_readonly_uri())
237 rodir.set_uri(u"nor", baz_file, baz_file)
242 # public/foo/blockingfile
245 # public/foo/sub/baz.txt
247 # public/reedownlee/nor
248 self.NEWFILE_CONTENTS = "newfile contents\n"
250 return foo.get_metadata_for(u"bar.txt")
252 def _got_metadata(metadata):
253 self._bar_txt_metadata = metadata
254 d.addCallback(_got_metadata)
257 def makefile(self, number):
258 contents = "contents of file %s\n" % number
259 n = create_chk_filenode(contents)
260 return contents, n, n.get_uri()
263 return self.s.stopService()
265 def failUnlessIsBarDotTxt(self, res):
266 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
268 def failUnlessIsBarJSON(self, res):
269 data = simplejson.loads(res)
270 self.failUnless(isinstance(data, list))
271 self.failUnlessEqual(data[0], "filenode")
272 self.failUnless(isinstance(data[1], dict))
273 self.failIf(data[1]["mutable"])
274 self.failIf("rw_uri" in data[1]) # immutable
275 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
276 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
277 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
279 def failUnlessIsFooJSON(self, res):
280 data = simplejson.loads(res)
281 self.failUnless(isinstance(data, list))
282 self.failUnlessEqual(data[0], "dirnode", res)
283 self.failUnless(isinstance(data[1], dict))
284 self.failUnless(data[1]["mutable"])
285 self.failUnless("rw_uri" in data[1]) # mutable
286 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
287 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
288 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
290 kidnames = sorted([unicode(n) for n in data[1]["children"]])
291 self.failUnlessEqual(kidnames,
292 [u"bar.txt", u"blockingfile", u"empty",
293 u"n\u00fc.txt", u"sub"])
294 kids = dict( [(unicode(name),value)
296 in data[1]["children"].iteritems()] )
297 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
298 self.failUnlessIn("metadata", kids[u"sub"][1])
299 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
300 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
301 self.failUnlessIn("linkcrtime", tahoe_md)
302 self.failUnlessIn("linkmotime", tahoe_md)
303 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
304 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
305 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
306 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
307 self._bar_txt_verifycap)
308 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
309 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
310 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
311 self._bar_txt_metadata["tahoe"]["linkcrtime"])
312 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
315 def GET(self, urlpath, followRedirect=False, return_response=False,
317 # if return_response=True, this fires with (data, statuscode,
318 # respheaders) instead of just data.
319 assert not isinstance(urlpath, unicode)
320 url = self.webish_url + urlpath
321 factory = HTTPClientGETFactory(url, method="GET",
322 followRedirect=followRedirect, **kwargs)
323 reactor.connectTCP("localhost", self.webish_port, factory)
326 return (data, factory.status, factory.response_headers)
328 d.addCallback(_got_data)
329 return factory.deferred
331 def HEAD(self, urlpath, return_response=False, **kwargs):
332 # this requires some surgery, because twisted.web.client doesn't want
333 # to give us back the response headers.
334 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
335 reactor.connectTCP("localhost", self.webish_port, factory)
338 return (data, factory.status, factory.response_headers)
340 d.addCallback(_got_data)
341 return factory.deferred
343 def PUT(self, urlpath, data, **kwargs):
344 url = self.webish_url + urlpath
345 return client.getPage(url, method="PUT", postdata=data, **kwargs)
347 def DELETE(self, urlpath):
348 url = self.webish_url + urlpath
349 return client.getPage(url, method="DELETE")
351 def POST(self, urlpath, followRedirect=False, **fields):
352 sepbase = "boogabooga"
356 form.append('Content-Disposition: form-data; name="_charset"')
360 for name, value in fields.iteritems():
361 if isinstance(value, tuple):
362 filename, value = value
363 form.append('Content-Disposition: form-data; name="%s"; '
364 'filename="%s"' % (name, filename.encode("utf-8")))
366 form.append('Content-Disposition: form-data; name="%s"' % name)
368 if isinstance(value, unicode):
369 value = value.encode("utf-8")
372 assert isinstance(value, str)
379 body = "\r\n".join(form) + "\r\n"
380 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
381 return self.POST2(urlpath, body, headers, followRedirect)
383 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
384 url = self.webish_url + urlpath
385 return client.getPage(url, method="POST", postdata=body,
386 headers=headers, followRedirect=followRedirect)
388 def shouldFail(self, res, expected_failure, which,
389 substring=None, response_substring=None):
390 if isinstance(res, failure.Failure):
391 res.trap(expected_failure)
393 self.failUnless(substring in str(res),
394 "substring '%s' not in '%s'"
395 % (substring, str(res)))
396 if response_substring:
397 self.failUnless(response_substring in res.value.response,
398 "response substring '%s' not in '%s'"
399 % (response_substring, res.value.response))
401 self.fail("%s was supposed to raise %s, not get '%s'" %
402 (which, expected_failure, res))
404 def shouldFail2(self, expected_failure, which, substring,
406 callable, *args, **kwargs):
407 assert substring is None or isinstance(substring, str)
408 assert response_substring is None or isinstance(response_substring, str)
409 d = defer.maybeDeferred(callable, *args, **kwargs)
411 if isinstance(res, failure.Failure):
412 res.trap(expected_failure)
414 self.failUnless(substring in str(res),
415 "%s: substring '%s' not in '%s'"
416 % (which, substring, str(res)))
417 if response_substring:
418 self.failUnless(response_substring in res.value.response,
419 "%s: response substring '%s' not in '%s'"
421 response_substring, res.value.response))
423 self.fail("%s was supposed to raise %s, not get '%s'" %
424 (which, expected_failure, res))
428 def should404(self, res, which):
429 if isinstance(res, failure.Failure):
430 res.trap(error.Error)
431 self.failUnlessReallyEqual(res.value.status, "404")
433 self.fail("%s was supposed to Error(404), not get '%s'" %
436 def should302(self, res, which):
437 if isinstance(res, failure.Failure):
438 res.trap(error.Error)
439 self.failUnlessReallyEqual(res.value.status, "302")
441 self.fail("%s was supposed to Error(302), not get '%s'" %
445 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
446 def test_create(self):
449 def test_welcome(self):
452 self.failUnless('Welcome To Tahoe-LAFS' in res, res)
454 self.s.basedir = 'web/test_welcome'
455 fileutil.make_dirs("web/test_welcome")
456 fileutil.make_dirs("web/test_welcome/private")
458 d.addCallback(_check)
461 def test_provisioning(self):
462 d = self.GET("/provisioning/")
464 self.failUnless('Provisioning Tool' in res)
465 fields = {'filled': True,
466 "num_users": int(50e3),
467 "files_per_user": 1000,
468 "space_per_user": int(1e9),
469 "sharing_ratio": 1.0,
470 "encoding_parameters": "3-of-10-5",
472 "ownership_mode": "A",
473 "download_rate": 100,
478 return self.POST("/provisioning/", **fields)
480 d.addCallback(_check)
482 self.failUnless('Provisioning Tool' in res)
483 self.failUnless("Share space consumed: 167.01TB" in res)
485 fields = {'filled': True,
486 "num_users": int(50e6),
487 "files_per_user": 1000,
488 "space_per_user": int(5e9),
489 "sharing_ratio": 1.0,
490 "encoding_parameters": "25-of-100-50",
491 "num_servers": 30000,
492 "ownership_mode": "E",
493 "drive_failure_model": "U",
495 "download_rate": 1000,
500 return self.POST("/provisioning/", **fields)
501 d.addCallback(_check2)
503 self.failUnless("Share space consumed: huge!" in res)
504 fields = {'filled': True}
505 return self.POST("/provisioning/", **fields)
506 d.addCallback(_check3)
508 self.failUnless("Share space consumed:" in res)
509 d.addCallback(_check4)
512 def test_reliability_tool(self):
514 from allmydata import reliability
515 _hush_pyflakes = reliability
518 raise unittest.SkipTest("reliability tool requires NumPy")
520 d = self.GET("/reliability/")
522 self.failUnless('Reliability Tool' in res)
523 fields = {'drive_lifetime': "8Y",
528 "check_period": "1M",
529 "report_period": "3M",
532 return self.POST("/reliability/", **fields)
534 d.addCallback(_check)
536 self.failUnless('Reliability Tool' in res)
537 r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
538 self.failUnless(re.search(r, res), res)
539 d.addCallback(_check2)
542 def test_status(self):
543 h = self.s.get_history()
544 dl_num = h.list_all_download_statuses()[0].get_counter()
545 ul_num = h.list_all_upload_statuses()[0].get_counter()
546 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
547 pub_num = h.list_all_publish_statuses()[0].get_counter()
548 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
549 d = self.GET("/status", followRedirect=True)
551 self.failUnless('Upload and Download Status' in res, res)
552 self.failUnless('"down-%d"' % dl_num in res, res)
553 self.failUnless('"up-%d"' % ul_num in res, res)
554 self.failUnless('"mapupdate-%d"' % mu_num in res, res)
555 self.failUnless('"publish-%d"' % pub_num in res, res)
556 self.failUnless('"retrieve-%d"' % ret_num in res, res)
557 d.addCallback(_check)
558 d.addCallback(lambda res: self.GET("/status/?t=json"))
559 def _check_json(res):
560 data = simplejson.loads(res)
561 self.failUnless(isinstance(data, dict))
562 #active = data["active"]
563 # TODO: test more. We need a way to fake an active operation
565 d.addCallback(_check_json)
567 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
569 self.failUnless("File Download Status" in res, res)
570 d.addCallback(_check_dl)
571 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
572 def _check_dl_json(res):
573 data = simplejson.loads(res)
574 self.failUnless(isinstance(data, dict))
575 self.failUnless("read" in data)
576 self.failUnlessEqual(data["read"][0]["length"], 120)
577 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
578 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
579 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
580 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
581 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
582 # serverids[] keys are strings, since that's what JSON does, but
583 # we'd really like them to be ints
584 self.failUnlessEqual(data["serverids"]["0"], "phwr")
585 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
586 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
587 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
588 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
589 self.failUnless("dyhb" in data)
590 d.addCallback(_check_dl_json)
591 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
593 self.failUnless("File Upload Status" in res, res)
594 d.addCallback(_check_ul)
595 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
596 def _check_mapupdate(res):
597 self.failUnless("Mutable File Servermap Update Status" in res, res)
598 d.addCallback(_check_mapupdate)
599 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
600 def _check_publish(res):
601 self.failUnless("Mutable File Publish Status" in res, res)
602 d.addCallback(_check_publish)
603 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
604 def _check_retrieve(res):
605 self.failUnless("Mutable File Retrieve Status" in res, res)
606 d.addCallback(_check_retrieve)
610 def test_status_numbers(self):
611 drrm = status.DownloadResultsRendererMixin()
612 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
613 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
614 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
615 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
616 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
617 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
618 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
619 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
620 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
622 urrm = status.UploadResultsRendererMixin()
623 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
624 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
625 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
626 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
627 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
628 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
629 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
630 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
631 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
633 def test_GET_FILEURL(self):
634 d = self.GET(self.public_url + "/foo/bar.txt")
635 d.addCallback(self.failUnlessIsBarDotTxt)
638 def test_GET_FILEURL_range(self):
639 headers = {"range": "bytes=1-10"}
640 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
641 return_response=True)
642 def _got((res, status, headers)):
643 self.failUnlessReallyEqual(int(status), 206)
644 self.failUnless(headers.has_key("content-range"))
645 self.failUnlessReallyEqual(headers["content-range"][0],
646 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
647 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
651 def test_GET_FILEURL_partial_range(self):
652 headers = {"range": "bytes=5-"}
653 length = len(self.BAR_CONTENTS)
654 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
655 return_response=True)
656 def _got((res, status, headers)):
657 self.failUnlessReallyEqual(int(status), 206)
658 self.failUnless(headers.has_key("content-range"))
659 self.failUnlessReallyEqual(headers["content-range"][0],
660 "bytes 5-%d/%d" % (length-1, length))
661 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
665 def test_GET_FILEURL_partial_end_range(self):
666 headers = {"range": "bytes=-5"}
667 length = len(self.BAR_CONTENTS)
668 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
669 return_response=True)
670 def _got((res, status, headers)):
671 self.failUnlessReallyEqual(int(status), 206)
672 self.failUnless(headers.has_key("content-range"))
673 self.failUnlessReallyEqual(headers["content-range"][0],
674 "bytes %d-%d/%d" % (length-5, length-1, length))
675 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
679 def test_GET_FILEURL_partial_range_overrun(self):
680 headers = {"range": "bytes=100-200"}
681 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
682 "416 Requested Range not satisfiable",
683 "First beyond end of file",
684 self.GET, self.public_url + "/foo/bar.txt",
688 def test_HEAD_FILEURL_range(self):
689 headers = {"range": "bytes=1-10"}
690 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
691 return_response=True)
692 def _got((res, status, headers)):
693 self.failUnlessReallyEqual(res, "")
694 self.failUnlessReallyEqual(int(status), 206)
695 self.failUnless(headers.has_key("content-range"))
696 self.failUnlessReallyEqual(headers["content-range"][0],
697 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
701 def test_HEAD_FILEURL_partial_range(self):
702 headers = {"range": "bytes=5-"}
703 length = len(self.BAR_CONTENTS)
704 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
705 return_response=True)
706 def _got((res, status, headers)):
707 self.failUnlessReallyEqual(int(status), 206)
708 self.failUnless(headers.has_key("content-range"))
709 self.failUnlessReallyEqual(headers["content-range"][0],
710 "bytes 5-%d/%d" % (length-1, length))
714 def test_HEAD_FILEURL_partial_end_range(self):
715 headers = {"range": "bytes=-5"}
716 length = len(self.BAR_CONTENTS)
717 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
718 return_response=True)
719 def _got((res, status, headers)):
720 self.failUnlessReallyEqual(int(status), 206)
721 self.failUnless(headers.has_key("content-range"))
722 self.failUnlessReallyEqual(headers["content-range"][0],
723 "bytes %d-%d/%d" % (length-5, length-1, length))
727 def test_HEAD_FILEURL_partial_range_overrun(self):
728 headers = {"range": "bytes=100-200"}
729 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
730 "416 Requested Range not satisfiable",
732 self.HEAD, self.public_url + "/foo/bar.txt",
736 def test_GET_FILEURL_range_bad(self):
737 headers = {"range": "BOGUS=fizbop-quarnak"}
738 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
739 return_response=True)
740 def _got((res, status, headers)):
741 self.failUnlessReallyEqual(int(status), 200)
742 self.failUnless(not headers.has_key("content-range"))
743 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
747 def test_HEAD_FILEURL(self):
748 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
749 def _got((res, status, headers)):
750 self.failUnlessReallyEqual(res, "")
751 self.failUnlessReallyEqual(headers["content-length"][0],
752 str(len(self.BAR_CONTENTS)))
753 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
757 def test_GET_FILEURL_named(self):
758 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
759 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
760 d = self.GET(base + "/@@name=/blah.txt")
761 d.addCallback(self.failUnlessIsBarDotTxt)
762 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
763 d.addCallback(self.failUnlessIsBarDotTxt)
764 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
765 d.addCallback(self.failUnlessIsBarDotTxt)
766 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
767 d.addCallback(self.failUnlessIsBarDotTxt)
768 save_url = base + "?save=true&filename=blah.txt"
769 d.addCallback(lambda res: self.GET(save_url))
770 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
771 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
772 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
773 u_url = base + "?save=true&filename=" + u_fn_e
774 d.addCallback(lambda res: self.GET(u_url))
775 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
778 def test_PUT_FILEURL_named_bad(self):
779 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
780 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
782 "/file can only be used with GET or HEAD",
783 self.PUT, base + "/@@name=/blah.txt", "")
786 def test_GET_DIRURL_named_bad(self):
787 base = "/file/%s" % urllib.quote(self._foo_uri)
788 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
791 self.GET, base + "/@@name=/blah.txt")
794 def test_GET_slash_file_bad(self):
795 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
797 "/file must be followed by a file-cap and a name",
801 def test_GET_unhandled_URI_named(self):
802 contents, n, newuri = self.makefile(12)
803 verifier_cap = n.get_verify_cap().to_string()
804 base = "/file/%s" % urllib.quote(verifier_cap)
805 # client.create_node_from_uri() can't handle verify-caps
806 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
807 "400 Bad Request", "is not a file-cap",
811 def test_GET_unhandled_URI(self):
812 contents, n, newuri = self.makefile(12)
813 verifier_cap = n.get_verify_cap().to_string()
814 base = "/uri/%s" % urllib.quote(verifier_cap)
815 # client.create_node_from_uri() can't handle verify-caps
816 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
818 "GET unknown URI type: can only do t=info",
822 def test_GET_FILE_URI(self):
823 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
825 d.addCallback(self.failUnlessIsBarDotTxt)
828 def test_GET_FILE_URI_badchild(self):
829 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
830 errmsg = "Files have no children, certainly not named 'boguschild'"
831 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
832 "400 Bad Request", errmsg,
836 def test_PUT_FILE_URI_badchild(self):
837 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
838 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
839 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
840 "400 Bad Request", errmsg,
844 # TODO: version of this with a Unicode filename
845 def test_GET_FILEURL_save(self):
846 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
847 return_response=True)
848 def _got((res, statuscode, headers)):
849 content_disposition = headers["content-disposition"][0]
850 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
851 self.failUnlessIsBarDotTxt(res)
855 def test_GET_FILEURL_missing(self):
856 d = self.GET(self.public_url + "/foo/missing")
857 d.addBoth(self.should404, "test_GET_FILEURL_missing")
860 def test_PUT_overwrite_only_files(self):
861 # create a directory, put a file in that directory.
862 contents, n, filecap = self.makefile(8)
863 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
864 d.addCallback(lambda res:
865 self.PUT(self.public_url + "/foo/dir/file1.txt",
866 self.NEWFILE_CONTENTS))
867 # try to overwrite the file with replace=only-files
869 d.addCallback(lambda res:
870 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
872 d.addCallback(lambda res:
873 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
874 "There was already a child by that name, and you asked me "
876 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
880 def test_PUT_NEWFILEURL(self):
881 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
882 # TODO: we lose the response code, so we can't check this
883 #self.failUnlessReallyEqual(responsecode, 201)
884 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
885 d.addCallback(lambda res:
886 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
887 self.NEWFILE_CONTENTS))
890 def test_PUT_NEWFILEURL_not_mutable(self):
891 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
892 self.NEWFILE_CONTENTS)
893 # TODO: we lose the response code, so we can't check this
894 #self.failUnlessReallyEqual(responsecode, 201)
895 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
896 d.addCallback(lambda res:
897 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
898 self.NEWFILE_CONTENTS))
901 def test_PUT_NEWFILEURL_range_bad(self):
902 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
903 target = self.public_url + "/foo/new.txt"
904 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
905 "501 Not Implemented",
906 "Content-Range in PUT not yet supported",
907 # (and certainly not for immutable files)
908 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
910 d.addCallback(lambda res:
911 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
914 def test_PUT_NEWFILEURL_mutable(self):
915 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
916 self.NEWFILE_CONTENTS)
917 # TODO: we lose the response code, so we can't check this
918 #self.failUnlessReallyEqual(responsecode, 201)
920 u = uri.from_string_mutable_filenode(res)
921 self.failUnless(u.is_mutable())
922 self.failIf(u.is_readonly())
924 d.addCallback(_check_uri)
925 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
926 d.addCallback(lambda res:
927 self.failUnlessMutableChildContentsAre(self._foo_node,
929 self.NEWFILE_CONTENTS))
932 def test_PUT_NEWFILEURL_mutable_toobig(self):
933 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
934 "413 Request Entity Too Large",
935 "SDMF is limited to one segment, and 10001 > 10000",
937 self.public_url + "/foo/new.txt?mutable=true",
938 "b" * (self.s.MUTABLE_SIZELIMIT+1))
941 def test_PUT_NEWFILEURL_replace(self):
942 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
943 # TODO: we lose the response code, so we can't check this
944 #self.failUnlessReallyEqual(responsecode, 200)
945 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
946 d.addCallback(lambda res:
947 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
948 self.NEWFILE_CONTENTS))
951 def test_PUT_NEWFILEURL_bad_t(self):
952 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
953 "PUT to a file: bad t=bogus",
954 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
958 def test_PUT_NEWFILEURL_no_replace(self):
959 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
960 self.NEWFILE_CONTENTS)
961 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
963 "There was already a child by that name, and you asked me "
967 def test_PUT_NEWFILEURL_mkdirs(self):
968 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
970 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
971 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
972 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
973 d.addCallback(lambda res:
974 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
975 self.NEWFILE_CONTENTS))
978 def test_PUT_NEWFILEURL_blocked(self):
979 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
980 self.NEWFILE_CONTENTS)
981 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
983 "Unable to create directory 'blockingfile': a file was in the way")
986 def test_PUT_NEWFILEURL_emptyname(self):
987 # an empty pathname component (i.e. a double-slash) is disallowed
988 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
990 "The webapi does not allow empty pathname components",
991 self.PUT, self.public_url + "/foo//new.txt", "")
994 def test_DELETE_FILEURL(self):
995 d = self.DELETE(self.public_url + "/foo/bar.txt")
996 d.addCallback(lambda res:
997 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1000 def test_DELETE_FILEURL_missing(self):
1001 d = self.DELETE(self.public_url + "/foo/missing")
1002 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1005 def test_DELETE_FILEURL_missing2(self):
1006 d = self.DELETE(self.public_url + "/missing/missing")
1007 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1010 def failUnlessHasBarDotTxtMetadata(self, res):
1011 data = simplejson.loads(res)
1012 self.failUnless(isinstance(data, list))
1013 self.failUnlessIn("metadata", data[1])
1014 self.failUnlessIn("tahoe", data[1]["metadata"])
1015 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1016 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1017 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1018 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1020 def test_GET_FILEURL_json(self):
1021 # twisted.web.http.parse_qs ignores any query args without an '=', so
1022 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1023 # instead. This may make it tricky to emulate the S3 interface
1025 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1027 self.failUnlessIsBarJSON(data)
1028 self.failUnlessHasBarDotTxtMetadata(data)
1030 d.addCallback(_check1)
1033 def test_GET_FILEURL_json_missing(self):
1034 d = self.GET(self.public_url + "/foo/missing?json")
1035 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1038 def test_GET_FILEURL_uri(self):
1039 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1041 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1042 d.addCallback(_check)
1043 d.addCallback(lambda res:
1044 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1046 # for now, for files, uris and readonly-uris are the same
1047 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1048 d.addCallback(_check2)
1051 def test_GET_FILEURL_badtype(self):
1052 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1055 self.public_url + "/foo/bar.txt?t=bogus")
1058 def test_CSS_FILE(self):
1059 d = self.GET("/tahoe_css", followRedirect=True)
1061 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1062 self.failUnless(CSS_STYLE.search(res), res)
1063 d.addCallback(_check)
1066 def test_GET_FILEURL_uri_missing(self):
1067 d = self.GET(self.public_url + "/foo/missing?t=uri")
1068 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1071 def test_GET_DIRECTORY_html_banner(self):
1072 d = self.GET(self.public_url + "/foo", followRedirect=True)
1074 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1075 d.addCallback(_check)
1078 def test_GET_DIRURL(self):
1079 # the addSlash means we get a redirect here
1080 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1082 d = self.GET(self.public_url + "/foo", followRedirect=True)
1084 self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1086 # the FILE reference points to a URI, but it should end in bar.txt
1087 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1088 (ROOT, urllib.quote(self._bar_txt_uri)))
1089 get_bar = "".join([r'<td>FILE</td>',
1091 r'<a href="%s">bar.txt</a>' % bar_url,
1093 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1095 self.failUnless(re.search(get_bar, res), res)
1096 for label in ['unlink', 'rename']:
1097 for line in res.split("\n"):
1098 # find the line that contains the relevant button for bar.txt
1099 if ("form action" in line and
1100 ('value="%s"' % (label,)) in line and
1101 'value="bar.txt"' in line):
1102 # the form target should use a relative URL
1103 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1104 self.failUnlessIn('action="%s"' % foo_url, line)
1105 # and the when_done= should too
1106 #done_url = urllib.quote(???)
1107 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1109 # 'unlink' needs to use POST because it directly has a side effect
1110 if label == 'unlink':
1111 self.failUnlessIn('method="post"', line)
1114 self.fail("unable to find '%s bar.txt' line" % (label,), res)
1116 # the DIR reference just points to a URI
1117 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1118 get_sub = ((r'<td>DIR</td>')
1119 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1120 self.failUnless(re.search(get_sub, res), res)
1121 d.addCallback(_check)
1123 # look at a readonly directory
1124 d.addCallback(lambda res:
1125 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1127 self.failUnless("(read-only)" in res, res)
1128 self.failIf("Upload a file" in res, res)
1129 d.addCallback(_check2)
1131 # and at a directory that contains a readonly directory
1132 d.addCallback(lambda res:
1133 self.GET(self.public_url, followRedirect=True))
1135 self.failUnless(re.search('<td>DIR-RO</td>'
1136 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1137 d.addCallback(_check3)
1139 # and an empty directory
1140 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1142 self.failUnless("directory is empty" in res, res)
1143 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1144 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1145 d.addCallback(_check4)
1147 # and at a literal directory
1148 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1149 d.addCallback(lambda res:
1150 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1152 self.failUnless('(immutable)' in res, res)
1153 self.failUnless(re.search('<td>FILE</td>'
1154 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1155 d.addCallback(_check5)
1158 def test_GET_DIRURL_badtype(self):
1159 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1163 self.public_url + "/foo?t=bogus")
1166 def test_GET_DIRURL_json(self):
1167 d = self.GET(self.public_url + "/foo?t=json")
1168 d.addCallback(self.failUnlessIsFooJSON)
1172 def test_POST_DIRURL_manifest_no_ophandle(self):
1173 d = self.shouldFail2(error.Error,
1174 "test_POST_DIRURL_manifest_no_ophandle",
1176 "slow operation requires ophandle=",
1177 self.POST, self.public_url, t="start-manifest")
1180 def test_POST_DIRURL_manifest(self):
1181 d = defer.succeed(None)
1182 def getman(ignored, output):
1183 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1184 followRedirect=True)
1185 d.addCallback(self.wait_for_operation, "125")
1186 d.addCallback(self.get_operation_results, "125", output)
1188 d.addCallback(getman, None)
1189 def _got_html(manifest):
1190 self.failUnless("Manifest of SI=" in manifest)
1191 self.failUnless("<td>sub</td>" in manifest)
1192 self.failUnless(self._sub_uri in manifest)
1193 self.failUnless("<td>sub/baz.txt</td>" in manifest)
1194 d.addCallback(_got_html)
1196 # both t=status and unadorned GET should be identical
1197 d.addCallback(lambda res: self.GET("/operations/125"))
1198 d.addCallback(_got_html)
1200 d.addCallback(getman, "html")
1201 d.addCallback(_got_html)
1202 d.addCallback(getman, "text")
1203 def _got_text(manifest):
1204 self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1205 self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1206 d.addCallback(_got_text)
1207 d.addCallback(getman, "JSON")
1209 data = res["manifest"]
1211 for (path_list, cap) in data:
1212 got[tuple(path_list)] = cap
1213 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1214 self.failUnless((u"sub",u"baz.txt") in got)
1215 self.failUnless("finished" in res)
1216 self.failUnless("origin" in res)
1217 self.failUnless("storage-index" in res)
1218 self.failUnless("verifycaps" in res)
1219 self.failUnless("stats" in res)
1220 d.addCallback(_got_json)
1223 def test_POST_DIRURL_deepsize_no_ophandle(self):
1224 d = self.shouldFail2(error.Error,
1225 "test_POST_DIRURL_deepsize_no_ophandle",
1227 "slow operation requires ophandle=",
1228 self.POST, self.public_url, t="start-deep-size")
1231 def test_POST_DIRURL_deepsize(self):
1232 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1233 followRedirect=True)
1234 d.addCallback(self.wait_for_operation, "126")
1235 d.addCallback(self.get_operation_results, "126", "json")
1236 def _got_json(data):
1237 self.failUnlessReallyEqual(data["finished"], True)
1239 self.failUnless(size > 1000)
1240 d.addCallback(_got_json)
1241 d.addCallback(self.get_operation_results, "126", "text")
1243 mo = re.search(r'^size: (\d+)$', res, re.M)
1244 self.failUnless(mo, res)
1245 size = int(mo.group(1))
1246 # with directories, the size varies.
1247 self.failUnless(size > 1000)
1248 d.addCallback(_got_text)
1251 def test_POST_DIRURL_deepstats_no_ophandle(self):
1252 d = self.shouldFail2(error.Error,
1253 "test_POST_DIRURL_deepstats_no_ophandle",
1255 "slow operation requires ophandle=",
1256 self.POST, self.public_url, t="start-deep-stats")
1259 def test_POST_DIRURL_deepstats(self):
1260 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1261 followRedirect=True)
1262 d.addCallback(self.wait_for_operation, "127")
1263 d.addCallback(self.get_operation_results, "127", "json")
1264 def _got_json(stats):
1265 expected = {"count-immutable-files": 3,
1266 "count-mutable-files": 0,
1267 "count-literal-files": 0,
1269 "count-directories": 3,
1270 "size-immutable-files": 57,
1271 "size-literal-files": 0,
1272 #"size-directories": 1912, # varies
1273 #"largest-directory": 1590,
1274 "largest-directory-children": 5,
1275 "largest-immutable-file": 19,
1277 for k,v in expected.iteritems():
1278 self.failUnlessReallyEqual(stats[k], v,
1279 "stats[%s] was %s, not %s" %
1281 self.failUnlessReallyEqual(stats["size-files-histogram"],
1283 d.addCallback(_got_json)
1286 def test_POST_DIRURL_stream_manifest(self):
1287 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1289 self.failUnless(res.endswith("\n"))
1290 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1291 self.failUnlessReallyEqual(len(units), 7)
1292 self.failUnlessEqual(units[-1]["type"], "stats")
1294 self.failUnlessEqual(first["path"], [])
1295 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1296 self.failUnlessEqual(first["type"], "directory")
1297 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1298 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1299 self.failIfEqual(baz["storage-index"], None)
1300 self.failIfEqual(baz["verifycap"], None)
1301 self.failIfEqual(baz["repaircap"], None)
1303 d.addCallback(_check)
1306 def test_GET_DIRURL_uri(self):
1307 d = self.GET(self.public_url + "/foo?t=uri")
1309 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1310 d.addCallback(_check)
1313 def test_GET_DIRURL_readonly_uri(self):
1314 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1316 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1317 d.addCallback(_check)
1320 def test_PUT_NEWDIRURL(self):
1321 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1322 d.addCallback(lambda res:
1323 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1324 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1325 d.addCallback(self.failUnlessNodeKeysAre, [])
1328 def test_POST_NEWDIRURL(self):
1329 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1330 d.addCallback(lambda res:
1331 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1332 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1333 d.addCallback(self.failUnlessNodeKeysAre, [])
1336 def test_POST_NEWDIRURL_emptyname(self):
1337 # an empty pathname component (i.e. a double-slash) is disallowed
1338 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1340 "The webapi does not allow empty pathname components, i.e. a double slash",
1341 self.POST, self.public_url + "//?t=mkdir")
1344 def test_POST_NEWDIRURL_initial_children(self):
1345 (newkids, caps) = self._create_initial_children()
1346 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1347 simplejson.dumps(newkids))
1349 n = self.s.create_node_from_uri(uri.strip())
1350 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1351 d2.addCallback(lambda ign:
1352 self.failUnlessROChildURIIs(n, u"child-imm",
1354 d2.addCallback(lambda ign:
1355 self.failUnlessRWChildURIIs(n, u"child-mutable",
1357 d2.addCallback(lambda ign:
1358 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1360 d2.addCallback(lambda ign:
1361 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1362 caps['unknown_rocap']))
1363 d2.addCallback(lambda ign:
1364 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1365 caps['unknown_rwcap']))
1366 d2.addCallback(lambda ign:
1367 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1368 caps['unknown_immcap']))
1369 d2.addCallback(lambda ign:
1370 self.failUnlessRWChildURIIs(n, u"dirchild",
1372 d2.addCallback(lambda ign:
1373 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1375 d2.addCallback(lambda ign:
1376 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1377 caps['emptydircap']))
1379 d.addCallback(_check)
1380 d.addCallback(lambda res:
1381 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1382 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1383 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1384 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1385 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1388 def test_POST_NEWDIRURL_immutable(self):
1389 (newkids, caps) = self._create_immutable_children()
1390 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1391 simplejson.dumps(newkids))
1393 n = self.s.create_node_from_uri(uri.strip())
1394 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1395 d2.addCallback(lambda ign:
1396 self.failUnlessROChildURIIs(n, u"child-imm",
1398 d2.addCallback(lambda ign:
1399 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1400 caps['unknown_immcap']))
1401 d2.addCallback(lambda ign:
1402 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1404 d2.addCallback(lambda ign:
1405 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1407 d2.addCallback(lambda ign:
1408 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1409 caps['emptydircap']))
1411 d.addCallback(_check)
1412 d.addCallback(lambda res:
1413 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1414 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1415 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1416 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1417 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1418 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1419 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1420 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1421 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1422 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1423 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1424 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1425 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1426 d.addErrback(self.explain_web_error)
1429 def test_POST_NEWDIRURL_immutable_bad(self):
1430 (newkids, caps) = self._create_initial_children()
1431 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1433 "needed to be immutable but was not",
1435 self.public_url + "/foo/newdir?t=mkdir-immutable",
1436 simplejson.dumps(newkids))
1439 def test_PUT_NEWDIRURL_exists(self):
1440 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1441 d.addCallback(lambda res:
1442 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1443 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1444 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1447 def test_PUT_NEWDIRURL_blocked(self):
1448 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1449 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1451 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1452 d.addCallback(lambda res:
1453 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1454 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1455 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1458 def test_PUT_NEWDIRURL_mkdir_p(self):
1459 d = defer.succeed(None)
1460 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1461 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1462 d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1463 def mkdir_p(mkpnode):
1464 url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1466 def made_subsub(ssuri):
1467 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1468 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1470 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1472 d.addCallback(made_subsub)
1474 d.addCallback(mkdir_p)
1477 def test_PUT_NEWDIRURL_mkdirs(self):
1478 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1479 d.addCallback(lambda res:
1480 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1481 d.addCallback(lambda res:
1482 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1483 d.addCallback(lambda res:
1484 self._foo_node.get_child_at_path(u"subdir/newdir"))
1485 d.addCallback(self.failUnlessNodeKeysAre, [])
1488 def test_DELETE_DIRURL(self):
1489 d = self.DELETE(self.public_url + "/foo")
1490 d.addCallback(lambda res:
1491 self.failIfNodeHasChild(self.public_root, u"foo"))
1494 def test_DELETE_DIRURL_missing(self):
1495 d = self.DELETE(self.public_url + "/foo/missing")
1496 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1497 d.addCallback(lambda res:
1498 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1501 def test_DELETE_DIRURL_missing2(self):
1502 d = self.DELETE(self.public_url + "/missing")
1503 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1506 def dump_root(self):
1508 w = webish.DirnodeWalkerMixin()
1509 def visitor(childpath, childnode, metadata):
1511 d = w.walk(self.public_root, visitor)
1514 def failUnlessNodeKeysAre(self, node, expected_keys):
1515 for k in expected_keys:
1516 assert isinstance(k, unicode)
1518 def _check(children):
1519 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1520 d.addCallback(_check)
1522 def failUnlessNodeHasChild(self, node, name):
1523 assert isinstance(name, unicode)
1525 def _check(children):
1526 self.failUnless(name in children)
1527 d.addCallback(_check)
1529 def failIfNodeHasChild(self, node, name):
1530 assert isinstance(name, unicode)
1532 def _check(children):
1533 self.failIf(name in children)
1534 d.addCallback(_check)
1537 def failUnlessChildContentsAre(self, node, name, expected_contents):
1538 assert isinstance(name, unicode)
1539 d = node.get_child_at_path(name)
1540 d.addCallback(lambda node: download_to_data(node))
1541 def _check(contents):
1542 self.failUnlessReallyEqual(contents, expected_contents)
1543 d.addCallback(_check)
1546 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1547 assert isinstance(name, unicode)
1548 d = node.get_child_at_path(name)
1549 d.addCallback(lambda node: node.download_best_version())
1550 def _check(contents):
1551 self.failUnlessReallyEqual(contents, expected_contents)
1552 d.addCallback(_check)
1555 def failUnlessRWChildURIIs(self, node, name, expected_uri):
1556 assert isinstance(name, unicode)
1557 d = node.get_child_at_path(name)
1559 self.failUnless(child.is_unknown() or not child.is_readonly())
1560 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1561 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1562 expected_ro_uri = self._make_readonly(expected_uri)
1564 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1565 d.addCallback(_check)
1568 def failUnlessROChildURIIs(self, node, name, expected_uri):
1569 assert isinstance(name, unicode)
1570 d = node.get_child_at_path(name)
1572 self.failUnless(child.is_unknown() or child.is_readonly())
1573 self.failUnlessReallyEqual(child.get_write_uri(), None)
1574 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1575 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1576 d.addCallback(_check)
1579 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1580 assert isinstance(name, unicode)
1581 d = node.get_child_at_path(name)
1583 self.failUnless(child.is_unknown() or not child.is_readonly())
1584 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1585 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1586 expected_ro_uri = self._make_readonly(got_uri)
1588 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1589 d.addCallback(_check)
1592 def failUnlessURIMatchesROChild(self, got_uri, node, name):
1593 assert isinstance(name, unicode)
1594 d = node.get_child_at_path(name)
1596 self.failUnless(child.is_unknown() or child.is_readonly())
1597 self.failUnlessReallyEqual(child.get_write_uri(), None)
1598 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1599 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1600 d.addCallback(_check)
1603 def failUnlessCHKURIHasContents(self, got_uri, contents):
1604 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1606 def test_POST_upload(self):
1607 d = self.POST(self.public_url + "/foo", t="upload",
1608 file=("new.txt", self.NEWFILE_CONTENTS))
1610 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1611 d.addCallback(lambda res:
1612 self.failUnlessChildContentsAre(fn, u"new.txt",
1613 self.NEWFILE_CONTENTS))
1616 def test_POST_upload_unicode(self):
1617 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1618 d = self.POST(self.public_url + "/foo", t="upload",
1619 file=(filename, self.NEWFILE_CONTENTS))
1621 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1622 d.addCallback(lambda res:
1623 self.failUnlessChildContentsAre(fn, filename,
1624 self.NEWFILE_CONTENTS))
1625 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1626 d.addCallback(lambda res: self.GET(target_url))
1627 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1628 self.NEWFILE_CONTENTS,
1632 def test_POST_upload_unicode_named(self):
1633 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1634 d = self.POST(self.public_url + "/foo", t="upload",
1636 file=("overridden", self.NEWFILE_CONTENTS))
1638 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1639 d.addCallback(lambda res:
1640 self.failUnlessChildContentsAre(fn, filename,
1641 self.NEWFILE_CONTENTS))
1642 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1643 d.addCallback(lambda res: self.GET(target_url))
1644 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1645 self.NEWFILE_CONTENTS,
1649 def test_POST_upload_no_link(self):
1650 d = self.POST("/uri", t="upload",
1651 file=("new.txt", self.NEWFILE_CONTENTS))
1652 def _check_upload_results(page):
1653 # this should be a page which describes the results of the upload
1654 # that just finished.
1655 self.failUnless("Upload Results:" in page)
1656 self.failUnless("URI:" in page)
1657 uri_re = re.compile("URI: <tt><span>(.*)</span>")
1658 mo = uri_re.search(page)
1659 self.failUnless(mo, page)
1660 new_uri = mo.group(1)
1662 d.addCallback(_check_upload_results)
1663 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1666 def test_POST_upload_no_link_whendone(self):
1667 d = self.POST("/uri", t="upload", when_done="/",
1668 file=("new.txt", self.NEWFILE_CONTENTS))
1669 d.addBoth(self.shouldRedirect, "/")
1672 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1673 d = defer.maybeDeferred(callable, *args, **kwargs)
1675 if isinstance(res, failure.Failure):
1676 res.trap(error.PageRedirect)
1677 statuscode = res.value.status
1678 target = res.value.location
1679 return checker(statuscode, target)
1680 self.fail("%s: callable was supposed to redirect, not return '%s'"
1685 def test_POST_upload_no_link_whendone_results(self):
1686 def check(statuscode, target):
1687 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1688 self.failUnless(target.startswith(self.webish_url), target)
1689 return client.getPage(target, method="GET")
1690 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1692 self.POST, "/uri", t="upload",
1693 when_done="/uri/%(uri)s",
1694 file=("new.txt", self.NEWFILE_CONTENTS))
1695 d.addCallback(lambda res:
1696 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
1699 def test_POST_upload_no_link_mutable(self):
1700 d = self.POST("/uri", t="upload", mutable="true",
1701 file=("new.txt", self.NEWFILE_CONTENTS))
1702 def _check(filecap):
1703 filecap = filecap.strip()
1704 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1705 self.filecap = filecap
1706 u = uri.WriteableSSKFileURI.init_from_string(filecap)
1707 self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1708 n = self.s.create_node_from_uri(filecap)
1709 return n.download_best_version()
1710 d.addCallback(_check)
1712 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1713 return self.GET("/uri/%s" % urllib.quote(self.filecap))
1714 d.addCallback(_check2)
1716 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1717 return self.GET("/file/%s" % urllib.quote(self.filecap))
1718 d.addCallback(_check3)
1720 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1721 d.addCallback(_check4)
1724 def test_POST_upload_no_link_mutable_toobig(self):
1725 d = self.shouldFail2(error.Error,
1726 "test_POST_upload_no_link_mutable_toobig",
1727 "413 Request Entity Too Large",
1728 "SDMF is limited to one segment, and 10001 > 10000",
1730 "/uri", t="upload", mutable="true",
1732 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1735 def test_POST_upload_mutable(self):
1736 # this creates a mutable file
1737 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1738 file=("new.txt", self.NEWFILE_CONTENTS))
1740 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1741 d.addCallback(lambda res:
1742 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1743 self.NEWFILE_CONTENTS))
1744 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1746 self.failUnless(IMutableFileNode.providedBy(newnode))
1747 self.failUnless(newnode.is_mutable())
1748 self.failIf(newnode.is_readonly())
1749 self._mutable_node = newnode
1750 self._mutable_uri = newnode.get_uri()
1753 # now upload it again and make sure that the URI doesn't change
1754 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1755 d.addCallback(lambda res:
1756 self.POST(self.public_url + "/foo", t="upload",
1758 file=("new.txt", NEWER_CONTENTS)))
1759 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1760 d.addCallback(lambda res:
1761 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1763 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1765 self.failUnless(IMutableFileNode.providedBy(newnode))
1766 self.failUnless(newnode.is_mutable())
1767 self.failIf(newnode.is_readonly())
1768 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1769 d.addCallback(_got2)
1771 # upload a second time, using PUT instead of POST
1772 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1773 d.addCallback(lambda res:
1774 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1775 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1776 d.addCallback(lambda res:
1777 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1780 # finally list the directory, since mutable files are displayed
1781 # slightly differently
1783 d.addCallback(lambda res:
1784 self.GET(self.public_url + "/foo/",
1785 followRedirect=True))
1786 def _check_page(res):
1787 # TODO: assert more about the contents
1788 self.failUnless("SSK" in res)
1790 d.addCallback(_check_page)
1792 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1794 self.failUnless(IMutableFileNode.providedBy(newnode))
1795 self.failUnless(newnode.is_mutable())
1796 self.failIf(newnode.is_readonly())
1797 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1798 d.addCallback(_got3)
1800 # look at the JSON form of the enclosing directory
1801 d.addCallback(lambda res:
1802 self.GET(self.public_url + "/foo/?t=json",
1803 followRedirect=True))
1804 def _check_page_json(res):
1805 parsed = simplejson.loads(res)
1806 self.failUnlessEqual(parsed[0], "dirnode")
1807 children = dict( [(unicode(name),value)
1809 in parsed[1]["children"].iteritems()] )
1810 self.failUnless(u"new.txt" in children)
1811 new_json = children[u"new.txt"]
1812 self.failUnlessEqual(new_json[0], "filenode")
1813 self.failUnless(new_json[1]["mutable"])
1814 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
1815 ro_uri = self._mutable_node.get_readonly().to_string()
1816 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
1817 d.addCallback(_check_page_json)
1819 # and the JSON form of the file
1820 d.addCallback(lambda res:
1821 self.GET(self.public_url + "/foo/new.txt?t=json"))
1822 def _check_file_json(res):
1823 parsed = simplejson.loads(res)
1824 self.failUnlessEqual(parsed[0], "filenode")
1825 self.failUnless(parsed[1]["mutable"])
1826 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
1827 ro_uri = self._mutable_node.get_readonly().to_string()
1828 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
1829 d.addCallback(_check_file_json)
1831 # and look at t=uri and t=readonly-uri
1832 d.addCallback(lambda res:
1833 self.GET(self.public_url + "/foo/new.txt?t=uri"))
1834 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
1835 d.addCallback(lambda res:
1836 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1837 def _check_ro_uri(res):
1838 ro_uri = self._mutable_node.get_readonly().to_string()
1839 self.failUnlessReallyEqual(res, ro_uri)
1840 d.addCallback(_check_ro_uri)
1842 # make sure we can get to it from /uri/URI
1843 d.addCallback(lambda res:
1844 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1845 d.addCallback(lambda res:
1846 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
1848 # and that HEAD computes the size correctly
1849 d.addCallback(lambda res:
1850 self.HEAD(self.public_url + "/foo/new.txt",
1851 return_response=True))
1852 def _got_headers((res, status, headers)):
1853 self.failUnlessReallyEqual(res, "")
1854 self.failUnlessReallyEqual(headers["content-length"][0],
1855 str(len(NEW2_CONTENTS)))
1856 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
1857 d.addCallback(_got_headers)
1859 # make sure that size errors are displayed correctly for overwrite
1860 d.addCallback(lambda res:
1861 self.shouldFail2(error.Error,
1862 "test_POST_upload_mutable-toobig",
1863 "413 Request Entity Too Large",
1864 "SDMF is limited to one segment, and 10001 > 10000",
1866 self.public_url + "/foo", t="upload",
1869 "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1872 d.addErrback(self.dump_error)
1875 def test_POST_upload_mutable_toobig(self):
1876 d = self.shouldFail2(error.Error,
1877 "test_POST_upload_mutable_toobig",
1878 "413 Request Entity Too Large",
1879 "SDMF is limited to one segment, and 10001 > 10000",
1881 self.public_url + "/foo",
1882 t="upload", mutable="true",
1884 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1887 def dump_error(self, f):
1888 # if the web server returns an error code (like 400 Bad Request),
1889 # web.client.getPage puts the HTTP response body into the .response
1890 # attribute of the exception object that it gives back. It does not
1891 # appear in the Failure's repr(), so the ERROR that trial displays
1892 # will be rather terse and unhelpful. addErrback this method to the
1893 # end of your chain to get more information out of these errors.
1894 if f.check(error.Error):
1895 print "web.error.Error:"
1897 print f.value.response
1900 def test_POST_upload_replace(self):
1901 d = self.POST(self.public_url + "/foo", t="upload",
1902 file=("bar.txt", self.NEWFILE_CONTENTS))
1904 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1905 d.addCallback(lambda res:
1906 self.failUnlessChildContentsAre(fn, u"bar.txt",
1907 self.NEWFILE_CONTENTS))
1910 def test_POST_upload_no_replace_ok(self):
1911 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1912 file=("new.txt", self.NEWFILE_CONTENTS))
1913 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1914 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
1915 self.NEWFILE_CONTENTS))
1918 def test_POST_upload_no_replace_queryarg(self):
1919 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1920 file=("bar.txt", self.NEWFILE_CONTENTS))
1921 d.addBoth(self.shouldFail, error.Error,
1922 "POST_upload_no_replace_queryarg",
1924 "There was already a child by that name, and you asked me "
1925 "to not replace it")
1926 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1927 d.addCallback(self.failUnlessIsBarDotTxt)
1930 def test_POST_upload_no_replace_field(self):
1931 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1932 file=("bar.txt", self.NEWFILE_CONTENTS))
1933 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1935 "There was already a child by that name, and you asked me "
1936 "to not replace it")
1937 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1938 d.addCallback(self.failUnlessIsBarDotTxt)
1941 def test_POST_upload_whendone(self):
1942 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1943 file=("new.txt", self.NEWFILE_CONTENTS))
1944 d.addBoth(self.shouldRedirect, "/THERE")
1946 d.addCallback(lambda res:
1947 self.failUnlessChildContentsAre(fn, u"new.txt",
1948 self.NEWFILE_CONTENTS))
1951 def test_POST_upload_named(self):
1953 d = self.POST(self.public_url + "/foo", t="upload",
1954 name="new.txt", file=self.NEWFILE_CONTENTS)
1955 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1956 d.addCallback(lambda res:
1957 self.failUnlessChildContentsAre(fn, u"new.txt",
1958 self.NEWFILE_CONTENTS))
1961 def test_POST_upload_named_badfilename(self):
1962 d = self.POST(self.public_url + "/foo", t="upload",
1963 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1964 d.addBoth(self.shouldFail, error.Error,
1965 "test_POST_upload_named_badfilename",
1967 "name= may not contain a slash",
1969 # make sure that nothing was added
1970 d.addCallback(lambda res:
1971 self.failUnlessNodeKeysAre(self._foo_node,
1972 [u"bar.txt", u"blockingfile",
1973 u"empty", u"n\u00fc.txt",
1977 def test_POST_FILEURL_check(self):
1978 bar_url = self.public_url + "/foo/bar.txt"
1979 d = self.POST(bar_url, t="check")
1981 self.failUnless("Healthy :" in res)
1982 d.addCallback(_check)
1983 redir_url = "http://allmydata.org/TARGET"
1984 def _check2(statuscode, target):
1985 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1986 self.failUnlessReallyEqual(target, redir_url)
1987 d.addCallback(lambda res:
1988 self.shouldRedirect2("test_POST_FILEURL_check",
1992 when_done=redir_url))
1993 d.addCallback(lambda res:
1994 self.POST(bar_url, t="check", return_to=redir_url))
1996 self.failUnless("Healthy :" in res)
1997 self.failUnless("Return to file" in res)
1998 self.failUnless(redir_url in res)
1999 d.addCallback(_check3)
2001 d.addCallback(lambda res:
2002 self.POST(bar_url, t="check", output="JSON"))
2003 def _check_json(res):
2004 data = simplejson.loads(res)
2005 self.failUnless("storage-index" in data)
2006 self.failUnless(data["results"]["healthy"])
2007 d.addCallback(_check_json)
2011 def test_POST_FILEURL_check_and_repair(self):
2012 bar_url = self.public_url + "/foo/bar.txt"
2013 d = self.POST(bar_url, t="check", repair="true")
2015 self.failUnless("Healthy :" in res)
2016 d.addCallback(_check)
2017 redir_url = "http://allmydata.org/TARGET"
2018 def _check2(statuscode, target):
2019 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2020 self.failUnlessReallyEqual(target, redir_url)
2021 d.addCallback(lambda res:
2022 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2025 t="check", repair="true",
2026 when_done=redir_url))
2027 d.addCallback(lambda res:
2028 self.POST(bar_url, t="check", return_to=redir_url))
2030 self.failUnless("Healthy :" in res)
2031 self.failUnless("Return to file" in res)
2032 self.failUnless(redir_url in res)
2033 d.addCallback(_check3)
2036 def test_POST_DIRURL_check(self):
2037 foo_url = self.public_url + "/foo/"
2038 d = self.POST(foo_url, t="check")
2040 self.failUnless("Healthy :" in res, res)
2041 d.addCallback(_check)
2042 redir_url = "http://allmydata.org/TARGET"
2043 def _check2(statuscode, target):
2044 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2045 self.failUnlessReallyEqual(target, redir_url)
2046 d.addCallback(lambda res:
2047 self.shouldRedirect2("test_POST_DIRURL_check",
2051 when_done=redir_url))
2052 d.addCallback(lambda res:
2053 self.POST(foo_url, t="check", return_to=redir_url))
2055 self.failUnless("Healthy :" in res, res)
2056 self.failUnless("Return to file/directory" in res)
2057 self.failUnless(redir_url in res)
2058 d.addCallback(_check3)
2060 d.addCallback(lambda res:
2061 self.POST(foo_url, t="check", output="JSON"))
2062 def _check_json(res):
2063 data = simplejson.loads(res)
2064 self.failUnless("storage-index" in data)
2065 self.failUnless(data["results"]["healthy"])
2066 d.addCallback(_check_json)
2070 def test_POST_DIRURL_check_and_repair(self):
2071 foo_url = self.public_url + "/foo/"
2072 d = self.POST(foo_url, t="check", repair="true")
2074 self.failUnless("Healthy :" in res, res)
2075 d.addCallback(_check)
2076 redir_url = "http://allmydata.org/TARGET"
2077 def _check2(statuscode, target):
2078 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2079 self.failUnlessReallyEqual(target, redir_url)
2080 d.addCallback(lambda res:
2081 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2084 t="check", repair="true",
2085 when_done=redir_url))
2086 d.addCallback(lambda res:
2087 self.POST(foo_url, t="check", return_to=redir_url))
2089 self.failUnless("Healthy :" in res)
2090 self.failUnless("Return to file/directory" in res)
2091 self.failUnless(redir_url in res)
2092 d.addCallback(_check3)
2095 def wait_for_operation(self, ignored, ophandle):
2096 url = "/operations/" + ophandle
2097 url += "?t=status&output=JSON"
2100 data = simplejson.loads(res)
2101 if not data["finished"]:
2102 d = self.stall(delay=1.0)
2103 d.addCallback(self.wait_for_operation, ophandle)
2109 def get_operation_results(self, ignored, ophandle, output=None):
2110 url = "/operations/" + ophandle
2113 url += "&output=" + output
2116 if output and output.lower() == "json":
2117 return simplejson.loads(res)
2122 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2123 d = self.shouldFail2(error.Error,
2124 "test_POST_DIRURL_deepcheck_no_ophandle",
2126 "slow operation requires ophandle=",
2127 self.POST, self.public_url, t="start-deep-check")
2130 def test_POST_DIRURL_deepcheck(self):
2131 def _check_redirect(statuscode, target):
2132 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2133 self.failUnless(target.endswith("/operations/123"))
2134 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2135 self.POST, self.public_url,
2136 t="start-deep-check", ophandle="123")
2137 d.addCallback(self.wait_for_operation, "123")
2138 def _check_json(data):
2139 self.failUnlessReallyEqual(data["finished"], True)
2140 self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2141 self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
2142 d.addCallback(_check_json)
2143 d.addCallback(self.get_operation_results, "123", "html")
2144 def _check_html(res):
2145 self.failUnless("Objects Checked: <span>8</span>" in res)
2146 self.failUnless("Objects Healthy: <span>8</span>" in res)
2147 d.addCallback(_check_html)
2149 d.addCallback(lambda res:
2150 self.GET("/operations/123/"))
2151 d.addCallback(_check_html) # should be the same as without the slash
2153 d.addCallback(lambda res:
2154 self.shouldFail2(error.Error, "one", "404 Not Found",
2155 "No detailed results for SI bogus",
2156 self.GET, "/operations/123/bogus"))
2158 foo_si = self._foo_node.get_storage_index()
2159 foo_si_s = base32.b2a(foo_si)
2160 d.addCallback(lambda res:
2161 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2162 def _check_foo_json(res):
2163 data = simplejson.loads(res)
2164 self.failUnlessEqual(data["storage-index"], foo_si_s)
2165 self.failUnless(data["results"]["healthy"])
2166 d.addCallback(_check_foo_json)
2169 def test_POST_DIRURL_deepcheck_and_repair(self):
2170 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2171 ophandle="124", output="json", followRedirect=True)
2172 d.addCallback(self.wait_for_operation, "124")
2173 def _check_json(data):
2174 self.failUnlessReallyEqual(data["finished"], True)
2175 self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2176 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
2177 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2178 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2179 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2180 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2181 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2182 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
2183 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2184 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2185 d.addCallback(_check_json)
2186 d.addCallback(self.get_operation_results, "124", "html")
2187 def _check_html(res):
2188 self.failUnless("Objects Checked: <span>8</span>" in res)
2190 self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2191 self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2192 self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2194 self.failUnless("Repairs Attempted: <span>0</span>" in res)
2195 self.failUnless("Repairs Successful: <span>0</span>" in res)
2196 self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2198 self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2199 self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2200 self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2201 d.addCallback(_check_html)
2204 def test_POST_FILEURL_bad_t(self):
2205 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2206 "POST to file: bad t=bogus",
2207 self.POST, self.public_url + "/foo/bar.txt",
2211 def test_POST_mkdir(self): # return value?
2212 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2213 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2214 d.addCallback(self.failUnlessNodeKeysAre, [])
2217 def test_POST_mkdir_initial_children(self):
2218 (newkids, caps) = self._create_initial_children()
2219 d = self.POST2(self.public_url +
2220 "/foo?t=mkdir-with-children&name=newdir",
2221 simplejson.dumps(newkids))
2222 d.addCallback(lambda res:
2223 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2224 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2225 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2226 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2227 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2230 def test_POST_mkdir_immutable(self):
2231 (newkids, caps) = self._create_immutable_children()
2232 d = self.POST2(self.public_url +
2233 "/foo?t=mkdir-immutable&name=newdir",
2234 simplejson.dumps(newkids))
2235 d.addCallback(lambda res:
2236 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2237 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2238 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2239 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2240 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2241 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2242 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2243 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2244 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2245 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2246 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2247 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2248 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2251 def test_POST_mkdir_immutable_bad(self):
2252 (newkids, caps) = self._create_initial_children()
2253 d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2255 "needed to be immutable but was not",
2258 "/foo?t=mkdir-immutable&name=newdir",
2259 simplejson.dumps(newkids))
2262 def test_POST_mkdir_2(self):
2263 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2264 d.addCallback(lambda res:
2265 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2266 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2267 d.addCallback(self.failUnlessNodeKeysAre, [])
2270 def test_POST_mkdirs_2(self):
2271 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2272 d.addCallback(lambda res:
2273 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2274 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2275 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2276 d.addCallback(self.failUnlessNodeKeysAre, [])
2279 def test_POST_mkdir_no_parentdir_noredirect(self):
2280 d = self.POST("/uri?t=mkdir")
2281 def _after_mkdir(res):
2282 uri.DirectoryURI.init_from_string(res)
2283 d.addCallback(_after_mkdir)
2286 def test_POST_mkdir_no_parentdir_noredirect2(self):
2287 # make sure form-based arguments (as on the welcome page) still work
2288 d = self.POST("/uri", t="mkdir")
2289 def _after_mkdir(res):
2290 uri.DirectoryURI.init_from_string(res)
2291 d.addCallback(_after_mkdir)
2292 d.addErrback(self.explain_web_error)
2295 def test_POST_mkdir_no_parentdir_redirect(self):
2296 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2297 d.addBoth(self.shouldRedirect, None, statuscode='303')
2298 def _check_target(target):
2299 target = urllib.unquote(target)
2300 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2301 d.addCallback(_check_target)
2304 def test_POST_mkdir_no_parentdir_redirect2(self):
2305 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2306 d.addBoth(self.shouldRedirect, None, statuscode='303')
2307 def _check_target(target):
2308 target = urllib.unquote(target)
2309 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2310 d.addCallback(_check_target)
2311 d.addErrback(self.explain_web_error)
2314 def _make_readonly(self, u):
2315 ro_uri = uri.from_string(u).get_readonly()
2318 return ro_uri.to_string()
2320 def _create_initial_children(self):
2321 contents, n, filecap1 = self.makefile(12)
2322 md1 = {"metakey1": "metavalue1"}
2323 filecap2 = make_mutable_file_uri()
2324 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2325 filecap3 = node3.get_readonly_uri()
2326 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2327 dircap = DirectoryNode(node4, None, None).get_uri()
2328 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2329 emptydircap = "URI:DIR2-LIT:"
2330 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
2331 "ro_uri": self._make_readonly(filecap1),
2332 "metadata": md1, }],
2333 u"child-mutable": ["filenode", {"rw_uri": filecap2,
2334 "ro_uri": self._make_readonly(filecap2)}],
2335 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2336 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
2337 "ro_uri": unknown_rocap}],
2338 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
2339 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2340 u"dirchild": ["dirnode", {"rw_uri": dircap,
2341 "ro_uri": self._make_readonly(dircap)}],
2342 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2343 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2345 return newkids, {'filecap1': filecap1,
2346 'filecap2': filecap2,
2347 'filecap3': filecap3,
2348 'unknown_rwcap': unknown_rwcap,
2349 'unknown_rocap': unknown_rocap,
2350 'unknown_immcap': unknown_immcap,
2352 'litdircap': litdircap,
2353 'emptydircap': emptydircap}
2355 def _create_immutable_children(self):
2356 contents, n, filecap1 = self.makefile(12)
2357 md1 = {"metakey1": "metavalue1"}
2358 tnode = create_chk_filenode("immutable directory contents\n"*10)
2359 dnode = DirectoryNode(tnode, None, None)
2360 assert not dnode.is_mutable()
2361 immdircap = dnode.get_uri()
2362 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2363 emptydircap = "URI:DIR2-LIT:"
2364 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2365 "metadata": md1, }],
2366 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2367 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2368 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2369 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2371 return newkids, {'filecap1': filecap1,
2372 'unknown_immcap': unknown_immcap,
2373 'immdircap': immdircap,
2374 'litdircap': litdircap,
2375 'emptydircap': emptydircap}
2377 def test_POST_mkdir_no_parentdir_initial_children(self):
2378 (newkids, caps) = self._create_initial_children()
2379 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2380 def _after_mkdir(res):
2381 self.failUnless(res.startswith("URI:DIR"), res)
2382 n = self.s.create_node_from_uri(res)
2383 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2384 d2.addCallback(lambda ign:
2385 self.failUnlessROChildURIIs(n, u"child-imm",
2387 d2.addCallback(lambda ign:
2388 self.failUnlessRWChildURIIs(n, u"child-mutable",
2390 d2.addCallback(lambda ign:
2391 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2393 d2.addCallback(lambda ign:
2394 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2395 caps['unknown_rwcap']))
2396 d2.addCallback(lambda ign:
2397 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2398 caps['unknown_rocap']))
2399 d2.addCallback(lambda ign:
2400 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2401 caps['unknown_immcap']))
2402 d2.addCallback(lambda ign:
2403 self.failUnlessRWChildURIIs(n, u"dirchild",
2406 d.addCallback(_after_mkdir)
2409 def test_POST_mkdir_no_parentdir_unexpected_children(self):
2410 # the regular /uri?t=mkdir operation is specified to ignore its body.
2411 # Only t=mkdir-with-children pays attention to it.
2412 (newkids, caps) = self._create_initial_children()
2413 d = self.shouldHTTPError("POST t=mkdir unexpected children",
2415 "t=mkdir does not accept children=, "
2416 "try t=mkdir-with-children instead",
2417 self.POST2, "/uri?t=mkdir", # without children
2418 simplejson.dumps(newkids))
2421 def test_POST_noparent_bad(self):
2422 d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2423 "/uri accepts only PUT, PUT?t=mkdir, "
2424 "POST?t=upload, and POST?t=mkdir",
2425 self.POST, "/uri?t=bogus")
2428 def test_POST_mkdir_no_parentdir_immutable(self):
2429 (newkids, caps) = self._create_immutable_children()
2430 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2431 def _after_mkdir(res):
2432 self.failUnless(res.startswith("URI:DIR"), res)
2433 n = self.s.create_node_from_uri(res)
2434 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2435 d2.addCallback(lambda ign:
2436 self.failUnlessROChildURIIs(n, u"child-imm",
2438 d2.addCallback(lambda ign:
2439 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2440 caps['unknown_immcap']))
2441 d2.addCallback(lambda ign:
2442 self.failUnlessROChildURIIs(n, u"dirchild-imm",
2444 d2.addCallback(lambda ign:
2445 self.failUnlessROChildURIIs(n, u"dirchild-lit",
2447 d2.addCallback(lambda ign:
2448 self.failUnlessROChildURIIs(n, u"dirchild-empty",
2449 caps['emptydircap']))
2451 d.addCallback(_after_mkdir)
2454 def test_POST_mkdir_no_parentdir_immutable_bad(self):
2455 (newkids, caps) = self._create_initial_children()
2456 d = self.shouldFail2(error.Error,
2457 "test_POST_mkdir_no_parentdir_immutable_bad",
2459 "needed to be immutable but was not",
2461 "/uri?t=mkdir-immutable",
2462 simplejson.dumps(newkids))
2465 def test_welcome_page_mkdir_button(self):
2466 # Fetch the welcome page.
2468 def _after_get_welcome_page(res):
2469 MKDIR_BUTTON_RE = re.compile(
2470 '<form action="([^"]*)" method="post".*?'
2471 '<input type="hidden" name="t" value="([^"]*)" />'
2472 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2473 '<input type="submit" value="Create a directory" />',
2475 mo = MKDIR_BUTTON_RE.search(res)
2476 formaction = mo.group(1)
2478 formaname = mo.group(3)
2479 formavalue = mo.group(4)
2480 return (formaction, formt, formaname, formavalue)
2481 d.addCallback(_after_get_welcome_page)
2482 def _after_parse_form(res):
2483 (formaction, formt, formaname, formavalue) = res
2484 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2485 d.addCallback(_after_parse_form)
2486 d.addBoth(self.shouldRedirect, None, statuscode='303')
2489 def test_POST_mkdir_replace(self): # return value?
2490 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2491 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2492 d.addCallback(self.failUnlessNodeKeysAre, [])
2495 def test_POST_mkdir_no_replace_queryarg(self): # return value?
2496 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2497 d.addBoth(self.shouldFail, error.Error,
2498 "POST_mkdir_no_replace_queryarg",
2500 "There was already a child by that name, and you asked me "
2501 "to not replace it")
2502 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2503 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2506 def test_POST_mkdir_no_replace_field(self): # return value?
2507 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2509 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2511 "There was already a child by that name, and you asked me "
2512 "to not replace it")
2513 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2514 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2517 def test_POST_mkdir_whendone_field(self):
2518 d = self.POST(self.public_url + "/foo",
2519 t="mkdir", name="newdir", when_done="/THERE")
2520 d.addBoth(self.shouldRedirect, "/THERE")
2521 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2522 d.addCallback(self.failUnlessNodeKeysAre, [])
2525 def test_POST_mkdir_whendone_queryarg(self):
2526 d = self.POST(self.public_url + "/foo?when_done=/THERE",
2527 t="mkdir", name="newdir")
2528 d.addBoth(self.shouldRedirect, "/THERE")
2529 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2530 d.addCallback(self.failUnlessNodeKeysAre, [])
2533 def test_POST_bad_t(self):
2534 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2535 "POST to a directory with bad t=BOGUS",
2536 self.POST, self.public_url + "/foo", t="BOGUS")
2539 def test_POST_set_children(self, command_name="set_children"):
2540 contents9, n9, newuri9 = self.makefile(9)
2541 contents10, n10, newuri10 = self.makefile(10)
2542 contents11, n11, newuri11 = self.makefile(11)
2545 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2548 "ctime": 1002777696.7564139,
2549 "mtime": 1002777696.7564139
2552 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2555 "ctime": 1002777696.7564139,
2556 "mtime": 1002777696.7564139
2559 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2562 "ctime": 1002777696.7564139,
2563 "mtime": 1002777696.7564139
2566 }""" % (newuri9, newuri10, newuri11)
2568 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2570 d = client.getPage(url, method="POST", postdata=reqbody)
2572 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2573 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2574 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2576 d.addCallback(_then)
2577 d.addErrback(self.dump_error)
2580 def test_POST_set_children_with_hyphen(self):
2581 return self.test_POST_set_children(command_name="set-children")
2583 def test_POST_link_uri(self):
2584 contents, n, newuri = self.makefile(8)
2585 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2586 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2587 d.addCallback(lambda res:
2588 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2592 def test_POST_link_uri_replace(self):
2593 contents, n, newuri = self.makefile(8)
2594 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2595 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2596 d.addCallback(lambda res:
2597 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2601 def test_POST_link_uri_unknown_bad(self):
2602 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2603 d.addBoth(self.shouldFail, error.Error,
2604 "POST_link_uri_unknown_bad",
2606 "unknown cap in a write slot")
2609 def test_POST_link_uri_unknown_ro_good(self):
2610 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2611 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2614 def test_POST_link_uri_unknown_imm_good(self):
2615 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2616 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2619 def test_POST_link_uri_no_replace_queryarg(self):
2620 contents, n, newuri = self.makefile(8)
2621 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2622 name="bar.txt", uri=newuri)
2623 d.addBoth(self.shouldFail, error.Error,
2624 "POST_link_uri_no_replace_queryarg",
2626 "There was already a child by that name, and you asked me "
2627 "to not replace it")
2628 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2629 d.addCallback(self.failUnlessIsBarDotTxt)
2632 def test_POST_link_uri_no_replace_field(self):
2633 contents, n, newuri = self.makefile(8)
2634 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2635 name="bar.txt", uri=newuri)
2636 d.addBoth(self.shouldFail, error.Error,
2637 "POST_link_uri_no_replace_field",
2639 "There was already a child by that name, and you asked me "
2640 "to not replace it")
2641 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2642 d.addCallback(self.failUnlessIsBarDotTxt)
2645 def test_POST_delete(self, command_name='delete'):
2646 d = self._foo_node.list()
2647 def _check_before(children):
2648 self.failUnless(u"bar.txt" in children)
2649 d.addCallback(_check_before)
2650 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
2651 d.addCallback(lambda res: self._foo_node.list())
2652 def _check_after(children):
2653 self.failIf(u"bar.txt" in children)
2654 d.addCallback(_check_after)
2657 def test_POST_unlink(self):
2658 return self.test_POST_delete(command_name='unlink')
2660 def test_POST_rename_file(self):
2661 d = self.POST(self.public_url + "/foo", t="rename",
2662 from_name="bar.txt", to_name='wibble.txt')
2663 d.addCallback(lambda res:
2664 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2665 d.addCallback(lambda res:
2666 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2667 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2668 d.addCallback(self.failUnlessIsBarDotTxt)
2669 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2670 d.addCallback(self.failUnlessIsBarJSON)
2673 def test_POST_rename_file_redundant(self):
2674 d = self.POST(self.public_url + "/foo", t="rename",
2675 from_name="bar.txt", to_name='bar.txt')
2676 d.addCallback(lambda res:
2677 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2678 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2679 d.addCallback(self.failUnlessIsBarDotTxt)
2680 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2681 d.addCallback(self.failUnlessIsBarJSON)
2684 def test_POST_rename_file_replace(self):
2685 # rename a file and replace a directory with it
2686 d = self.POST(self.public_url + "/foo", t="rename",
2687 from_name="bar.txt", to_name='empty')
2688 d.addCallback(lambda res:
2689 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2690 d.addCallback(lambda res:
2691 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2692 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2693 d.addCallback(self.failUnlessIsBarDotTxt)
2694 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2695 d.addCallback(self.failUnlessIsBarJSON)
2698 def test_POST_rename_file_no_replace_queryarg(self):
2699 # rename a file and replace a directory with it
2700 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2701 from_name="bar.txt", to_name='empty')
2702 d.addBoth(self.shouldFail, error.Error,
2703 "POST_rename_file_no_replace_queryarg",
2705 "There was already a child by that name, and you asked me "
2706 "to not replace it")
2707 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2708 d.addCallback(self.failUnlessIsEmptyJSON)
2711 def test_POST_rename_file_no_replace_field(self):
2712 # rename a file and replace a directory with it
2713 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2714 from_name="bar.txt", to_name='empty')
2715 d.addBoth(self.shouldFail, error.Error,
2716 "POST_rename_file_no_replace_field",
2718 "There was already a child by that name, and you asked me "
2719 "to not replace it")
2720 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2721 d.addCallback(self.failUnlessIsEmptyJSON)
2724 def failUnlessIsEmptyJSON(self, res):
2725 data = simplejson.loads(res)
2726 self.failUnlessEqual(data[0], "dirnode", data)
2727 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
2729 def test_POST_rename_file_slash_fail(self):
2730 d = self.POST(self.public_url + "/foo", t="rename",
2731 from_name="bar.txt", to_name='kirk/spock.txt')
2732 d.addBoth(self.shouldFail, error.Error,
2733 "test_POST_rename_file_slash_fail",
2735 "to_name= may not contain a slash",
2737 d.addCallback(lambda res:
2738 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2741 def test_POST_rename_dir(self):
2742 d = self.POST(self.public_url, t="rename",
2743 from_name="foo", to_name='plunk')
2744 d.addCallback(lambda res:
2745 self.failIfNodeHasChild(self.public_root, u"foo"))
2746 d.addCallback(lambda res:
2747 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2748 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2749 d.addCallback(self.failUnlessIsFooJSON)
2752 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2753 """ If target is not None then the redirection has to go to target. If
2754 statuscode is not None then the redirection has to be accomplished with
2755 that HTTP status code."""
2756 if not isinstance(res, failure.Failure):
2757 to_where = (target is None) and "somewhere" or ("to " + target)
2758 self.fail("%s: we were expecting to get redirected %s, not get an"
2759 " actual page: %s" % (which, to_where, res))
2760 res.trap(error.PageRedirect)
2761 if statuscode is not None:
2762 self.failUnlessReallyEqual(res.value.status, statuscode,
2763 "%s: not a redirect" % which)
2764 if target is not None:
2765 # the PageRedirect does not seem to capture the uri= query arg
2766 # properly, so we can't check for it.
2767 realtarget = self.webish_url + target
2768 self.failUnlessReallyEqual(res.value.location, realtarget,
2769 "%s: wrong target" % which)
2770 return res.value.location
2772 def test_GET_URI_form(self):
2773 base = "/uri?uri=%s" % self._bar_txt_uri
2774 # this is supposed to give us a redirect to /uri/$URI, plus arguments
2775 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2777 d.addBoth(self.shouldRedirect, targetbase)
2778 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2779 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2780 d.addCallback(lambda res: self.GET(base+"&t=json"))
2781 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2782 d.addCallback(self.log, "about to get file by uri")
2783 d.addCallback(lambda res: self.GET(base, followRedirect=True))
2784 d.addCallback(self.failUnlessIsBarDotTxt)
2785 d.addCallback(self.log, "got file by uri, about to get dir by uri")
2786 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2787 followRedirect=True))
2788 d.addCallback(self.failUnlessIsFooJSON)
2789 d.addCallback(self.log, "got dir by uri")
2793 def test_GET_URI_form_bad(self):
2794 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2795 "400 Bad Request", "GET /uri requires uri=",
2799 def test_GET_rename_form(self):
2800 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2801 followRedirect=True)
2803 self.failUnless('name="when_done" value="."' in res, res)
2804 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2805 d.addCallback(_check)
2808 def log(self, res, msg):
2809 #print "MSG: %s RES: %s" % (msg, res)
2813 def test_GET_URI_URL(self):
2814 base = "/uri/%s" % self._bar_txt_uri
2816 d.addCallback(self.failUnlessIsBarDotTxt)
2817 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2818 d.addCallback(self.failUnlessIsBarDotTxt)
2819 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2820 d.addCallback(self.failUnlessIsBarDotTxt)
2823 def test_GET_URI_URL_dir(self):
2824 base = "/uri/%s?t=json" % self._foo_uri
2826 d.addCallback(self.failUnlessIsFooJSON)
2829 def test_GET_URI_URL_missing(self):
2830 base = "/uri/%s" % self._bad_file_uri
2831 d = self.shouldHTTPError("test_GET_URI_URL_missing",
2832 http.GONE, None, "NotEnoughSharesError",
2834 # TODO: how can we exercise both sides of WebDownloadTarget.fail
2835 # here? we must arrange for a download to fail after target.open()
2836 # has been called, and then inspect the response to see that it is
2837 # shorter than we expected.
2840 def test_PUT_DIRURL_uri(self):
2841 d = self.s.create_dirnode()
2843 new_uri = dn.get_uri()
2844 # replace /foo with a new (empty) directory
2845 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2846 d.addCallback(lambda res:
2847 self.failUnlessReallyEqual(res.strip(), new_uri))
2848 d.addCallback(lambda res:
2849 self.failUnlessRWChildURIIs(self.public_root,
2853 d.addCallback(_made_dir)
2856 def test_PUT_DIRURL_uri_noreplace(self):
2857 d = self.s.create_dirnode()
2859 new_uri = dn.get_uri()
2860 # replace /foo with a new (empty) directory, but ask that
2861 # replace=false, so it should fail
2862 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2863 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2865 self.public_url + "/foo?t=uri&replace=false",
2867 d.addCallback(lambda res:
2868 self.failUnlessRWChildURIIs(self.public_root,
2872 d.addCallback(_made_dir)
2875 def test_PUT_DIRURL_bad_t(self):
2876 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2877 "400 Bad Request", "PUT to a directory",
2878 self.PUT, self.public_url + "/foo?t=BOGUS", "")
2879 d.addCallback(lambda res:
2880 self.failUnlessRWChildURIIs(self.public_root,
2885 def test_PUT_NEWFILEURL_uri(self):
2886 contents, n, new_uri = self.makefile(8)
2887 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2888 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2889 d.addCallback(lambda res:
2890 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2894 def test_PUT_NEWFILEURL_uri_replace(self):
2895 contents, n, new_uri = self.makefile(8)
2896 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2897 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2898 d.addCallback(lambda res:
2899 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2903 def test_PUT_NEWFILEURL_uri_no_replace(self):
2904 contents, n, new_uri = self.makefile(8)
2905 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2906 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2908 "There was already a child by that name, and you asked me "
2909 "to not replace it")
2912 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2913 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2914 d.addBoth(self.shouldFail, error.Error,
2915 "POST_put_uri_unknown_bad",
2917 "unknown cap in a write slot")
2920 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2921 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2922 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2923 u"put-future-ro.txt")
2926 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2927 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2928 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2929 u"put-future-imm.txt")
2932 def test_PUT_NEWFILE_URI(self):
2933 file_contents = "New file contents here\n"
2934 d = self.PUT("/uri", file_contents)
2936 assert isinstance(uri, str), uri
2937 self.failUnless(uri in FakeCHKFileNode.all_contents)
2938 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2940 return self.GET("/uri/%s" % uri)
2941 d.addCallback(_check)
2943 self.failUnlessReallyEqual(res, file_contents)
2944 d.addCallback(_check2)
2947 def test_PUT_NEWFILE_URI_not_mutable(self):
2948 file_contents = "New file contents here\n"
2949 d = self.PUT("/uri?mutable=false", file_contents)
2951 assert isinstance(uri, str), uri
2952 self.failUnless(uri in FakeCHKFileNode.all_contents)
2953 self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2955 return self.GET("/uri/%s" % uri)
2956 d.addCallback(_check)
2958 self.failUnlessReallyEqual(res, file_contents)
2959 d.addCallback(_check2)
2962 def test_PUT_NEWFILE_URI_only_PUT(self):
2963 d = self.PUT("/uri?t=bogus", "")
2964 d.addBoth(self.shouldFail, error.Error,
2965 "PUT_NEWFILE_URI_only_PUT",
2967 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2970 def test_PUT_NEWFILE_URI_mutable(self):
2971 file_contents = "New file contents here\n"
2972 d = self.PUT("/uri?mutable=true", file_contents)
2973 def _check1(filecap):
2974 filecap = filecap.strip()
2975 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2976 self.filecap = filecap
2977 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2978 self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2979 n = self.s.create_node_from_uri(filecap)
2980 return n.download_best_version()
2981 d.addCallback(_check1)
2983 self.failUnlessReallyEqual(data, file_contents)
2984 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2985 d.addCallback(_check2)
2987 self.failUnlessReallyEqual(res, file_contents)
2988 d.addCallback(_check3)
2991 def test_PUT_mkdir(self):
2992 d = self.PUT("/uri?t=mkdir", "")
2994 n = self.s.create_node_from_uri(uri.strip())
2995 d2 = self.failUnlessNodeKeysAre(n, [])
2996 d2.addCallback(lambda res:
2997 self.GET("/uri/%s?t=json" % uri))
2999 d.addCallback(_check)
3000 d.addCallback(self.failUnlessIsEmptyJSON)
3003 def test_POST_check(self):
3004 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3006 # this returns a string form of the results, which are probably
3007 # None since we're using fake filenodes.
3008 # TODO: verify that the check actually happened, by changing
3009 # FakeCHKFileNode to count how many times .check() has been
3012 d.addCallback(_done)
3015 def test_bad_method(self):
3016 url = self.webish_url + self.public_url + "/foo/bar.txt"
3017 d = self.shouldHTTPError("test_bad_method",
3018 501, "Not Implemented",
3019 "I don't know how to treat a BOGUS request.",
3020 client.getPage, url, method="BOGUS")
3023 def test_short_url(self):
3024 url = self.webish_url + "/uri"
3025 d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3026 "I don't know how to treat a DELETE request.",
3027 client.getPage, url, method="DELETE")
3030 def test_ophandle_bad(self):
3031 url = self.webish_url + "/operations/bogus?t=status"
3032 d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3033 "unknown/expired handle 'bogus'",
3034 client.getPage, url)
3037 def test_ophandle_cancel(self):
3038 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3039 followRedirect=True)
3040 d.addCallback(lambda ignored:
3041 self.GET("/operations/128?t=status&output=JSON"))
3043 data = simplejson.loads(res)
3044 self.failUnless("finished" in data, res)
3045 monitor = self.ws.root.child_operations.handles["128"][0]
3046 d = self.POST("/operations/128?t=cancel&output=JSON")
3048 data = simplejson.loads(res)
3049 self.failUnless("finished" in data, res)
3050 # t=cancel causes the handle to be forgotten
3051 self.failUnless(monitor.is_cancelled())
3052 d.addCallback(_check2)
3054 d.addCallback(_check1)
3055 d.addCallback(lambda ignored:
3056 self.shouldHTTPError("test_ophandle_cancel",
3057 404, "404 Not Found",
3058 "unknown/expired handle '128'",
3060 "/operations/128?t=status&output=JSON"))
3063 def test_ophandle_retainfor(self):
3064 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3065 followRedirect=True)
3066 d.addCallback(lambda ignored:
3067 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3069 data = simplejson.loads(res)
3070 self.failUnless("finished" in data, res)
3071 d.addCallback(_check1)
3072 # the retain-for=0 will cause the handle to be expired very soon
3073 d.addCallback(lambda ign:
3074 self.clock.advance(2.0))
3075 d.addCallback(lambda ignored:
3076 self.shouldHTTPError("test_ophandle_retainfor",
3077 404, "404 Not Found",
3078 "unknown/expired handle '129'",
3080 "/operations/129?t=status&output=JSON"))
3083 def test_ophandle_release_after_complete(self):
3084 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3085 followRedirect=True)
3086 d.addCallback(self.wait_for_operation, "130")
3087 d.addCallback(lambda ignored:
3088 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3089 # the release-after-complete=true will cause the handle to be expired
3090 d.addCallback(lambda ignored:
3091 self.shouldHTTPError("test_ophandle_release_after_complete",
3092 404, "404 Not Found",
3093 "unknown/expired handle '130'",
3095 "/operations/130?t=status&output=JSON"))
3098 def test_uncollected_ophandle_expiration(self):
3099 # uncollected ophandles should expire after 4 days
3100 def _make_uncollected_ophandle(ophandle):
3101 d = self.POST(self.public_url +
3102 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3103 followRedirect=False)
3104 # When we start the operation, the webapi server will want
3105 # to redirect us to the page for the ophandle, so we get
3106 # confirmation that the operation has started. If the
3107 # manifest operation has finished by the time we get there,
3108 # following that redirect (by setting followRedirect=True
3109 # above) has the side effect of collecting the ophandle that
3110 # we've just created, which means that we can't use the
3111 # ophandle to test the uncollected timeout anymore. So,
3112 # instead, catch the 302 here and don't follow it.
3113 d.addBoth(self.should302, "uncollected_ophandle_creation")
3115 # Create an ophandle, don't collect it, then advance the clock by
3116 # 4 days - 1 second and make sure that the ophandle is still there.
3117 d = _make_uncollected_ophandle(131)
3118 d.addCallback(lambda ign:
3119 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3120 d.addCallback(lambda ign:
3121 self.GET("/operations/131?t=status&output=JSON"))
3123 data = simplejson.loads(res)
3124 self.failUnless("finished" in data, res)
3125 d.addCallback(_check1)
3126 # Create an ophandle, don't collect it, then try to collect it
3127 # after 4 days. It should be gone.
3128 d.addCallback(lambda ign:
3129 _make_uncollected_ophandle(132))
3130 d.addCallback(lambda ign:
3131 self.clock.advance(96*60*60))
3132 d.addCallback(lambda ign:
3133 self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3134 404, "404 Not Found",
3135 "unknown/expired handle '132'",
3137 "/operations/132?t=status&output=JSON"))
3140 def test_collected_ophandle_expiration(self):
3141 # collected ophandles should expire after 1 day
3142 def _make_collected_ophandle(ophandle):
3143 d = self.POST(self.public_url +
3144 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3145 followRedirect=True)
3146 # By following the initial redirect, we collect the ophandle
3147 # we've just created.
3149 # Create a collected ophandle, then collect it after 23 hours
3150 # and 59 seconds to make sure that it is still there.
3151 d = _make_collected_ophandle(133)
3152 d.addCallback(lambda ign:
3153 self.clock.advance((24*60*60) - 1))
3154 d.addCallback(lambda ign:
3155 self.GET("/operations/133?t=status&output=JSON"))
3157 data = simplejson.loads(res)
3158 self.failUnless("finished" in data, res)
3159 d.addCallback(_check1)
3160 # Create another uncollected ophandle, then try to collect it
3161 # after 24 hours to make sure that it is gone.
3162 d.addCallback(lambda ign:
3163 _make_collected_ophandle(134))
3164 d.addCallback(lambda ign:
3165 self.clock.advance(24*60*60))
3166 d.addCallback(lambda ign:
3167 self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3168 404, "404 Not Found",
3169 "unknown/expired handle '134'",
3171 "/operations/134?t=status&output=JSON"))
3174 def test_incident(self):
3175 d = self.POST("/report_incident", details="eek")
3177 self.failUnless("Thank you for your report!" in res, res)
3178 d.addCallback(_done)
3181 def test_static(self):
3182 webdir = os.path.join(self.staticdir, "subdir")
3183 fileutil.make_dirs(webdir)
3184 f = open(os.path.join(webdir, "hello.txt"), "wb")
3188 d = self.GET("/static/subdir/hello.txt")
3190 self.failUnlessReallyEqual(res, "hello")
3191 d.addCallback(_check)
3195 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3196 def test_load_file(self):
3197 # This will raise an exception unless a well-formed XML file is found under that name.
3198 common.getxmlfile('directory.xhtml').load()
3200 def test_parse_replace_arg(self):
3201 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3202 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3203 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3205 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3206 common.parse_replace_arg, "only_fles")
3208 def test_abbreviate_time(self):
3209 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3210 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3211 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3212 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3213 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3214 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3216 def test_compute_rate(self):
3217 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3218 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3219 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3220 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3221 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3222 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3223 self.shouldFail(AssertionError, "test_compute_rate", "",
3224 common.compute_rate, -100, 10)
3225 self.shouldFail(AssertionError, "test_compute_rate", "",
3226 common.compute_rate, 100, -10)
3229 rate = common.compute_rate(10*1000*1000, 1)
3230 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3232 def test_abbreviate_rate(self):
3233 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3234 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3235 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3236 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3238 def test_abbreviate_size(self):
3239 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3240 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3241 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3242 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3243 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3245 def test_plural(self):
3247 return "%d second%s" % (s, status.plural(s))
3248 self.failUnlessReallyEqual(convert(0), "0 seconds")
3249 self.failUnlessReallyEqual(convert(1), "1 second")
3250 self.failUnlessReallyEqual(convert(2), "2 seconds")
3252 return "has share%s: %s" % (status.plural(s), ",".join(s))
3253 self.failUnlessReallyEqual(convert2([]), "has shares: ")
3254 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3255 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3258 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3260 def CHECK(self, ign, which, args, clientnum=0):
3261 fileurl = self.fileurls[which]
3262 url = fileurl + "?" + args
3263 return self.GET(url, method="POST", clientnum=clientnum)
3265 def test_filecheck(self):
3266 self.basedir = "web/Grid/filecheck"
3268 c0 = self.g.clients[0]
3271 d = c0.upload(upload.Data(DATA, convergence=""))
3272 def _stash_uri(ur, which):
3273 self.uris[which] = ur.uri
3274 d.addCallback(_stash_uri, "good")
3275 d.addCallback(lambda ign:
3276 c0.upload(upload.Data(DATA+"1", convergence="")))
3277 d.addCallback(_stash_uri, "sick")
3278 d.addCallback(lambda ign:
3279 c0.upload(upload.Data(DATA+"2", convergence="")))
3280 d.addCallback(_stash_uri, "dead")
3281 def _stash_mutable_uri(n, which):
3282 self.uris[which] = n.get_uri()
3283 assert isinstance(self.uris[which], str)
3284 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3285 d.addCallback(_stash_mutable_uri, "corrupt")
3286 d.addCallback(lambda ign:
3287 c0.upload(upload.Data("literal", convergence="")))
3288 d.addCallback(_stash_uri, "small")
3289 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3290 d.addCallback(_stash_mutable_uri, "smalldir")
3292 def _compute_fileurls(ignored):
3294 for which in self.uris:
3295 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3296 d.addCallback(_compute_fileurls)
3298 def _clobber_shares(ignored):
3299 good_shares = self.find_uri_shares(self.uris["good"])
3300 self.failUnlessReallyEqual(len(good_shares), 10)
3301 sick_shares = self.find_uri_shares(self.uris["sick"])
3302 os.unlink(sick_shares[0][2])
3303 dead_shares = self.find_uri_shares(self.uris["dead"])
3304 for i in range(1, 10):
3305 os.unlink(dead_shares[i][2])
3306 c_shares = self.find_uri_shares(self.uris["corrupt"])
3307 cso = CorruptShareOptions()
3308 cso.stdout = StringIO()
3309 cso.parseOptions([c_shares[0][2]])
3311 d.addCallback(_clobber_shares)
3313 d.addCallback(self.CHECK, "good", "t=check")
3314 def _got_html_good(res):
3315 self.failUnless("Healthy" in res, res)
3316 self.failIf("Not Healthy" in res, res)
3317 d.addCallback(_got_html_good)
3318 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3319 def _got_html_good_return_to(res):
3320 self.failUnless("Healthy" in res, res)
3321 self.failIf("Not Healthy" in res, res)
3322 self.failUnless('<a href="somewhere">Return to file'
3324 d.addCallback(_got_html_good_return_to)
3325 d.addCallback(self.CHECK, "good", "t=check&output=json")
3326 def _got_json_good(res):
3327 r = simplejson.loads(res)
3328 self.failUnlessEqual(r["summary"], "Healthy")
3329 self.failUnless(r["results"]["healthy"])
3330 self.failIf(r["results"]["needs-rebalancing"])
3331 self.failUnless(r["results"]["recoverable"])
3332 d.addCallback(_got_json_good)
3334 d.addCallback(self.CHECK, "small", "t=check")
3335 def _got_html_small(res):
3336 self.failUnless("Literal files are always healthy" in res, res)
3337 self.failIf("Not Healthy" in res, res)
3338 d.addCallback(_got_html_small)
3339 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3340 def _got_html_small_return_to(res):
3341 self.failUnless("Literal files are always healthy" in res, res)
3342 self.failIf("Not Healthy" in res, res)
3343 self.failUnless('<a href="somewhere">Return to file'
3345 d.addCallback(_got_html_small_return_to)
3346 d.addCallback(self.CHECK, "small", "t=check&output=json")
3347 def _got_json_small(res):
3348 r = simplejson.loads(res)
3349 self.failUnlessEqual(r["storage-index"], "")
3350 self.failUnless(r["results"]["healthy"])
3351 d.addCallback(_got_json_small)
3353 d.addCallback(self.CHECK, "smalldir", "t=check")
3354 def _got_html_smalldir(res):
3355 self.failUnless("Literal files are always healthy" in res, res)
3356 self.failIf("Not Healthy" in res, res)
3357 d.addCallback(_got_html_smalldir)
3358 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3359 def _got_json_smalldir(res):
3360 r = simplejson.loads(res)
3361 self.failUnlessEqual(r["storage-index"], "")
3362 self.failUnless(r["results"]["healthy"])
3363 d.addCallback(_got_json_smalldir)
3365 d.addCallback(self.CHECK, "sick", "t=check")
3366 def _got_html_sick(res):
3367 self.failUnless("Not Healthy" in res, res)
3368 d.addCallback(_got_html_sick)
3369 d.addCallback(self.CHECK, "sick", "t=check&output=json")
3370 def _got_json_sick(res):
3371 r = simplejson.loads(res)
3372 self.failUnlessEqual(r["summary"],
3373 "Not Healthy: 9 shares (enc 3-of-10)")
3374 self.failIf(r["results"]["healthy"])
3375 self.failIf(r["results"]["needs-rebalancing"])
3376 self.failUnless(r["results"]["recoverable"])
3377 d.addCallback(_got_json_sick)
3379 d.addCallback(self.CHECK, "dead", "t=check")
3380 def _got_html_dead(res):
3381 self.failUnless("Not Healthy" in res, res)
3382 d.addCallback(_got_html_dead)
3383 d.addCallback(self.CHECK, "dead", "t=check&output=json")
3384 def _got_json_dead(res):
3385 r = simplejson.loads(res)
3386 self.failUnlessEqual(r["summary"],
3387 "Not Healthy: 1 shares (enc 3-of-10)")
3388 self.failIf(r["results"]["healthy"])
3389 self.failIf(r["results"]["needs-rebalancing"])
3390 self.failIf(r["results"]["recoverable"])
3391 d.addCallback(_got_json_dead)
3393 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3394 def _got_html_corrupt(res):
3395 self.failUnless("Not Healthy! : Unhealthy" in res, res)
3396 d.addCallback(_got_html_corrupt)
3397 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3398 def _got_json_corrupt(res):
3399 r = simplejson.loads(res)
3400 self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3402 self.failIf(r["results"]["healthy"])
3403 self.failUnless(r["results"]["recoverable"])
3404 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
3405 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
3406 d.addCallback(_got_json_corrupt)
3408 d.addErrback(self.explain_web_error)
3411 def test_repair_html(self):
3412 self.basedir = "web/Grid/repair_html"
3414 c0 = self.g.clients[0]
3417 d = c0.upload(upload.Data(DATA, convergence=""))
3418 def _stash_uri(ur, which):
3419 self.uris[which] = ur.uri
3420 d.addCallback(_stash_uri, "good")
3421 d.addCallback(lambda ign:
3422 c0.upload(upload.Data(DATA+"1", convergence="")))
3423 d.addCallback(_stash_uri, "sick")
3424 d.addCallback(lambda ign:
3425 c0.upload(upload.Data(DATA+"2", convergence="")))
3426 d.addCallback(_stash_uri, "dead")
3427 def _stash_mutable_uri(n, which):
3428 self.uris[which] = n.get_uri()
3429 assert isinstance(self.uris[which], str)
3430 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3431 d.addCallback(_stash_mutable_uri, "corrupt")
3433 def _compute_fileurls(ignored):
3435 for which in self.uris:
3436 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3437 d.addCallback(_compute_fileurls)
3439 def _clobber_shares(ignored):
3440 good_shares = self.find_uri_shares(self.uris["good"])
3441 self.failUnlessReallyEqual(len(good_shares), 10)
3442 sick_shares = self.find_uri_shares(self.uris["sick"])
3443 os.unlink(sick_shares[0][2])
3444 dead_shares = self.find_uri_shares(self.uris["dead"])
3445 for i in range(1, 10):
3446 os.unlink(dead_shares[i][2])
3447 c_shares = self.find_uri_shares(self.uris["corrupt"])
3448 cso = CorruptShareOptions()
3449 cso.stdout = StringIO()
3450 cso.parseOptions([c_shares[0][2]])
3452 d.addCallback(_clobber_shares)
3454 d.addCallback(self.CHECK, "good", "t=check&repair=true")
3455 def _got_html_good(res):
3456 self.failUnless("Healthy" in res, res)
3457 self.failIf("Not Healthy" in res, res)
3458 self.failUnless("No repair necessary" in res, res)
3459 d.addCallback(_got_html_good)
3461 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3462 def _got_html_sick(res):
3463 self.failUnless("Healthy : healthy" in res, res)
3464 self.failIf("Not Healthy" in res, res)
3465 self.failUnless("Repair successful" in res, res)
3466 d.addCallback(_got_html_sick)
3468 # repair of a dead file will fail, of course, but it isn't yet
3469 # clear how this should be reported. Right now it shows up as
3472 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3473 #def _got_html_dead(res):
3475 # self.failUnless("Healthy : healthy" in res, res)
3476 # self.failIf("Not Healthy" in res, res)
3477 # self.failUnless("No repair necessary" in res, res)
3478 #d.addCallback(_got_html_dead)
3480 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3481 def _got_html_corrupt(res):
3482 self.failUnless("Healthy : Healthy" in res, res)
3483 self.failIf("Not Healthy" in res, res)
3484 self.failUnless("Repair successful" in res, res)
3485 d.addCallback(_got_html_corrupt)
3487 d.addErrback(self.explain_web_error)
3490 def test_repair_json(self):
3491 self.basedir = "web/Grid/repair_json"
3493 c0 = self.g.clients[0]
3496 d = c0.upload(upload.Data(DATA+"1", convergence=""))
3497 def _stash_uri(ur, which):
3498 self.uris[which] = ur.uri
3499 d.addCallback(_stash_uri, "sick")
3501 def _compute_fileurls(ignored):
3503 for which in self.uris:
3504 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3505 d.addCallback(_compute_fileurls)
3507 def _clobber_shares(ignored):
3508 sick_shares = self.find_uri_shares(self.uris["sick"])
3509 os.unlink(sick_shares[0][2])
3510 d.addCallback(_clobber_shares)
3512 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3513 def _got_json_sick(res):
3514 r = simplejson.loads(res)
3515 self.failUnlessReallyEqual(r["repair-attempted"], True)
3516 self.failUnlessReallyEqual(r["repair-successful"], True)
3517 self.failUnlessEqual(r["pre-repair-results"]["summary"],
3518 "Not Healthy: 9 shares (enc 3-of-10)")
3519 self.failIf(r["pre-repair-results"]["results"]["healthy"])
3520 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3521 self.failUnless(r["post-repair-results"]["results"]["healthy"])
3522 d.addCallback(_got_json_sick)
3524 d.addErrback(self.explain_web_error)
3527 def test_unknown(self, immutable=False):
3528 self.basedir = "web/Grid/unknown"
3530 self.basedir = "web/Grid/unknown-immutable"
3533 c0 = self.g.clients[0]
3537 # the future cap format may contain slashes, which must be tolerated
3538 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3542 name = u"future-imm"
3543 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3544 d = c0.create_immutable_dirnode({name: (future_node, {})})
3547 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3548 d = c0.create_dirnode()
3550 def _stash_root_and_create_file(n):
3552 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3553 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3555 return self.rootnode.set_node(name, future_node)
3556 d.addCallback(_stash_root_and_create_file)
3558 # make sure directory listing tolerates unknown nodes
3559 d.addCallback(lambda ign: self.GET(self.rooturl))
3560 def _check_directory_html(res, expected_type_suffix):
3561 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3562 '<td>%s</td>' % (expected_type_suffix, str(name)),
3564 self.failUnless(re.search(pattern, res), res)
3565 # find the More Info link for name, should be relative
3566 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3567 info_url = mo.group(1)
3568 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
3570 d.addCallback(_check_directory_html, "-IMM")
3572 d.addCallback(_check_directory_html, "")
3574 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3575 def _check_directory_json(res, expect_rw_uri):
3576 data = simplejson.loads(res)
3577 self.failUnlessEqual(data[0], "dirnode")
3578 f = data[1]["children"][name]
3579 self.failUnlessEqual(f[0], "unknown")
3581 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
3583 self.failIfIn("rw_uri", f[1])
3585 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
3587 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
3588 self.failUnless("metadata" in f[1])
3589 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3591 def _check_info(res, expect_rw_uri, expect_ro_uri):
3592 self.failUnlessIn("Object Type: <span>unknown</span>", res)
3594 self.failUnlessIn(unknown_rwcap, res)
3597 self.failUnlessIn(unknown_immcap, res)
3599 self.failUnlessIn(unknown_rocap, res)
3601 self.failIfIn(unknown_rocap, res)
3602 self.failIfIn("Raw data as", res)
3603 self.failIfIn("Directory writecap", res)
3604 self.failIfIn("Checker Operations", res)
3605 self.failIfIn("Mutable File Operations", res)
3606 self.failIfIn("Directory Operations", res)
3608 # FIXME: these should have expect_rw_uri=not immutable; I don't know
3609 # why they fail. Possibly related to ticket #922.
3611 d.addCallback(lambda ign: self.GET(expected_info_url))
3612 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3613 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3614 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3616 def _check_json(res, expect_rw_uri):
3617 data = simplejson.loads(res)
3618 self.failUnlessEqual(data[0], "unknown")
3620 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
3622 self.failIfIn("rw_uri", data[1])
3625 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
3626 self.failUnlessReallyEqual(data[1]["mutable"], False)
3628 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3629 self.failUnlessReallyEqual(data[1]["mutable"], True)
3631 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3632 self.failIf("mutable" in data[1], data[1])
3634 # TODO: check metadata contents
3635 self.failUnless("metadata" in data[1])
3637 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3638 d.addCallback(_check_json, expect_rw_uri=not immutable)
3640 # and make sure that a read-only version of the directory can be
3641 # rendered too. This version will not have unknown_rwcap, whether
3642 # or not future_node was immutable.
3643 d.addCallback(lambda ign: self.GET(self.rourl))
3645 d.addCallback(_check_directory_html, "-IMM")
3647 d.addCallback(_check_directory_html, "-RO")
3649 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3650 d.addCallback(_check_directory_json, expect_rw_uri=False)
3652 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3653 d.addCallback(_check_json, expect_rw_uri=False)
3655 # TODO: check that getting t=info from the Info link in the ro directory
3656 # works, and does not include the writecap URI.
3659 def test_immutable_unknown(self):
3660 return self.test_unknown(immutable=True)
3662 def test_mutant_dirnodes_are_omitted(self):
3663 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3666 c = self.g.clients[0]
3671 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3672 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3673 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3675 # This method tests mainly dirnode, but we'd have to duplicate code in order to
3676 # test the dirnode and web layers separately.
3678 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3679 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3680 # When the directory is read, the mutants should be silently disposed of, leaving
3681 # their lonely sibling.
3682 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3683 # because immutable directories don't have a writecap and therefore that field
3684 # isn't (and can't be) decrypted.
3685 # TODO: The field still exists in the netstring. Technically we should check what
3686 # happens if something is put there (_unpack_contents should raise ValueError),
3687 # but that can wait.
3689 lonely_child = nm.create_from_cap(lonely_uri)
3690 mutant_ro_child = nm.create_from_cap(mut_read_uri)
3691 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3693 def _by_hook_or_by_crook():
3695 for n in [mutant_ro_child, mutant_write_in_ro_child]:
3696 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3698 mutant_write_in_ro_child.get_write_uri = lambda: None
3699 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3701 kids = {u"lonely": (lonely_child, {}),
3702 u"ro": (mutant_ro_child, {}),
3703 u"write-in-ro": (mutant_write_in_ro_child, {}),
3705 d = c.create_immutable_dirnode(kids)
3708 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3709 self.failIf(dn.is_mutable())
3710 self.failUnless(dn.is_readonly())
3711 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3712 self.failIf(hasattr(dn._node, 'get_writekey'))
3714 self.failUnless("RO-IMM" in rep)
3716 self.failUnlessIn("CHK", cap.to_string())
3719 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3720 return download_to_data(dn._node)
3721 d.addCallback(_created)
3723 def _check_data(data):
3724 # Decode the netstring representation of the directory to check that all children
3725 # are present. This is a bit of an abstraction violation, but there's not really
3726 # any other way to do it given that the real DirectoryNode._unpack_contents would
3727 # strip the mutant children out (which is what we're trying to test, later).
3730 while position < len(data):
3731 entries, position = split_netstring(data, 1, position)
3733 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3734 name = name_utf8.decode("utf-8")
3735 self.failUnless(rwcapdata == "")
3736 self.failUnless(name in kids)
3737 (expected_child, ign) = kids[name]
3738 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
3741 self.failUnlessReallyEqual(numkids, 3)
3742 return self.rootnode.list()
3743 d.addCallback(_check_data)
3745 # Now when we use the real directory listing code, the mutants should be absent.
3746 def _check_kids(children):
3747 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
3748 lonely_node, lonely_metadata = children[u"lonely"]
3750 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
3751 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
3752 d.addCallback(_check_kids)
3754 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3755 d.addCallback(lambda n: n.list())
3756 d.addCallback(_check_kids) # again with dirnode recreated from cap
3758 # Make sure the lonely child can be listed in HTML...
3759 d.addCallback(lambda ign: self.GET(self.rooturl))
3760 def _check_html(res):
3761 self.failIfIn("URI:SSK", res)
3762 get_lonely = "".join([r'<td>FILE</td>',
3764 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3766 r'\s+<td align="right">%d</td>' % len("one"),
3768 self.failUnless(re.search(get_lonely, res), res)
3770 # find the More Info link for name, should be relative
3771 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3772 info_url = mo.group(1)
3773 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3774 d.addCallback(_check_html)
3777 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3778 def _check_json(res):
3779 data = simplejson.loads(res)
3780 self.failUnlessEqual(data[0], "dirnode")
3781 listed_children = data[1]["children"]
3782 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
3783 ll_type, ll_data = listed_children[u"lonely"]
3784 self.failUnlessEqual(ll_type, "filenode")
3785 self.failIf("rw_uri" in ll_data)
3786 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
3787 d.addCallback(_check_json)
3790 def test_deep_check(self):
3791 self.basedir = "web/Grid/deep_check"
3793 c0 = self.g.clients[0]
3797 d = c0.create_dirnode()
3798 def _stash_root_and_create_file(n):
3800 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3801 return n.add_file(u"good", upload.Data(DATA, convergence=""))
3802 d.addCallback(_stash_root_and_create_file)
3803 def _stash_uri(fn, which):
3804 self.uris[which] = fn.get_uri()
3806 d.addCallback(_stash_uri, "good")
3807 d.addCallback(lambda ign:
3808 self.rootnode.add_file(u"small",
3809 upload.Data("literal",
3811 d.addCallback(_stash_uri, "small")
3812 d.addCallback(lambda ign:
3813 self.rootnode.add_file(u"sick",
3814 upload.Data(DATA+"1",
3816 d.addCallback(_stash_uri, "sick")
3818 # this tests that deep-check and stream-manifest will ignore
3819 # UnknownNode instances. Hopefully this will also cover deep-stats.
3820 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3821 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3823 def _clobber_shares(ignored):
3824 self.delete_shares_numbered(self.uris["sick"], [0,1])
3825 d.addCallback(_clobber_shares)
3833 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3836 units = [simplejson.loads(line)
3837 for line in res.splitlines()
3840 print "response is:", res
3841 print "undecodeable line was '%s'" % line
3843 self.failUnlessReallyEqual(len(units), 5+1)
3844 # should be parent-first
3846 self.failUnlessEqual(u0["path"], [])
3847 self.failUnlessEqual(u0["type"], "directory")
3848 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
3849 u0cr = u0["check-results"]
3850 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
3852 ugood = [u for u in units
3853 if u["type"] == "file" and u["path"] == [u"good"]][0]
3854 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
3855 ugoodcr = ugood["check-results"]
3856 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
3859 self.failUnlessEqual(stats["type"], "stats")
3861 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3862 self.failUnlessReallyEqual(s["count-literal-files"], 1)
3863 self.failUnlessReallyEqual(s["count-directories"], 1)
3864 self.failUnlessReallyEqual(s["count-unknown"], 1)
3865 d.addCallback(_done)
3867 d.addCallback(self.CHECK, "root", "t=stream-manifest")
3868 def _check_manifest(res):
3869 self.failUnless(res.endswith("\n"))
3870 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3871 self.failUnlessReallyEqual(len(units), 5+1)
3872 self.failUnlessEqual(units[-1]["type"], "stats")
3874 self.failUnlessEqual(first["path"], [])
3875 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
3876 self.failUnlessEqual(first["type"], "directory")
3877 stats = units[-1]["stats"]
3878 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3879 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
3880 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
3881 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3882 self.failUnlessReallyEqual(stats["count-unknown"], 1)
3883 d.addCallback(_check_manifest)
3885 # now add root/subdir and root/subdir/grandchild, then make subdir
3886 # unrecoverable, then see what happens
3888 d.addCallback(lambda ign:
3889 self.rootnode.create_subdirectory(u"subdir"))
3890 d.addCallback(_stash_uri, "subdir")
3891 d.addCallback(lambda subdir_node:
3892 subdir_node.add_file(u"grandchild",
3893 upload.Data(DATA+"2",
3895 d.addCallback(_stash_uri, "grandchild")
3897 d.addCallback(lambda ign:
3898 self.delete_shares_numbered(self.uris["subdir"],
3906 # root/subdir [unrecoverable]
3907 # root/subdir/grandchild
3909 # how should a streaming-JSON API indicate fatal error?
3910 # answer: emit ERROR: instead of a JSON string
3912 d.addCallback(self.CHECK, "root", "t=stream-manifest")
3913 def _check_broken_manifest(res):
3914 lines = res.splitlines()
3916 for (i,line) in enumerate(lines)
3917 if line.startswith("ERROR:")]
3919 self.fail("no ERROR: in output: %s" % (res,))
3920 first_error = error_lines[0]
3921 error_line = lines[first_error]
3922 error_msg = lines[first_error+1:]
3923 error_msg_s = "\n".join(error_msg) + "\n"
3924 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3926 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3927 units = [simplejson.loads(line) for line in lines[:first_error]]
3928 self.failUnlessReallyEqual(len(units), 6) # includes subdir
3929 last_unit = units[-1]
3930 self.failUnlessEqual(last_unit["path"], ["subdir"])
3931 d.addCallback(_check_broken_manifest)
3933 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3934 def _check_broken_deepcheck(res):
3935 lines = res.splitlines()
3937 for (i,line) in enumerate(lines)
3938 if line.startswith("ERROR:")]
3940 self.fail("no ERROR: in output: %s" % (res,))
3941 first_error = error_lines[0]
3942 error_line = lines[first_error]
3943 error_msg = lines[first_error+1:]
3944 error_msg_s = "\n".join(error_msg) + "\n"
3945 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3947 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3948 units = [simplejson.loads(line) for line in lines[:first_error]]
3949 self.failUnlessReallyEqual(len(units), 6) # includes subdir
3950 last_unit = units[-1]
3951 self.failUnlessEqual(last_unit["path"], ["subdir"])
3952 r = last_unit["check-results"]["results"]
3953 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
3954 self.failUnlessReallyEqual(r["count-shares-good"], 1)
3955 self.failUnlessReallyEqual(r["recoverable"], False)
3956 d.addCallback(_check_broken_deepcheck)
3958 d.addErrback(self.explain_web_error)
3961 def test_deep_check_and_repair(self):
3962 self.basedir = "web/Grid/deep_check_and_repair"
3964 c0 = self.g.clients[0]
3968 d = c0.create_dirnode()
3969 def _stash_root_and_create_file(n):
3971 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3972 return n.add_file(u"good", upload.Data(DATA, convergence=""))
3973 d.addCallback(_stash_root_and_create_file)
3974 def _stash_uri(fn, which):
3975 self.uris[which] = fn.get_uri()
3976 d.addCallback(_stash_uri, "good")
3977 d.addCallback(lambda ign:
3978 self.rootnode.add_file(u"small",
3979 upload.Data("literal",
3981 d.addCallback(_stash_uri, "small")
3982 d.addCallback(lambda ign:
3983 self.rootnode.add_file(u"sick",
3984 upload.Data(DATA+"1",
3986 d.addCallback(_stash_uri, "sick")
3987 #d.addCallback(lambda ign:
3988 # self.rootnode.add_file(u"dead",
3989 # upload.Data(DATA+"2",
3991 #d.addCallback(_stash_uri, "dead")
3993 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3994 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3995 #d.addCallback(_stash_uri, "corrupt")
3997 def _clobber_shares(ignored):
3998 good_shares = self.find_uri_shares(self.uris["good"])
3999 self.failUnlessReallyEqual(len(good_shares), 10)
4000 sick_shares = self.find_uri_shares(self.uris["sick"])
4001 os.unlink(sick_shares[0][2])
4002 #dead_shares = self.find_uri_shares(self.uris["dead"])
4003 #for i in range(1, 10):
4004 # os.unlink(dead_shares[i][2])
4006 #c_shares = self.find_uri_shares(self.uris["corrupt"])
4007 #cso = CorruptShareOptions()
4008 #cso.stdout = StringIO()
4009 #cso.parseOptions([c_shares[0][2]])
4011 d.addCallback(_clobber_shares)
4014 # root/good CHK, 10 shares
4016 # root/sick CHK, 9 shares
4018 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4020 units = [simplejson.loads(line)
4021 for line in res.splitlines()
4023 self.failUnlessReallyEqual(len(units), 4+1)
4024 # should be parent-first
4026 self.failUnlessEqual(u0["path"], [])
4027 self.failUnlessEqual(u0["type"], "directory")
4028 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4029 u0crr = u0["check-and-repair-results"]
4030 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4031 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4033 ugood = [u for u in units
4034 if u["type"] == "file" and u["path"] == [u"good"]][0]
4035 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4036 ugoodcrr = ugood["check-and-repair-results"]
4037 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4038 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4040 usick = [u for u in units
4041 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4042 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4043 usickcrr = usick["check-and-repair-results"]
4044 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4045 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4046 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4047 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4050 self.failUnlessEqual(stats["type"], "stats")
4052 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4053 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4054 self.failUnlessReallyEqual(s["count-directories"], 1)
4055 d.addCallback(_done)
4057 d.addErrback(self.explain_web_error)
4060 def _count_leases(self, ignored, which):
4061 u = self.uris[which]
4062 shares = self.find_uri_shares(u)
4064 for shnum, serverid, fn in shares:
4065 sf = get_share_file(fn)
4066 num_leases = len(list(sf.get_leases()))
4067 lease_counts.append( (fn, num_leases) )
4070 def _assert_leasecount(self, lease_counts, expected):
4071 for (fn, num_leases) in lease_counts:
4072 if num_leases != expected:
4073 self.fail("expected %d leases, have %d, on %s" %
4074 (expected, num_leases, fn))
4076 def test_add_lease(self):
4077 self.basedir = "web/Grid/add_lease"
4078 self.set_up_grid(num_clients=2)
4079 c0 = self.g.clients[0]
4082 d = c0.upload(upload.Data(DATA, convergence=""))
4083 def _stash_uri(ur, which):
4084 self.uris[which] = ur.uri
4085 d.addCallback(_stash_uri, "one")
4086 d.addCallback(lambda ign:
4087 c0.upload(upload.Data(DATA+"1", convergence="")))
4088 d.addCallback(_stash_uri, "two")
4089 def _stash_mutable_uri(n, which):
4090 self.uris[which] = n.get_uri()
4091 assert isinstance(self.uris[which], str)
4092 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
4093 d.addCallback(_stash_mutable_uri, "mutable")
4095 def _compute_fileurls(ignored):
4097 for which in self.uris:
4098 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4099 d.addCallback(_compute_fileurls)
4101 d.addCallback(self._count_leases, "one")
4102 d.addCallback(self._assert_leasecount, 1)
4103 d.addCallback(self._count_leases, "two")
4104 d.addCallback(self._assert_leasecount, 1)
4105 d.addCallback(self._count_leases, "mutable")
4106 d.addCallback(self._assert_leasecount, 1)
4108 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4109 def _got_html_good(res):
4110 self.failUnless("Healthy" in res, res)
4111 self.failIf("Not Healthy" in res, res)
4112 d.addCallback(_got_html_good)
4114 d.addCallback(self._count_leases, "one")
4115 d.addCallback(self._assert_leasecount, 1)
4116 d.addCallback(self._count_leases, "two")
4117 d.addCallback(self._assert_leasecount, 1)
4118 d.addCallback(self._count_leases, "mutable")
4119 d.addCallback(self._assert_leasecount, 1)
4121 # this CHECK uses the original client, which uses the same
4122 # lease-secrets, so it will just renew the original lease
4123 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4124 d.addCallback(_got_html_good)
4126 d.addCallback(self._count_leases, "one")
4127 d.addCallback(self._assert_leasecount, 1)
4128 d.addCallback(self._count_leases, "two")
4129 d.addCallback(self._assert_leasecount, 1)
4130 d.addCallback(self._count_leases, "mutable")
4131 d.addCallback(self._assert_leasecount, 1)
4133 # this CHECK uses an alternate client, which adds a second lease
4134 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4135 d.addCallback(_got_html_good)
4137 d.addCallback(self._count_leases, "one")
4138 d.addCallback(self._assert_leasecount, 2)
4139 d.addCallback(self._count_leases, "two")
4140 d.addCallback(self._assert_leasecount, 1)
4141 d.addCallback(self._count_leases, "mutable")
4142 d.addCallback(self._assert_leasecount, 1)
4144 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4145 d.addCallback(_got_html_good)
4147 d.addCallback(self._count_leases, "one")
4148 d.addCallback(self._assert_leasecount, 2)
4149 d.addCallback(self._count_leases, "two")
4150 d.addCallback(self._assert_leasecount, 1)
4151 d.addCallback(self._count_leases, "mutable")
4152 d.addCallback(self._assert_leasecount, 1)
4154 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4156 d.addCallback(_got_html_good)
4158 d.addCallback(self._count_leases, "one")
4159 d.addCallback(self._assert_leasecount, 2)
4160 d.addCallback(self._count_leases, "two")
4161 d.addCallback(self._assert_leasecount, 1)
4162 d.addCallback(self._count_leases, "mutable")
4163 d.addCallback(self._assert_leasecount, 2)
4165 d.addErrback(self.explain_web_error)
4168 def test_deep_add_lease(self):
4169 self.basedir = "web/Grid/deep_add_lease"
4170 self.set_up_grid(num_clients=2)
4171 c0 = self.g.clients[0]
4175 d = c0.create_dirnode()
4176 def _stash_root_and_create_file(n):
4178 self.uris["root"] = n.get_uri()
4179 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4180 return n.add_file(u"one", upload.Data(DATA, convergence=""))
4181 d.addCallback(_stash_root_and_create_file)
4182 def _stash_uri(fn, which):
4183 self.uris[which] = fn.get_uri()
4184 d.addCallback(_stash_uri, "one")
4185 d.addCallback(lambda ign:
4186 self.rootnode.add_file(u"small",
4187 upload.Data("literal",
4189 d.addCallback(_stash_uri, "small")
4191 d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4192 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4193 d.addCallback(_stash_uri, "mutable")
4195 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4197 units = [simplejson.loads(line)
4198 for line in res.splitlines()
4200 # root, one, small, mutable, stats
4201 self.failUnlessReallyEqual(len(units), 4+1)
4202 d.addCallback(_done)
4204 d.addCallback(self._count_leases, "root")
4205 d.addCallback(self._assert_leasecount, 1)
4206 d.addCallback(self._count_leases, "one")
4207 d.addCallback(self._assert_leasecount, 1)
4208 d.addCallback(self._count_leases, "mutable")
4209 d.addCallback(self._assert_leasecount, 1)
4211 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4212 d.addCallback(_done)
4214 d.addCallback(self._count_leases, "root")
4215 d.addCallback(self._assert_leasecount, 1)
4216 d.addCallback(self._count_leases, "one")
4217 d.addCallback(self._assert_leasecount, 1)
4218 d.addCallback(self._count_leases, "mutable")
4219 d.addCallback(self._assert_leasecount, 1)
4221 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4223 d.addCallback(_done)
4225 d.addCallback(self._count_leases, "root")
4226 d.addCallback(self._assert_leasecount, 2)
4227 d.addCallback(self._count_leases, "one")
4228 d.addCallback(self._assert_leasecount, 2)
4229 d.addCallback(self._count_leases, "mutable")
4230 d.addCallback(self._assert_leasecount, 2)
4232 d.addErrback(self.explain_web_error)
4236 def test_exceptions(self):
4237 self.basedir = "web/Grid/exceptions"
4238 self.set_up_grid(num_clients=1, num_servers=2)
4239 c0 = self.g.clients[0]
4240 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4243 d = c0.create_dirnode()
4245 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4246 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4248 d.addCallback(_stash_root)
4249 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4251 self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4252 self.delete_shares_numbered(ur.uri, range(1,10))
4254 u = uri.from_string(ur.uri)
4255 u.key = testutil.flip_bit(u.key, 0)
4256 baduri = u.to_string()
4257 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4258 d.addCallback(_stash_bad)
4259 d.addCallback(lambda ign: c0.create_dirnode())
4260 def _mangle_dirnode_1share(n):
4262 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4263 self.fileurls["dir-1share-json"] = url + "?t=json"
4264 self.delete_shares_numbered(u, range(1,10))
4265 d.addCallback(_mangle_dirnode_1share)
4266 d.addCallback(lambda ign: c0.create_dirnode())
4267 def _mangle_dirnode_0share(n):
4269 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4270 self.fileurls["dir-0share-json"] = url + "?t=json"
4271 self.delete_shares_numbered(u, range(0,10))
4272 d.addCallback(_mangle_dirnode_0share)
4274 # NotEnoughSharesError should be reported sensibly, with a
4275 # text/plain explanation of the problem, and perhaps some
4276 # information on which shares *could* be found.
4278 d.addCallback(lambda ignored:
4279 self.shouldHTTPError("GET unrecoverable",
4280 410, "Gone", "NoSharesError",
4281 self.GET, self.fileurls["0shares"]))
4282 def _check_zero_shares(body):
4283 self.failIf("<html>" in body, body)
4284 body = " ".join(body.strip().split())
4285 exp = ("NoSharesError: no shares could be found. "
4286 "Zero shares usually indicates a corrupt URI, or that "
4287 "no servers were connected, but it might also indicate "
4288 "severe corruption. You should perform a filecheck on "
4289 "this object to learn more. The full error message is: "
4290 "no shares (need 3). Last failure: None")
4291 self.failUnlessReallyEqual(exp, body)
4292 d.addCallback(_check_zero_shares)
4295 d.addCallback(lambda ignored:
4296 self.shouldHTTPError("GET 1share",
4297 410, "Gone", "NotEnoughSharesError",
4298 self.GET, self.fileurls["1share"]))
4299 def _check_one_share(body):
4300 self.failIf("<html>" in body, body)
4301 body = " ".join(body.strip().split())
4302 msgbase = ("NotEnoughSharesError: This indicates that some "
4303 "servers were unavailable, or that shares have been "
4304 "lost to server departure, hard drive failure, or disk "
4305 "corruption. You should perform a filecheck on "
4306 "this object to learn more. The full error message is:"
4308 msg1 = msgbase + (" ran out of shares:"
4311 " overdue= unused= need 3. Last failure: None")
4312 msg2 = msgbase + (" ran out of shares:"
4314 " pending=Share(sh0-on-xgru5)"
4315 " overdue= unused= need 3. Last failure: None")
4316 self.failUnless(body == msg1 or body == msg2, body)
4317 d.addCallback(_check_one_share)
4319 d.addCallback(lambda ignored:
4320 self.shouldHTTPError("GET imaginary",
4321 404, "Not Found", None,
4322 self.GET, self.fileurls["imaginary"]))
4323 def _missing_child(body):
4324 self.failUnless("No such child: imaginary" in body, body)
4325 d.addCallback(_missing_child)
4327 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4328 def _check_0shares_dir_html(body):
4329 self.failUnless("<html>" in body, body)
4330 # we should see the regular page, but without the child table or
4332 body = " ".join(body.strip().split())
4333 self.failUnlessIn('href="?t=info">More info on this directory',
4335 exp = ("UnrecoverableFileError: the directory (or mutable file) "
4336 "could not be retrieved, because there were insufficient "
4337 "good shares. This might indicate that no servers were "
4338 "connected, insufficient servers were connected, the URI "
4339 "was corrupt, or that shares have been lost due to server "
4340 "departure, hard drive failure, or disk corruption. You "
4341 "should perform a filecheck on this object to learn more.")
4342 self.failUnlessIn(exp, body)
4343 self.failUnlessIn("No upload forms: directory is unreadable", body)
4344 d.addCallback(_check_0shares_dir_html)
4346 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4347 def _check_1shares_dir_html(body):
4348 # at some point, we'll split UnrecoverableFileError into 0-shares
4349 # and some-shares like we did for immutable files (since there
4350 # are different sorts of advice to offer in each case). For now,
4351 # they present the same way.
4352 self.failUnless("<html>" in body, body)
4353 body = " ".join(body.strip().split())
4354 self.failUnlessIn('href="?t=info">More info on this directory',
4356 exp = ("UnrecoverableFileError: the directory (or mutable file) "
4357 "could not be retrieved, because there were insufficient "
4358 "good shares. This might indicate that no servers were "
4359 "connected, insufficient servers were connected, the URI "
4360 "was corrupt, or that shares have been lost due to server "
4361 "departure, hard drive failure, or disk corruption. You "
4362 "should perform a filecheck on this object to learn more.")
4363 self.failUnlessIn(exp, body)
4364 self.failUnlessIn("No upload forms: directory is unreadable", body)
4365 d.addCallback(_check_1shares_dir_html)
4367 d.addCallback(lambda ignored:
4368 self.shouldHTTPError("GET dir-0share-json",
4369 410, "Gone", "UnrecoverableFileError",
4371 self.fileurls["dir-0share-json"]))
4372 def _check_unrecoverable_file(body):
4373 self.failIf("<html>" in body, body)
4374 body = " ".join(body.strip().split())
4375 exp = ("UnrecoverableFileError: the directory (or mutable file) "
4376 "could not be retrieved, because there were insufficient "
4377 "good shares. This might indicate that no servers were "
4378 "connected, insufficient servers were connected, the URI "
4379 "was corrupt, or that shares have been lost due to server "
4380 "departure, hard drive failure, or disk corruption. You "
4381 "should perform a filecheck on this object to learn more.")
4382 self.failUnlessReallyEqual(exp, body)
4383 d.addCallback(_check_unrecoverable_file)
4385 d.addCallback(lambda ignored:
4386 self.shouldHTTPError("GET dir-1share-json",
4387 410, "Gone", "UnrecoverableFileError",
4389 self.fileurls["dir-1share-json"]))
4390 d.addCallback(_check_unrecoverable_file)
4392 d.addCallback(lambda ignored:
4393 self.shouldHTTPError("GET imaginary",
4394 404, "Not Found", None,
4395 self.GET, self.fileurls["imaginary"]))
4397 # attach a webapi child that throws a random error, to test how it
4399 w = c0.getServiceNamed("webish")
4400 w.root.putChild("ERRORBOOM", ErrorBoom())
4402 # "Accept: */*" : should get a text/html stack trace
4403 # "Accept: text/plain" : should get a text/plain stack trace
4404 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4405 # no Accept header: should get a text/html stack trace
4407 d.addCallback(lambda ignored:
4408 self.shouldHTTPError("GET errorboom_html",
4409 500, "Internal Server Error", None,
4410 self.GET, "ERRORBOOM",
4411 headers={"accept": ["*/*"]}))
4412 def _internal_error_html1(body):
4413 self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4414 d.addCallback(_internal_error_html1)
4416 d.addCallback(lambda ignored:
4417 self.shouldHTTPError("GET errorboom_text",
4418 500, "Internal Server Error", None,
4419 self.GET, "ERRORBOOM",
4420 headers={"accept": ["text/plain"]}))
4421 def _internal_error_text2(body):
4422 self.failIf("<html>" in body, body)
4423 self.failUnless(body.startswith("Traceback "), body)
4424 d.addCallback(_internal_error_text2)
4426 CLI_accepts = "text/plain, application/octet-stream"
4427 d.addCallback(lambda ignored:
4428 self.shouldHTTPError("GET errorboom_text",
4429 500, "Internal Server Error", None,
4430 self.GET, "ERRORBOOM",
4431 headers={"accept": [CLI_accepts]}))
4432 def _internal_error_text3(body):
4433 self.failIf("<html>" in body, body)
4434 self.failUnless(body.startswith("Traceback "), body)
4435 d.addCallback(_internal_error_text3)
4437 d.addCallback(lambda ignored:
4438 self.shouldHTTPError("GET errorboom_text",
4439 500, "Internal Server Error", None,
4440 self.GET, "ERRORBOOM"))
4441 def _internal_error_html4(body):
4442 self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4443 d.addCallback(_internal_error_html4)
4445 def _flush_errors(res):
4446 # Trial: please ignore the CompletelyUnhandledError in the logs
4447 self.flushLoggedErrors(CompletelyUnhandledError)
4449 d.addBoth(_flush_errors)
4453 class CompletelyUnhandledError(Exception):
4455 class ErrorBoom(rend.Page):
4456 def beforeRender(self, ctx):
4457 raise CompletelyUnhandledError("whoops")