1 import os.path, re, urllib
3 from StringIO import StringIO
4 from twisted.application import service
5 from twisted.trial import unittest
6 from twisted.internet import defer, reactor
7 from twisted.web import client, error, http
8 from twisted.python import failure, log
9 from allmydata import interfaces, uri, webish
10 from allmydata.immutable import upload, download
11 from allmydata.web import status, common
12 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
13 from allmydata.util import fileutil, base32
14 from allmydata.util.assertutil import precondition
15 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
16 FakeMutableFileNode, create_chk_filenode, WebErrorMixin
17 from allmydata.interfaces import IURI, INewDirectoryURI, \
18 IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
19 from allmydata.mutable import servermap, publish, retrieve
20 import common_util as testutil
21 from allmydata.test.no_network import GridTestMixin
23 # create a fake uploader/downloader, and a couple of fake dirnodes, then
24 # create a webserver that works against them
26 class FakeIntroducerClient:
27 def get_all_connectors(self):
29 def get_all_connections_for(self, service_name):
31 def get_all_peerids(self):
34 class FakeClient(service.MultiService):
35 nodeid = "fake_nodeid"
36 nickname = "fake_nickname"
37 basedir = "fake_basedir"
38 def get_versions(self):
39 return {'allmydata': "fake",
44 introducer_furl = "None"
45 introducer_client = FakeIntroducerClient()
46 _all_upload_status = [upload.UploadStatus()]
47 _all_download_status = [download.DownloadStatus()]
48 _all_mapupdate_statuses = [servermap.UpdateStatus()]
49 _all_publish_statuses = [publish.PublishStatus()]
50 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
51 convergence = "some random string"
53 def connected_to_introducer(self):
56 def get_nickname_for_peerid(self, peerid):
59 def get_permuted_peers(self, service_name, key):
62 def create_node_from_uri(self, auri):
63 precondition(isinstance(auri, str), auri)
64 u = uri.from_string(auri)
65 if (INewDirectoryURI.providedBy(u)
66 or IReadonlyNewDirectoryURI.providedBy(u)):
67 return FakeDirectoryNode(self).init_from_uri(u)
68 if IFileURI.providedBy(u):
69 return FakeCHKFileNode(u, self)
70 assert IMutableFileURI.providedBy(u), u
71 return FakeMutableFileNode(self).init_from_uri(u)
73 def create_empty_dirnode(self):
74 n = FakeDirectoryNode(self)
76 d.addCallback(lambda res: n)
79 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
80 def create_mutable_file(self, contents=""):
81 n = FakeMutableFileNode(self)
82 return n.create(contents)
84 def upload(self, uploadable):
85 d = uploadable.get_size()
86 d.addCallback(lambda size: uploadable.read(size))
89 n = create_chk_filenode(self, data)
90 results = upload.UploadResults()
91 results.uri = n.get_uri()
93 d.addCallback(_got_data)
96 def list_all_upload_statuses(self):
97 return self._all_upload_status
98 def list_all_download_statuses(self):
99 return self._all_download_status
100 def list_all_mapupdate_statuses(self):
101 return self._all_mapupdate_statuses
102 def list_all_publish_statuses(self):
103 return self._all_publish_statuses
104 def list_all_retrieve_statuses(self):
105 return self._all_retrieve_statuses
106 def list_all_helper_statuses(self):
109 class MyGetter(client.HTTPPageGetter):
110 handleStatus_206 = lambda self: self.handleStatus_200()
112 class HTTPClientHEADFactory(client.HTTPClientFactory):
115 def noPage(self, reason):
116 # Twisted-2.5.0 and earlier had a bug, in which they would raise an
117 # exception when the response to a HEAD request had no body (when in
118 # fact they are defined to never have a body). This was fixed in
119 # Twisted-8.0 . To work around this, we catch the
120 # PartialDownloadError and make it disappear.
121 if (reason.check(client.PartialDownloadError)
122 and self.method.upper() == "HEAD"):
125 return client.HTTPClientFactory.noPage(self, reason)
127 class HTTPClientGETFactory(client.HTTPClientFactory):
130 class WebMixin(object):
132 self.s = FakeClient()
133 self.s.startService()
134 self.staticdir = self.mktemp()
135 self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
136 s.setServiceParent(self.s)
137 self.webish_port = port = s.listener._port.getHost().port
138 self.webish_url = "http://localhost:%d" % port
140 l = [ self.s.create_empty_dirnode() for x in range(6) ]
141 d = defer.DeferredList(l)
143 self.public_root = res[0][1]
144 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
145 self.public_url = "/uri/" + self.public_root.get_uri()
146 self.private_root = res[1][1]
150 self._foo_uri = foo.get_uri()
151 self._foo_readonly_uri = foo.get_readonly_uri()
152 self._foo_verifycap = foo.get_verify_cap().to_string()
153 # NOTE: we ignore the deferred on all set_uri() calls, because we
154 # know the fake nodes do these synchronously
155 self.public_root.set_uri(u"foo", foo.get_uri())
157 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
158 foo.set_uri(u"bar.txt", self._bar_txt_uri)
159 self._bar_txt_verifycap = n.get_verify_cap().to_string()
161 foo.set_uri(u"empty", res[3][1].get_uri())
162 sub_uri = res[4][1].get_uri()
163 self._sub_uri = sub_uri
164 foo.set_uri(u"sub", sub_uri)
165 sub = self.s.create_node_from_uri(sub_uri)
167 _ign, n, blocking_uri = self.makefile(1)
168 foo.set_uri(u"blockingfile", blocking_uri)
170 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
171 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
172 # still think of it as an umlaut
173 foo.set_uri(unicode_filename, self._bar_txt_uri)
175 _ign, n, baz_file = self.makefile(2)
176 self._baz_file_uri = baz_file
177 sub.set_uri(u"baz.txt", baz_file)
179 _ign, n, self._bad_file_uri = self.makefile(3)
180 # this uri should not be downloadable
181 del FakeCHKFileNode.all_contents[self._bad_file_uri]
184 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
185 rodir.set_uri(u"nor", baz_file)
190 # public/foo/blockingfile
193 # public/foo/sub/baz.txt
195 # public/reedownlee/nor
196 self.NEWFILE_CONTENTS = "newfile contents\n"
198 return foo.get_metadata_for(u"bar.txt")
200 def _got_metadata(metadata):
201 self._bar_txt_metadata = metadata
202 d.addCallback(_got_metadata)
205 def makefile(self, number):
206 contents = "contents of file %s\n" % number
207 n = create_chk_filenode(self.s, contents)
208 return contents, n, n.get_uri()
211 return self.s.stopService()
213 def failUnlessIsBarDotTxt(self, res):
214 self.failUnlessEqual(res, self.BAR_CONTENTS, res)
216 def failUnlessIsBarJSON(self, res):
217 data = simplejson.loads(res)
218 self.failUnless(isinstance(data, list))
219 self.failUnlessEqual(data[0], u"filenode")
220 self.failUnless(isinstance(data[1], dict))
221 self.failIf(data[1]["mutable"])
222 self.failIf("rw_uri" in data[1]) # immutable
223 self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
224 self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
225 self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
227 def failUnlessIsFooJSON(self, res):
228 data = simplejson.loads(res)
229 self.failUnless(isinstance(data, list))
230 self.failUnlessEqual(data[0], "dirnode", res)
231 self.failUnless(isinstance(data[1], dict))
232 self.failUnless(data[1]["mutable"])
233 self.failUnless("rw_uri" in data[1]) # mutable
234 self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
235 self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
236 self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
238 kidnames = sorted([unicode(n) for n in data[1]["children"]])
239 self.failUnlessEqual(kidnames,
240 [u"bar.txt", u"blockingfile", u"empty",
241 u"n\u00fc.txt", u"sub"])
242 kids = dict( [(unicode(name),value)
244 in data[1]["children"].iteritems()] )
245 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
246 self.failUnless("metadata" in kids[u"sub"][1])
247 self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
248 self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
249 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
250 self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
251 self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
252 self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
253 self._bar_txt_verifycap)
254 self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
255 self._bar_txt_metadata["ctime"])
256 self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
259 def GET(self, urlpath, followRedirect=False, return_response=False,
261 # if return_response=True, this fires with (data, statuscode,
262 # respheaders) instead of just data.
263 assert not isinstance(urlpath, unicode)
264 url = self.webish_url + urlpath
265 factory = HTTPClientGETFactory(url, method="GET",
266 followRedirect=followRedirect, **kwargs)
267 reactor.connectTCP("localhost", self.webish_port, factory)
270 return (data, factory.status, factory.response_headers)
272 d.addCallback(_got_data)
273 return factory.deferred
275 def HEAD(self, urlpath, return_response=False, **kwargs):
276 # this requires some surgery, because twisted.web.client doesn't want
277 # to give us back the response headers.
278 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
279 reactor.connectTCP("localhost", self.webish_port, factory)
282 return (data, factory.status, factory.response_headers)
284 d.addCallback(_got_data)
285 return factory.deferred
287 def PUT(self, urlpath, data, **kwargs):
288 url = self.webish_url + urlpath
289 return client.getPage(url, method="PUT", postdata=data, **kwargs)
291 def DELETE(self, urlpath):
292 url = self.webish_url + urlpath
293 return client.getPage(url, method="DELETE")
295 def POST(self, urlpath, followRedirect=False, **fields):
296 url = self.webish_url + urlpath
297 sepbase = "boogabooga"
301 form.append('Content-Disposition: form-data; name="_charset"')
305 for name, value in fields.iteritems():
306 if isinstance(value, tuple):
307 filename, value = value
308 form.append('Content-Disposition: form-data; name="%s"; '
309 'filename="%s"' % (name, filename.encode("utf-8")))
311 form.append('Content-Disposition: form-data; name="%s"' % name)
313 if isinstance(value, unicode):
314 value = value.encode("utf-8")
317 assert isinstance(value, str)
321 body = "\r\n".join(form) + "\r\n"
322 headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
324 return client.getPage(url, method="POST", postdata=body,
325 headers=headers, followRedirect=followRedirect)
327 def shouldFail(self, res, expected_failure, which,
328 substring=None, response_substring=None):
329 if isinstance(res, failure.Failure):
330 res.trap(expected_failure)
332 self.failUnless(substring in str(res),
333 "substring '%s' not in '%s'"
334 % (substring, str(res)))
335 if response_substring:
336 self.failUnless(response_substring in res.value.response,
337 "response substring '%s' not in '%s'"
338 % (response_substring, res.value.response))
340 self.fail("%s was supposed to raise %s, not get '%s'" %
341 (which, expected_failure, res))
343 def shouldFail2(self, expected_failure, which, substring,
345 callable, *args, **kwargs):
346 assert substring is None or isinstance(substring, str)
347 assert response_substring is None or isinstance(response_substring, str)
348 d = defer.maybeDeferred(callable, *args, **kwargs)
350 if isinstance(res, failure.Failure):
351 res.trap(expected_failure)
353 self.failUnless(substring in str(res),
354 "%s: substring '%s' not in '%s'"
355 % (which, substring, str(res)))
356 if response_substring:
357 self.failUnless(response_substring in res.value.response,
358 "%s: response substring '%s' not in '%s'"
360 response_substring, res.value.response))
362 self.fail("%s was supposed to raise %s, not get '%s'" %
363 (which, expected_failure, res))
367 def should404(self, res, which):
368 if isinstance(res, failure.Failure):
369 res.trap(error.Error)
370 self.failUnlessEqual(res.value.status, "404")
372 self.fail("%s was supposed to Error(404), not get '%s'" %
375 def shouldHTTPError(self, res, which, code=None, substring=None,
376 response_substring=None):
377 if isinstance(res, failure.Failure):
378 res.trap(error.Error)
380 self.failUnlessEqual(res.value.status, str(code))
382 self.failUnless(substring in str(res),
383 "substring '%s' not in '%s'"
384 % (substring, str(res)))
385 if response_substring:
386 self.failUnless(response_substring in res.value.response,
387 "response substring '%s' not in '%s'"
388 % (response_substring, res.value.response))
390 self.fail("%s was supposed to Error(%s), not get '%s'" %
393 def shouldHTTPError2(self, which,
394 code=None, substring=None, response_substring=None,
395 callable=None, *args, **kwargs):
396 assert substring is None or isinstance(substring, str)
398 d = defer.maybeDeferred(callable, *args, **kwargs)
399 d.addBoth(self.shouldHTTPError, which,
400 code, substring, response_substring)
404 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
405 def test_create(self):
408 def test_welcome(self):
411 self.failUnless('Welcome To AllMyData' in res)
412 self.failUnless('Tahoe' in res)
414 self.s.basedir = 'web/test_welcome'
415 fileutil.make_dirs("web/test_welcome")
416 fileutil.make_dirs("web/test_welcome/private")
418 d.addCallback(_check)
421 def test_provisioning(self):
422 d = self.GET("/provisioning/")
424 self.failUnless('Tahoe Provisioning Tool' in res)
425 fields = {'filled': True,
426 "num_users": int(50e3),
427 "files_per_user": 1000,
428 "space_per_user": int(1e9),
429 "sharing_ratio": 1.0,
430 "encoding_parameters": "3-of-10-5",
432 "ownership_mode": "A",
433 "download_rate": 100,
438 return self.POST("/provisioning/", **fields)
440 d.addCallback(_check)
442 self.failUnless('Tahoe Provisioning Tool' in res)
443 self.failUnless("Share space consumed: 167.01TB" in res)
445 fields = {'filled': True,
446 "num_users": int(50e6),
447 "files_per_user": 1000,
448 "space_per_user": int(5e9),
449 "sharing_ratio": 1.0,
450 "encoding_parameters": "25-of-100-50",
451 "num_servers": 30000,
452 "ownership_mode": "E",
453 "drive_failure_model": "U",
455 "download_rate": 1000,
460 return self.POST("/provisioning/", **fields)
461 d.addCallback(_check2)
463 self.failUnless("Share space consumed: huge!" in res)
464 fields = {'filled': True}
465 return self.POST("/provisioning/", **fields)
466 d.addCallback(_check3)
468 self.failUnless("Share space consumed:" in res)
469 d.addCallback(_check4)
472 def test_reliability_tool(self):
474 from allmydata import reliability
475 _hush_pyflakes = reliability
477 raise unittest.SkipTest("reliability tool requires Numeric")
479 d = self.GET("/reliability/")
481 self.failUnless('Tahoe Reliability Tool' in res)
482 fields = {'drive_lifetime': "8Y",
487 "check_period": "1M",
488 "report_period": "3M",
491 return self.POST("/reliability/", **fields)
493 d.addCallback(_check)
495 self.failUnless('Tahoe Reliability Tool' in res)
496 r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
497 self.failUnless(re.search(r, res), res)
498 d.addCallback(_check2)
501 def test_status(self):
502 dl_num = self.s.list_all_download_statuses()[0].get_counter()
503 ul_num = self.s.list_all_upload_statuses()[0].get_counter()
504 mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
505 pub_num = self.s.list_all_publish_statuses()[0].get_counter()
506 ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
507 d = self.GET("/status", followRedirect=True)
509 self.failUnless('Upload and Download Status' in res, res)
510 self.failUnless('"down-%d"' % dl_num in res, res)
511 self.failUnless('"up-%d"' % ul_num in res, res)
512 self.failUnless('"mapupdate-%d"' % mu_num in res, res)
513 self.failUnless('"publish-%d"' % pub_num in res, res)
514 self.failUnless('"retrieve-%d"' % ret_num in res, res)
515 d.addCallback(_check)
516 d.addCallback(lambda res: self.GET("/status/?t=json"))
517 def _check_json(res):
518 data = simplejson.loads(res)
519 self.failUnless(isinstance(data, dict))
520 active = data["active"]
521 # TODO: test more. We need a way to fake an active operation
523 d.addCallback(_check_json)
525 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
527 self.failUnless("File Download Status" in res, res)
528 d.addCallback(_check_dl)
529 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
531 self.failUnless("File Upload Status" in res, res)
532 d.addCallback(_check_ul)
533 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
534 def _check_mapupdate(res):
535 self.failUnless("Mutable File Servermap Update Status" in res, res)
536 d.addCallback(_check_mapupdate)
537 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
538 def _check_publish(res):
539 self.failUnless("Mutable File Publish Status" in res, res)
540 d.addCallback(_check_publish)
541 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
542 def _check_retrieve(res):
543 self.failUnless("Mutable File Retrieve Status" in res, res)
544 d.addCallback(_check_retrieve)
548 def test_status_numbers(self):
549 drrm = status.DownloadResultsRendererMixin()
550 self.failUnlessEqual(drrm.render_time(None, None), "")
551 self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
552 self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
553 self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
554 self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
555 self.failUnlessEqual(drrm.render_rate(None, None), "")
556 self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
557 self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
558 self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
560 urrm = status.UploadResultsRendererMixin()
561 self.failUnlessEqual(urrm.render_time(None, None), "")
562 self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
563 self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
564 self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
565 self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
566 self.failUnlessEqual(urrm.render_rate(None, None), "")
567 self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
568 self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
569 self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
571 def test_GET_FILEURL(self):
572 d = self.GET(self.public_url + "/foo/bar.txt")
573 d.addCallback(self.failUnlessIsBarDotTxt)
576 def test_GET_FILEURL_range(self):
577 headers = {"range": "bytes=1-10"}
578 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
579 return_response=True)
580 def _got((res, status, headers)):
581 self.failUnlessEqual(int(status), 206)
582 self.failUnless(headers.has_key("content-range"))
583 self.failUnlessEqual(headers["content-range"][0],
584 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
585 self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
589 def test_GET_FILEURL_partial_range(self):
590 headers = {"range": "bytes=5-"}
591 length = len(self.BAR_CONTENTS)
592 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
593 return_response=True)
594 def _got((res, status, headers)):
595 self.failUnlessEqual(int(status), 206)
596 self.failUnless(headers.has_key("content-range"))
597 self.failUnlessEqual(headers["content-range"][0],
598 "bytes 5-%d/%d" % (length-1, length))
599 self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
603 def test_HEAD_FILEURL_range(self):
604 headers = {"range": "bytes=1-10"}
605 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
606 return_response=True)
607 def _got((res, status, headers)):
608 self.failUnlessEqual(res, "")
609 self.failUnlessEqual(int(status), 206)
610 self.failUnless(headers.has_key("content-range"))
611 self.failUnlessEqual(headers["content-range"][0],
612 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
616 def test_HEAD_FILEURL_partial_range(self):
617 headers = {"range": "bytes=5-"}
618 length = len(self.BAR_CONTENTS)
619 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
620 return_response=True)
621 def _got((res, status, headers)):
622 self.failUnlessEqual(int(status), 206)
623 self.failUnless(headers.has_key("content-range"))
624 self.failUnlessEqual(headers["content-range"][0],
625 "bytes 5-%d/%d" % (length-1, length))
629 def test_GET_FILEURL_range_bad(self):
630 headers = {"range": "BOGUS=fizbop-quarnak"}
631 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
633 "Syntactically invalid http range header",
634 self.GET, self.public_url + "/foo/bar.txt",
638 def test_HEAD_FILEURL(self):
639 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
640 def _got((res, status, headers)):
641 self.failUnlessEqual(res, "")
642 self.failUnlessEqual(headers["content-length"][0],
643 str(len(self.BAR_CONTENTS)))
644 self.failUnlessEqual(headers["content-type"], ["text/plain"])
648 def test_GET_FILEURL_named(self):
649 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
650 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
651 d = self.GET(base + "/@@name=/blah.txt")
652 d.addCallback(self.failUnlessIsBarDotTxt)
653 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
654 d.addCallback(self.failUnlessIsBarDotTxt)
655 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
656 d.addCallback(self.failUnlessIsBarDotTxt)
657 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
658 d.addCallback(self.failUnlessIsBarDotTxt)
659 save_url = base + "?save=true&filename=blah.txt"
660 d.addCallback(lambda res: self.GET(save_url))
661 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
662 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
663 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
664 u_url = base + "?save=true&filename=" + u_fn_e
665 d.addCallback(lambda res: self.GET(u_url))
666 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
669 def test_PUT_FILEURL_named_bad(self):
670 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
671 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
673 "/file can only be used with GET or HEAD",
674 self.PUT, base + "/@@name=/blah.txt", "")
677 def test_GET_DIRURL_named_bad(self):
678 base = "/file/%s" % urllib.quote(self._foo_uri)
679 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
682 self.GET, base + "/@@name=/blah.txt")
685 def test_GET_slash_file_bad(self):
686 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
688 "/file must be followed by a file-cap and a name",
692 def test_GET_unhandled_URI_named(self):
693 contents, n, newuri = self.makefile(12)
694 verifier_cap = n.get_verify_cap().to_string()
695 base = "/file/%s" % urllib.quote(verifier_cap)
696 # client.create_node_from_uri() can't handle verify-caps
697 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
699 "is not a valid file- or directory- cap",
703 def test_GET_unhandled_URI(self):
704 contents, n, newuri = self.makefile(12)
705 verifier_cap = n.get_verify_cap().to_string()
706 base = "/uri/%s" % urllib.quote(verifier_cap)
707 # client.create_node_from_uri() can't handle verify-caps
708 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
710 "is not a valid file- or directory- cap",
714 def test_GET_FILE_URI(self):
715 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
717 d.addCallback(self.failUnlessIsBarDotTxt)
720 def test_GET_FILE_URI_badchild(self):
721 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
722 errmsg = "Files have no children, certainly not named 'boguschild'"
723 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
724 "400 Bad Request", errmsg,
728 def test_PUT_FILE_URI_badchild(self):
729 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
730 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
731 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
732 "400 Bad Request", errmsg,
736 def test_GET_FILEURL_save(self):
737 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
738 # TODO: look at the headers, expect a Content-Disposition: attachment
740 d.addCallback(self.failUnlessIsBarDotTxt)
743 def test_GET_FILEURL_missing(self):
744 d = self.GET(self.public_url + "/foo/missing")
745 d.addBoth(self.should404, "test_GET_FILEURL_missing")
748 def test_PUT_NEWFILEURL(self):
749 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
750 # TODO: we lose the response code, so we can't check this
751 #self.failUnlessEqual(responsecode, 201)
752 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
753 d.addCallback(lambda res:
754 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
755 self.NEWFILE_CONTENTS))
758 def test_PUT_NEWFILEURL_range_bad(self):
759 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
760 target = self.public_url + "/foo/new.txt"
761 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
762 "501 Not Implemented",
763 "Content-Range in PUT not yet supported",
764 # (and certainly not for immutable files)
765 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
767 d.addCallback(lambda res:
768 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
771 def test_PUT_NEWFILEURL_mutable(self):
772 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
773 self.NEWFILE_CONTENTS)
774 # TODO: we lose the response code, so we can't check this
775 #self.failUnlessEqual(responsecode, 201)
777 u = uri.from_string_mutable_filenode(res)
778 self.failUnless(u.is_mutable())
779 self.failIf(u.is_readonly())
781 d.addCallback(_check_uri)
782 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
783 d.addCallback(lambda res:
784 self.failUnlessMutableChildContentsAre(self._foo_node,
786 self.NEWFILE_CONTENTS))
789 def test_PUT_NEWFILEURL_mutable_toobig(self):
790 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
791 "413 Request Entity Too Large",
792 "SDMF is limited to one segment, and 10001 > 10000",
794 self.public_url + "/foo/new.txt?mutable=true",
795 "b" * (self.s.MUTABLE_SIZELIMIT+1))
798 def test_PUT_NEWFILEURL_replace(self):
799 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
800 # TODO: we lose the response code, so we can't check this
801 #self.failUnlessEqual(responsecode, 200)
802 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
803 d.addCallback(lambda res:
804 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
805 self.NEWFILE_CONTENTS))
808 def test_PUT_NEWFILEURL_bad_t(self):
809 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
810 "PUT to a file: bad t=bogus",
811 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
815 def test_PUT_NEWFILEURL_no_replace(self):
816 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
817 self.NEWFILE_CONTENTS)
818 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
820 "There was already a child by that name, and you asked me "
824 def test_PUT_NEWFILEURL_mkdirs(self):
825 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
827 d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
828 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
829 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
830 d.addCallback(lambda res:
831 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
832 self.NEWFILE_CONTENTS))
835 def test_PUT_NEWFILEURL_blocked(self):
836 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
837 self.NEWFILE_CONTENTS)
838 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
840 "Unable to create directory 'blockingfile': a file was in the way")
843 def test_DELETE_FILEURL(self):
844 d = self.DELETE(self.public_url + "/foo/bar.txt")
845 d.addCallback(lambda res:
846 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
849 def test_DELETE_FILEURL_missing(self):
850 d = self.DELETE(self.public_url + "/foo/missing")
851 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
854 def test_DELETE_FILEURL_missing2(self):
855 d = self.DELETE(self.public_url + "/missing/missing")
856 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
859 def test_GET_FILEURL_json(self):
860 # twisted.web.http.parse_qs ignores any query args without an '=', so
861 # I can't do "GET /path?json", I have to do "GET /path/t=json"
862 # instead. This may make it tricky to emulate the S3 interface
864 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
865 d.addCallback(self.failUnlessIsBarJSON)
868 def test_GET_FILEURL_json_missing(self):
869 d = self.GET(self.public_url + "/foo/missing?json")
870 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
873 def test_GET_FILEURL_uri(self):
874 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
876 self.failUnlessEqual(res, self._bar_txt_uri)
877 d.addCallback(_check)
878 d.addCallback(lambda res:
879 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
881 # for now, for files, uris and readonly-uris are the same
882 self.failUnlessEqual(res, self._bar_txt_uri)
883 d.addCallback(_check2)
886 def test_GET_FILEURL_badtype(self):
887 d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
890 self.public_url + "/foo/bar.txt?t=bogus")
893 def test_GET_FILEURL_uri_missing(self):
894 d = self.GET(self.public_url + "/foo/missing?t=uri")
895 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
898 def test_GET_DIRURL(self):
899 # the addSlash means we get a redirect here
900 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
902 d = self.GET(self.public_url + "/foo", followRedirect=True)
904 self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
906 # the FILE reference points to a URI, but it should end in bar.txt
907 bar_url = ("%s/file/%s/@@named=/bar.txt" %
908 (ROOT, urllib.quote(self._bar_txt_uri)))
909 get_bar = "".join([r'<td>',
910 r'<a href="%s">bar.txt</a>' % bar_url,
913 r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
915 self.failUnless(re.search(get_bar, res), res)
916 for line in res.split("\n"):
917 # find the line that contains the delete button for bar.txt
918 if ("form action" in line and
919 'value="delete"' in line and
920 'value="bar.txt"' in line):
921 # the form target should use a relative URL
922 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
923 self.failUnless(('action="%s"' % foo_url) in line, line)
924 # and the when_done= should too
925 #done_url = urllib.quote(???)
926 #self.failUnless(('name="when_done" value="%s"' % done_url)
930 self.fail("unable to find delete-bar.txt line", res)
932 # the DIR reference just points to a URI
933 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
934 get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
935 + r'\s+<td>DIR</td>')
936 self.failUnless(re.search(get_sub, res), res)
937 d.addCallback(_check)
939 # look at a directory which is readonly
940 d.addCallback(lambda res:
941 self.GET(self.public_url + "/reedownlee", followRedirect=True))
943 self.failUnless("(readonly)" in res, res)
944 self.failIf("Upload a file" in res, res)
945 d.addCallback(_check2)
947 # and at a directory that contains a readonly directory
948 d.addCallback(lambda res:
949 self.GET(self.public_url, followRedirect=True))
951 self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
952 '</td>\s+<td>DIR-RO</td>', res))
953 d.addCallback(_check3)
957 def test_GET_DIRURL_badtype(self):
958 d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
962 self.public_url + "/foo?t=bogus")
965 def test_GET_DIRURL_json(self):
966 d = self.GET(self.public_url + "/foo?t=json")
967 d.addCallback(self.failUnlessIsFooJSON)
971 def test_POST_DIRURL_manifest_no_ophandle(self):
972 d = self.shouldFail2(error.Error,
973 "test_POST_DIRURL_manifest_no_ophandle",
975 "slow operation requires ophandle=",
976 self.POST, self.public_url, t="start-manifest")
979 def test_POST_DIRURL_manifest(self):
980 d = defer.succeed(None)
981 def getman(ignored, output):
982 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
984 d.addCallback(self.wait_for_operation, "125")
985 d.addCallback(self.get_operation_results, "125", output)
987 d.addCallback(getman, None)
988 def _got_html(manifest):
989 self.failUnless("Manifest of SI=" in manifest)
990 self.failUnless("<td>sub</td>" in manifest)
991 self.failUnless(self._sub_uri in manifest)
992 self.failUnless("<td>sub/baz.txt</td>" in manifest)
993 d.addCallback(_got_html)
995 # both t=status and unadorned GET should be identical
996 d.addCallback(lambda res: self.GET("/operations/125"))
997 d.addCallback(_got_html)
999 d.addCallback(getman, "html")
1000 d.addCallback(_got_html)
1001 d.addCallback(getman, "text")
1002 def _got_text(manifest):
1003 self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1004 self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1005 d.addCallback(_got_text)
1006 d.addCallback(getman, "JSON")
1008 data = res["manifest"]
1010 for (path_list, cap) in data:
1011 got[tuple(path_list)] = cap
1012 self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1013 self.failUnless((u"sub",u"baz.txt") in got)
1014 self.failUnless("finished" in res)
1015 self.failUnless("origin" in res)
1016 self.failUnless("storage-index" in res)
1017 self.failUnless("verifycaps" in res)
1018 self.failUnless("stats" in res)
1019 d.addCallback(_got_json)
1022 def test_POST_DIRURL_deepsize_no_ophandle(self):
1023 d = self.shouldFail2(error.Error,
1024 "test_POST_DIRURL_deepsize_no_ophandle",
1026 "slow operation requires ophandle=",
1027 self.POST, self.public_url, t="start-deep-size")
1030 def test_POST_DIRURL_deepsize(self):
1031 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1032 followRedirect=True)
1033 d.addCallback(self.wait_for_operation, "126")
1034 d.addCallback(self.get_operation_results, "126", "json")
1035 def _got_json(data):
1036 self.failUnlessEqual(data["finished"], True)
1038 self.failUnless(size > 1000)
1039 d.addCallback(_got_json)
1040 d.addCallback(self.get_operation_results, "126", "text")
1042 mo = re.search(r'^size: (\d+)$', res, re.M)
1043 self.failUnless(mo, res)
1044 size = int(mo.group(1))
1045 # with directories, the size varies.
1046 self.failUnless(size > 1000)
1047 d.addCallback(_got_text)
1050 def test_POST_DIRURL_deepstats_no_ophandle(self):
1051 d = self.shouldFail2(error.Error,
1052 "test_POST_DIRURL_deepstats_no_ophandle",
1054 "slow operation requires ophandle=",
1055 self.POST, self.public_url, t="start-deep-stats")
1058 def test_POST_DIRURL_deepstats(self):
1059 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1060 followRedirect=True)
1061 d.addCallback(self.wait_for_operation, "127")
1062 d.addCallback(self.get_operation_results, "127", "json")
1063 def _got_json(stats):
1064 expected = {"count-immutable-files": 3,
1065 "count-mutable-files": 0,
1066 "count-literal-files": 0,
1068 "count-directories": 3,
1069 "size-immutable-files": 57,
1070 "size-literal-files": 0,
1071 #"size-directories": 1912, # varies
1072 #"largest-directory": 1590,
1073 "largest-directory-children": 5,
1074 "largest-immutable-file": 19,
1076 for k,v in expected.iteritems():
1077 self.failUnlessEqual(stats[k], v,
1078 "stats[%s] was %s, not %s" %
1080 self.failUnlessEqual(stats["size-files-histogram"],
1082 d.addCallback(_got_json)
1085 def test_POST_DIRURL_stream_manifest(self):
1086 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1088 self.failUnless(res.endswith("\n"))
1089 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1090 self.failUnlessEqual(len(units), 7)
1091 self.failUnlessEqual(units[-1]["type"], "stats")
1093 self.failUnlessEqual(first["path"], [])
1094 self.failUnlessEqual(first["cap"], self._foo_uri)
1095 self.failUnlessEqual(first["type"], "directory")
1096 baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1097 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1098 self.failIfEqual(baz["storage-index"], None)
1099 self.failIfEqual(baz["verifycap"], None)
1100 self.failIfEqual(baz["repaircap"], None)
1102 d.addCallback(_check)
1105 def test_GET_DIRURL_uri(self):
1106 d = self.GET(self.public_url + "/foo?t=uri")
1108 self.failUnlessEqual(res, self._foo_uri)
1109 d.addCallback(_check)
1112 def test_GET_DIRURL_readonly_uri(self):
1113 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1115 self.failUnlessEqual(res, self._foo_readonly_uri)
1116 d.addCallback(_check)
1119 def test_PUT_NEWDIRURL(self):
1120 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1121 d.addCallback(lambda res:
1122 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1123 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1124 d.addCallback(self.failUnlessNodeKeysAre, [])
1127 def test_PUT_NEWDIRURL_exists(self):
1128 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1129 d.addCallback(lambda res:
1130 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1131 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1132 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1135 def test_PUT_NEWDIRURL_blocked(self):
1136 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1137 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1139 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1140 d.addCallback(lambda res:
1141 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1142 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1143 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1146 def test_PUT_NEWDIRURL_mkdir_p(self):
1147 d = defer.succeed(None)
1148 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1149 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1150 d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1151 def mkdir_p(mkpnode):
1152 url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1154 def made_subsub(ssuri):
1155 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1156 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1158 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1160 d.addCallback(made_subsub)
1162 d.addCallback(mkdir_p)
1165 def test_PUT_NEWDIRURL_mkdirs(self):
1166 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1167 d.addCallback(lambda res:
1168 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1169 d.addCallback(lambda res:
1170 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1171 d.addCallback(lambda res:
1172 self._foo_node.get_child_at_path(u"subdir/newdir"))
1173 d.addCallback(self.failUnlessNodeKeysAre, [])
1176 def test_DELETE_DIRURL(self):
1177 d = self.DELETE(self.public_url + "/foo")
1178 d.addCallback(lambda res:
1179 self.failIfNodeHasChild(self.public_root, u"foo"))
1182 def test_DELETE_DIRURL_missing(self):
1183 d = self.DELETE(self.public_url + "/foo/missing")
1184 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1185 d.addCallback(lambda res:
1186 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1189 def test_DELETE_DIRURL_missing2(self):
1190 d = self.DELETE(self.public_url + "/missing")
1191 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1194 def dump_root(self):
1196 w = webish.DirnodeWalkerMixin()
1197 def visitor(childpath, childnode, metadata):
1199 d = w.walk(self.public_root, visitor)
1202 def failUnlessNodeKeysAre(self, node, expected_keys):
1203 for k in expected_keys:
1204 assert isinstance(k, unicode)
1206 def _check(children):
1207 self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1208 d.addCallback(_check)
1210 def failUnlessNodeHasChild(self, node, name):
1211 assert isinstance(name, unicode)
1213 def _check(children):
1214 self.failUnless(name in children)
1215 d.addCallback(_check)
1217 def failIfNodeHasChild(self, node, name):
1218 assert isinstance(name, unicode)
1220 def _check(children):
1221 self.failIf(name in children)
1222 d.addCallback(_check)
1225 def failUnlessChildContentsAre(self, node, name, expected_contents):
1226 assert isinstance(name, unicode)
1227 d = node.get_child_at_path(name)
1228 d.addCallback(lambda node: node.download_to_data())
1229 def _check(contents):
1230 self.failUnlessEqual(contents, expected_contents)
1231 d.addCallback(_check)
1234 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1235 assert isinstance(name, unicode)
1236 d = node.get_child_at_path(name)
1237 d.addCallback(lambda node: node.download_best_version())
1238 def _check(contents):
1239 self.failUnlessEqual(contents, expected_contents)
1240 d.addCallback(_check)
1243 def failUnlessChildURIIs(self, node, name, expected_uri):
1244 assert isinstance(name, unicode)
1245 d = node.get_child_at_path(name)
1247 self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1248 d.addCallback(_check)
1251 def failUnlessURIMatchesChild(self, got_uri, node, name):
1252 assert isinstance(name, unicode)
1253 d = node.get_child_at_path(name)
1255 self.failUnlessEqual(got_uri.strip(), child.get_uri())
1256 d.addCallback(_check)
1259 def failUnlessCHKURIHasContents(self, got_uri, contents):
1260 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1262 def test_POST_upload(self):
1263 d = self.POST(self.public_url + "/foo", t="upload",
1264 file=("new.txt", self.NEWFILE_CONTENTS))
1266 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1267 d.addCallback(lambda res:
1268 self.failUnlessChildContentsAre(fn, u"new.txt",
1269 self.NEWFILE_CONTENTS))
1272 def test_POST_upload_unicode(self):
1273 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1274 d = self.POST(self.public_url + "/foo", t="upload",
1275 file=(filename, self.NEWFILE_CONTENTS))
1277 d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1278 d.addCallback(lambda res:
1279 self.failUnlessChildContentsAre(fn, filename,
1280 self.NEWFILE_CONTENTS))
1281 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1282 d.addCallback(lambda res: self.GET(target_url))
1283 d.addCallback(lambda contents: self.failUnlessEqual(contents,
1284 self.NEWFILE_CONTENTS,
1288 def test_POST_upload_unicode_named(self):
1289 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1290 d = self.POST(self.public_url + "/foo", t="upload",
1292 file=("overridden", self.NEWFILE_CONTENTS))
1294 d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1295 d.addCallback(lambda res:
1296 self.failUnlessChildContentsAre(fn, filename,
1297 self.NEWFILE_CONTENTS))
1298 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1299 d.addCallback(lambda res: self.GET(target_url))
1300 d.addCallback(lambda contents: self.failUnlessEqual(contents,
1301 self.NEWFILE_CONTENTS,
1305 def test_POST_upload_no_link(self):
1306 d = self.POST("/uri", t="upload",
1307 file=("new.txt", self.NEWFILE_CONTENTS))
1308 def _check_upload_results(page):
1309 # this should be a page which describes the results of the upload
1310 # that just finished.
1311 self.failUnless("Upload Results:" in page)
1312 self.failUnless("URI:" in page)
1313 uri_re = re.compile("URI: <tt><span>(.*)</span>")
1314 mo = uri_re.search(page)
1315 self.failUnless(mo, page)
1316 new_uri = mo.group(1)
1318 d.addCallback(_check_upload_results)
1319 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1322 def test_POST_upload_no_link_whendone(self):
1323 d = self.POST("/uri", t="upload", when_done="/",
1324 file=("new.txt", self.NEWFILE_CONTENTS))
1325 d.addBoth(self.shouldRedirect, "/")
1328 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1329 d = defer.maybeDeferred(callable, *args, **kwargs)
1331 if isinstance(res, failure.Failure):
1332 res.trap(error.PageRedirect)
1333 statuscode = res.value.status
1334 target = res.value.location
1335 return checker(statuscode, target)
1336 self.fail("%s: callable was supposed to redirect, not return '%s'"
1341 def test_POST_upload_no_link_whendone_results(self):
1342 def check(statuscode, target):
1343 self.failUnlessEqual(statuscode, str(http.FOUND))
1344 self.failUnless(target.startswith(self.webish_url), target)
1345 return client.getPage(target, method="GET")
1346 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1348 self.POST, "/uri", t="upload",
1349 when_done="/uri/%(uri)s",
1350 file=("new.txt", self.NEWFILE_CONTENTS))
1351 d.addCallback(lambda res:
1352 self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1355 def test_POST_upload_no_link_mutable(self):
1356 d = self.POST("/uri", t="upload", mutable="true",
1357 file=("new.txt", self.NEWFILE_CONTENTS))
1358 def _check(new_uri):
1359 new_uri = new_uri.strip()
1360 self.new_uri = new_uri
1362 self.failUnless(IMutableFileURI.providedBy(u))
1363 self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1364 n = self.s.create_node_from_uri(new_uri)
1365 return n.download_best_version()
1366 d.addCallback(_check)
1368 self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1369 return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1370 d.addCallback(_check2)
1372 self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1373 return self.GET("/file/%s" % urllib.quote(self.new_uri))
1374 d.addCallback(_check3)
1376 self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1377 d.addCallback(_check4)
1380 def test_POST_upload_no_link_mutable_toobig(self):
1381 d = self.shouldFail2(error.Error,
1382 "test_POST_upload_no_link_mutable_toobig",
1383 "413 Request Entity Too Large",
1384 "SDMF is limited to one segment, and 10001 > 10000",
1386 "/uri", t="upload", mutable="true",
1388 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1391 def test_POST_upload_mutable(self):
1392 # this creates a mutable file
1393 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1394 file=("new.txt", self.NEWFILE_CONTENTS))
1396 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1397 d.addCallback(lambda res:
1398 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1399 self.NEWFILE_CONTENTS))
1400 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1402 self.failUnless(IMutableFileNode.providedBy(newnode))
1403 self.failUnless(newnode.is_mutable())
1404 self.failIf(newnode.is_readonly())
1405 self._mutable_node = newnode
1406 self._mutable_uri = newnode.get_uri()
1409 # now upload it again and make sure that the URI doesn't change
1410 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1411 d.addCallback(lambda res:
1412 self.POST(self.public_url + "/foo", t="upload",
1414 file=("new.txt", NEWER_CONTENTS)))
1415 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1416 d.addCallback(lambda res:
1417 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1419 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1421 self.failUnless(IMutableFileNode.providedBy(newnode))
1422 self.failUnless(newnode.is_mutable())
1423 self.failIf(newnode.is_readonly())
1424 self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1425 d.addCallback(_got2)
1427 # upload a second time, using PUT instead of POST
1428 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1429 d.addCallback(lambda res:
1430 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1431 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1432 d.addCallback(lambda res:
1433 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1436 # finally list the directory, since mutable files are displayed
1437 # slightly differently
1439 d.addCallback(lambda res:
1440 self.GET(self.public_url + "/foo/",
1441 followRedirect=True))
1442 def _check_page(res):
1443 # TODO: assert more about the contents
1444 self.failUnless("SSK" in res)
1446 d.addCallback(_check_page)
1448 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1450 self.failUnless(IMutableFileNode.providedBy(newnode))
1451 self.failUnless(newnode.is_mutable())
1452 self.failIf(newnode.is_readonly())
1453 self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1454 d.addCallback(_got3)
1456 # look at the JSON form of the enclosing directory
1457 d.addCallback(lambda res:
1458 self.GET(self.public_url + "/foo/?t=json",
1459 followRedirect=True))
1460 def _check_page_json(res):
1461 parsed = simplejson.loads(res)
1462 self.failUnlessEqual(parsed[0], "dirnode")
1463 children = dict( [(unicode(name),value)
1465 in parsed[1]["children"].iteritems()] )
1466 self.failUnless("new.txt" in children)
1467 new_json = children["new.txt"]
1468 self.failUnlessEqual(new_json[0], "filenode")
1469 self.failUnless(new_json[1]["mutable"])
1470 self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1471 ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1472 self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1473 d.addCallback(_check_page_json)
1475 # and the JSON form of the file
1476 d.addCallback(lambda res:
1477 self.GET(self.public_url + "/foo/new.txt?t=json"))
1478 def _check_file_json(res):
1479 parsed = simplejson.loads(res)
1480 self.failUnlessEqual(parsed[0], "filenode")
1481 self.failUnless(parsed[1]["mutable"])
1482 self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1483 ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1484 self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1485 d.addCallback(_check_file_json)
1487 # and look at t=uri and t=readonly-uri
1488 d.addCallback(lambda res:
1489 self.GET(self.public_url + "/foo/new.txt?t=uri"))
1490 d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1491 d.addCallback(lambda res:
1492 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1493 def _check_ro_uri(res):
1494 ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1495 self.failUnlessEqual(res, ro_uri)
1496 d.addCallback(_check_ro_uri)
1498 # make sure we can get to it from /uri/URI
1499 d.addCallback(lambda res:
1500 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1501 d.addCallback(lambda res:
1502 self.failUnlessEqual(res, NEW2_CONTENTS))
1504 # and that HEAD computes the size correctly
1505 d.addCallback(lambda res:
1506 self.HEAD(self.public_url + "/foo/new.txt",
1507 return_response=True))
1508 def _got_headers((res, status, headers)):
1509 self.failUnlessEqual(res, "")
1510 self.failUnlessEqual(headers["content-length"][0],
1511 str(len(NEW2_CONTENTS)))
1512 self.failUnlessEqual(headers["content-type"], ["text/plain"])
1513 d.addCallback(_got_headers)
1515 # make sure that size errors are displayed correctly for overwrite
1516 d.addCallback(lambda res:
1517 self.shouldFail2(error.Error,
1518 "test_POST_upload_mutable-toobig",
1519 "413 Request Entity Too Large",
1520 "SDMF is limited to one segment, and 10001 > 10000",
1522 self.public_url + "/foo", t="upload",
1525 "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1528 d.addErrback(self.dump_error)
1531 def test_POST_upload_mutable_toobig(self):
1532 d = self.shouldFail2(error.Error,
1533 "test_POST_upload_no_link_mutable_toobig",
1534 "413 Request Entity Too Large",
1535 "SDMF is limited to one segment, and 10001 > 10000",
1537 self.public_url + "/foo",
1538 t="upload", mutable="true",
1540 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1543 def dump_error(self, f):
1544 # if the web server returns an error code (like 400 Bad Request),
1545 # web.client.getPage puts the HTTP response body into the .response
1546 # attribute of the exception object that it gives back. It does not
1547 # appear in the Failure's repr(), so the ERROR that trial displays
1548 # will be rather terse and unhelpful. addErrback this method to the
1549 # end of your chain to get more information out of these errors.
1550 if f.check(error.Error):
1551 print "web.error.Error:"
1553 print f.value.response
1556 def test_POST_upload_replace(self):
1557 d = self.POST(self.public_url + "/foo", t="upload",
1558 file=("bar.txt", self.NEWFILE_CONTENTS))
1560 d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1561 d.addCallback(lambda res:
1562 self.failUnlessChildContentsAre(fn, u"bar.txt",
1563 self.NEWFILE_CONTENTS))
1566 def test_POST_upload_no_replace_ok(self):
1567 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1568 file=("new.txt", self.NEWFILE_CONTENTS))
1569 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1570 d.addCallback(lambda res: self.failUnlessEqual(res,
1571 self.NEWFILE_CONTENTS))
1574 def test_POST_upload_no_replace_queryarg(self):
1575 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1576 file=("bar.txt", self.NEWFILE_CONTENTS))
1577 d.addBoth(self.shouldFail, error.Error,
1578 "POST_upload_no_replace_queryarg",
1580 "There was already a child by that name, and you asked me "
1581 "to not replace it")
1582 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1583 d.addCallback(self.failUnlessIsBarDotTxt)
1586 def test_POST_upload_no_replace_field(self):
1587 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1588 file=("bar.txt", self.NEWFILE_CONTENTS))
1589 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1591 "There was already a child by that name, and you asked me "
1592 "to not replace it")
1593 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1594 d.addCallback(self.failUnlessIsBarDotTxt)
1597 def test_POST_upload_whendone(self):
1598 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1599 file=("new.txt", self.NEWFILE_CONTENTS))
1600 d.addBoth(self.shouldRedirect, "/THERE")
1602 d.addCallback(lambda res:
1603 self.failUnlessChildContentsAre(fn, u"new.txt",
1604 self.NEWFILE_CONTENTS))
1607 def test_POST_upload_named(self):
1609 d = self.POST(self.public_url + "/foo", t="upload",
1610 name="new.txt", file=self.NEWFILE_CONTENTS)
1611 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1612 d.addCallback(lambda res:
1613 self.failUnlessChildContentsAre(fn, u"new.txt",
1614 self.NEWFILE_CONTENTS))
1617 def test_POST_upload_named_badfilename(self):
1618 d = self.POST(self.public_url + "/foo", t="upload",
1619 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1620 d.addBoth(self.shouldFail, error.Error,
1621 "test_POST_upload_named_badfilename",
1623 "name= may not contain a slash",
1625 # make sure that nothing was added
1626 d.addCallback(lambda res:
1627 self.failUnlessNodeKeysAre(self._foo_node,
1628 [u"bar.txt", u"blockingfile",
1629 u"empty", u"n\u00fc.txt",
1633 def test_POST_FILEURL_check(self):
1634 bar_url = self.public_url + "/foo/bar.txt"
1635 d = self.POST(bar_url, t="check")
1637 self.failUnless("Healthy :" in res)
1638 d.addCallback(_check)
1639 redir_url = "http://allmydata.org/TARGET"
1640 def _check2(statuscode, target):
1641 self.failUnlessEqual(statuscode, str(http.FOUND))
1642 self.failUnlessEqual(target, redir_url)
1643 d.addCallback(lambda res:
1644 self.shouldRedirect2("test_POST_FILEURL_check",
1648 when_done=redir_url))
1649 d.addCallback(lambda res:
1650 self.POST(bar_url, t="check", return_to=redir_url))
1652 self.failUnless("Healthy :" in res)
1653 self.failUnless("Return to parent directory" in res)
1654 self.failUnless(redir_url in res)
1655 d.addCallback(_check3)
1657 d.addCallback(lambda res:
1658 self.POST(bar_url, t="check", output="JSON"))
1659 def _check_json(res):
1660 data = simplejson.loads(res)
1661 self.failUnless("storage-index" in data)
1662 self.failUnless(data["results"]["healthy"])
1663 d.addCallback(_check_json)
1667 def test_POST_FILEURL_check_and_repair(self):
1668 bar_url = self.public_url + "/foo/bar.txt"
1669 d = self.POST(bar_url, t="check", repair="true")
1671 self.failUnless("Healthy :" in res)
1672 d.addCallback(_check)
1673 redir_url = "http://allmydata.org/TARGET"
1674 def _check2(statuscode, target):
1675 self.failUnlessEqual(statuscode, str(http.FOUND))
1676 self.failUnlessEqual(target, redir_url)
1677 d.addCallback(lambda res:
1678 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1681 t="check", repair="true",
1682 when_done=redir_url))
1683 d.addCallback(lambda res:
1684 self.POST(bar_url, t="check", return_to=redir_url))
1686 self.failUnless("Healthy :" in res)
1687 self.failUnless("Return to parent directory" in res)
1688 self.failUnless(redir_url in res)
1689 d.addCallback(_check3)
1692 def test_POST_DIRURL_check(self):
1693 foo_url = self.public_url + "/foo/"
1694 d = self.POST(foo_url, t="check")
1696 self.failUnless("Healthy :" in res, res)
1697 d.addCallback(_check)
1698 redir_url = "http://allmydata.org/TARGET"
1699 def _check2(statuscode, target):
1700 self.failUnlessEqual(statuscode, str(http.FOUND))
1701 self.failUnlessEqual(target, redir_url)
1702 d.addCallback(lambda res:
1703 self.shouldRedirect2("test_POST_DIRURL_check",
1707 when_done=redir_url))
1708 d.addCallback(lambda res:
1709 self.POST(foo_url, t="check", return_to=redir_url))
1711 self.failUnless("Healthy :" in res, res)
1712 self.failUnless("Return to parent directory" in res)
1713 self.failUnless(redir_url in res)
1714 d.addCallback(_check3)
1716 d.addCallback(lambda res:
1717 self.POST(foo_url, t="check", output="JSON"))
1718 def _check_json(res):
1719 data = simplejson.loads(res)
1720 self.failUnless("storage-index" in data)
1721 self.failUnless(data["results"]["healthy"])
1722 d.addCallback(_check_json)
1726 def test_POST_DIRURL_check_and_repair(self):
1727 foo_url = self.public_url + "/foo/"
1728 d = self.POST(foo_url, t="check", repair="true")
1730 self.failUnless("Healthy :" in res, res)
1731 d.addCallback(_check)
1732 redir_url = "http://allmydata.org/TARGET"
1733 def _check2(statuscode, target):
1734 self.failUnlessEqual(statuscode, str(http.FOUND))
1735 self.failUnlessEqual(target, redir_url)
1736 d.addCallback(lambda res:
1737 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1740 t="check", repair="true",
1741 when_done=redir_url))
1742 d.addCallback(lambda res:
1743 self.POST(foo_url, t="check", return_to=redir_url))
1745 self.failUnless("Healthy :" in res)
1746 self.failUnless("Return to parent directory" in res)
1747 self.failUnless(redir_url in res)
1748 d.addCallback(_check3)
1751 def wait_for_operation(self, ignored, ophandle):
1752 url = "/operations/" + ophandle
1753 url += "?t=status&output=JSON"
1756 data = simplejson.loads(res)
1757 if not data["finished"]:
1758 d = self.stall(delay=1.0)
1759 d.addCallback(self.wait_for_operation, ophandle)
1765 def get_operation_results(self, ignored, ophandle, output=None):
1766 url = "/operations/" + ophandle
1769 url += "&output=" + output
1772 if output and output.lower() == "json":
1773 return simplejson.loads(res)
1778 def test_POST_DIRURL_deepcheck_no_ophandle(self):
1779 d = self.shouldFail2(error.Error,
1780 "test_POST_DIRURL_deepcheck_no_ophandle",
1782 "slow operation requires ophandle=",
1783 self.POST, self.public_url, t="start-deep-check")
1786 def test_POST_DIRURL_deepcheck(self):
1787 def _check_redirect(statuscode, target):
1788 self.failUnlessEqual(statuscode, str(http.FOUND))
1789 self.failUnless(target.endswith("/operations/123"))
1790 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1791 self.POST, self.public_url,
1792 t="start-deep-check", ophandle="123")
1793 d.addCallback(self.wait_for_operation, "123")
1794 def _check_json(data):
1795 self.failUnlessEqual(data["finished"], True)
1796 self.failUnlessEqual(data["count-objects-checked"], 8)
1797 self.failUnlessEqual(data["count-objects-healthy"], 8)
1798 d.addCallback(_check_json)
1799 d.addCallback(self.get_operation_results, "123", "html")
1800 def _check_html(res):
1801 self.failUnless("Objects Checked: <span>8</span>" in res)
1802 self.failUnless("Objects Healthy: <span>8</span>" in res)
1803 d.addCallback(_check_html)
1805 d.addCallback(lambda res:
1806 self.GET("/operations/123/"))
1807 d.addCallback(_check_html) # should be the same as without the slash
1809 d.addCallback(lambda res:
1810 self.shouldFail2(error.Error, "one", "404 Not Found",
1811 "No detailed results for SI bogus",
1812 self.GET, "/operations/123/bogus"))
1814 foo_si = self._foo_node.get_storage_index()
1815 foo_si_s = base32.b2a(foo_si)
1816 d.addCallback(lambda res:
1817 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1818 def _check_foo_json(res):
1819 data = simplejson.loads(res)
1820 self.failUnlessEqual(data["storage-index"], foo_si_s)
1821 self.failUnless(data["results"]["healthy"])
1822 d.addCallback(_check_foo_json)
1825 def test_POST_DIRURL_deepcheck_and_repair(self):
1826 d = self.POST(self.public_url, t="start-deep-check", repair="true",
1827 ophandle="124", output="json", followRedirect=True)
1828 d.addCallback(self.wait_for_operation, "124")
1829 def _check_json(data):
1830 self.failUnlessEqual(data["finished"], True)
1831 self.failUnlessEqual(data["count-objects-checked"], 8)
1832 self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1833 self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1834 self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1835 self.failUnlessEqual(data["count-repairs-attempted"], 0)
1836 self.failUnlessEqual(data["count-repairs-successful"], 0)
1837 self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1838 self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1839 self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1840 self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1841 d.addCallback(_check_json)
1842 d.addCallback(self.get_operation_results, "124", "html")
1843 def _check_html(res):
1844 self.failUnless("Objects Checked: <span>8</span>" in res)
1846 self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1847 self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1848 self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1850 self.failUnless("Repairs Attempted: <span>0</span>" in res)
1851 self.failUnless("Repairs Successful: <span>0</span>" in res)
1852 self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1854 self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1855 self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1856 self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1857 d.addCallback(_check_html)
1860 def test_POST_FILEURL_bad_t(self):
1861 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1862 "POST to file: bad t=bogus",
1863 self.POST, self.public_url + "/foo/bar.txt",
1867 def test_POST_mkdir(self): # return value?
1868 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1869 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1870 d.addCallback(self.failUnlessNodeKeysAre, [])
1873 def test_POST_mkdir_2(self):
1874 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1875 d.addCallback(lambda res:
1876 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1877 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1878 d.addCallback(self.failUnlessNodeKeysAre, [])
1881 def test_POST_mkdirs_2(self):
1882 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1883 d.addCallback(lambda res:
1884 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1885 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1886 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1887 d.addCallback(self.failUnlessNodeKeysAre, [])
1890 def test_POST_mkdir_no_parentdir_noredirect(self):
1891 d = self.POST("/uri?t=mkdir")
1892 def _after_mkdir(res):
1893 uri.NewDirectoryURI.init_from_string(res)
1894 d.addCallback(_after_mkdir)
1897 def test_POST_mkdir_no_parentdir_redirect(self):
1898 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1899 d.addBoth(self.shouldRedirect, None, statuscode='303')
1900 def _check_target(target):
1901 target = urllib.unquote(target)
1902 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1903 d.addCallback(_check_target)
1906 def test_POST_noparent_bad(self):
1907 d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1908 "/uri accepts only PUT, PUT?t=mkdir, "
1909 "POST?t=upload, and POST?t=mkdir",
1910 self.POST, "/uri?t=bogus")
1913 def test_welcome_page_mkdir_button(self):
1914 # Fetch the welcome page.
1916 def _after_get_welcome_page(res):
1917 MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1918 mo = MKDIR_BUTTON_RE.search(res)
1919 formaction = mo.group(1)
1921 formaname = mo.group(3)
1922 formavalue = mo.group(4)
1923 return (formaction, formt, formaname, formavalue)
1924 d.addCallback(_after_get_welcome_page)
1925 def _after_parse_form(res):
1926 (formaction, formt, formaname, formavalue) = res
1927 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1928 d.addCallback(_after_parse_form)
1929 d.addBoth(self.shouldRedirect, None, statuscode='303')
1932 def test_POST_mkdir_replace(self): # return value?
1933 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1934 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1935 d.addCallback(self.failUnlessNodeKeysAre, [])
1938 def test_POST_mkdir_no_replace_queryarg(self): # return value?
1939 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1940 d.addBoth(self.shouldFail, error.Error,
1941 "POST_mkdir_no_replace_queryarg",
1943 "There was already a child by that name, and you asked me "
1944 "to not replace it")
1945 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1946 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1949 def test_POST_mkdir_no_replace_field(self): # return value?
1950 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1952 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1954 "There was already a child by that name, and you asked me "
1955 "to not replace it")
1956 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1957 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1960 def test_POST_mkdir_whendone_field(self):
1961 d = self.POST(self.public_url + "/foo",
1962 t="mkdir", name="newdir", when_done="/THERE")
1963 d.addBoth(self.shouldRedirect, "/THERE")
1964 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1965 d.addCallback(self.failUnlessNodeKeysAre, [])
1968 def test_POST_mkdir_whendone_queryarg(self):
1969 d = self.POST(self.public_url + "/foo?when_done=/THERE",
1970 t="mkdir", name="newdir")
1971 d.addBoth(self.shouldRedirect, "/THERE")
1972 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1973 d.addCallback(self.failUnlessNodeKeysAre, [])
1976 def test_POST_bad_t(self):
1977 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1978 "POST to a directory with bad t=BOGUS",
1979 self.POST, self.public_url + "/foo", t="BOGUS")
1982 def test_POST_set_children(self):
1983 contents9, n9, newuri9 = self.makefile(9)
1984 contents10, n10, newuri10 = self.makefile(10)
1985 contents11, n11, newuri11 = self.makefile(11)
1988 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1991 "ctime": 1002777696.7564139,
1992 "mtime": 1002777696.7564139
1995 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1998 "ctime": 1002777696.7564139,
1999 "mtime": 1002777696.7564139
2002 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2005 "ctime": 1002777696.7564139,
2006 "mtime": 1002777696.7564139
2009 }""" % (newuri9, newuri10, newuri11)
2011 url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2013 d = client.getPage(url, method="POST", postdata=reqbody)
2015 self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2016 self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2017 self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2019 d.addCallback(_then)
2020 d.addErrback(self.dump_error)
2023 def test_POST_put_uri(self):
2024 contents, n, newuri = self.makefile(8)
2025 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2026 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2027 d.addCallback(lambda res:
2028 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2032 def test_POST_put_uri_replace(self):
2033 contents, n, newuri = self.makefile(8)
2034 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2035 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2036 d.addCallback(lambda res:
2037 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2041 def test_POST_put_uri_no_replace_queryarg(self):
2042 contents, n, newuri = self.makefile(8)
2043 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2044 name="bar.txt", uri=newuri)
2045 d.addBoth(self.shouldFail, error.Error,
2046 "POST_put_uri_no_replace_queryarg",
2048 "There was already a child by that name, and you asked me "
2049 "to not replace it")
2050 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2051 d.addCallback(self.failUnlessIsBarDotTxt)
2054 def test_POST_put_uri_no_replace_field(self):
2055 contents, n, newuri = self.makefile(8)
2056 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2057 name="bar.txt", uri=newuri)
2058 d.addBoth(self.shouldFail, error.Error,
2059 "POST_put_uri_no_replace_field",
2061 "There was already a child by that name, and you asked me "
2062 "to not replace it")
2063 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2064 d.addCallback(self.failUnlessIsBarDotTxt)
2067 def test_POST_delete(self):
2068 d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2069 d.addCallback(lambda res: self._foo_node.list())
2070 def _check(children):
2071 self.failIf(u"bar.txt" in children)
2072 d.addCallback(_check)
2075 def test_POST_rename_file(self):
2076 d = self.POST(self.public_url + "/foo", t="rename",
2077 from_name="bar.txt", to_name='wibble.txt')
2078 d.addCallback(lambda res:
2079 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2080 d.addCallback(lambda res:
2081 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2082 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2083 d.addCallback(self.failUnlessIsBarDotTxt)
2084 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2085 d.addCallback(self.failUnlessIsBarJSON)
2088 def test_POST_rename_file_redundant(self):
2089 d = self.POST(self.public_url + "/foo", t="rename",
2090 from_name="bar.txt", to_name='bar.txt')
2091 d.addCallback(lambda res:
2092 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2093 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2094 d.addCallback(self.failUnlessIsBarDotTxt)
2095 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2096 d.addCallback(self.failUnlessIsBarJSON)
2099 def test_POST_rename_file_replace(self):
2100 # rename a file and replace a directory with it
2101 d = self.POST(self.public_url + "/foo", t="rename",
2102 from_name="bar.txt", to_name='empty')
2103 d.addCallback(lambda res:
2104 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2105 d.addCallback(lambda res:
2106 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2107 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2108 d.addCallback(self.failUnlessIsBarDotTxt)
2109 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2110 d.addCallback(self.failUnlessIsBarJSON)
2113 def test_POST_rename_file_no_replace_queryarg(self):
2114 # rename a file and replace a directory with it
2115 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2116 from_name="bar.txt", to_name='empty')
2117 d.addBoth(self.shouldFail, error.Error,
2118 "POST_rename_file_no_replace_queryarg",
2120 "There was already a child by that name, and you asked me "
2121 "to not replace it")
2122 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2123 d.addCallback(self.failUnlessIsEmptyJSON)
2126 def test_POST_rename_file_no_replace_field(self):
2127 # rename a file and replace a directory with it
2128 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2129 from_name="bar.txt", to_name='empty')
2130 d.addBoth(self.shouldFail, error.Error,
2131 "POST_rename_file_no_replace_field",
2133 "There was already a child by that name, and you asked me "
2134 "to not replace it")
2135 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2136 d.addCallback(self.failUnlessIsEmptyJSON)
2139 def failUnlessIsEmptyJSON(self, res):
2140 data = simplejson.loads(res)
2141 self.failUnlessEqual(data[0], "dirnode", data)
2142 self.failUnlessEqual(len(data[1]["children"]), 0)
2144 def test_POST_rename_file_slash_fail(self):
2145 d = self.POST(self.public_url + "/foo", t="rename",
2146 from_name="bar.txt", to_name='kirk/spock.txt')
2147 d.addBoth(self.shouldFail, error.Error,
2148 "test_POST_rename_file_slash_fail",
2150 "to_name= may not contain a slash",
2152 d.addCallback(lambda res:
2153 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2156 def test_POST_rename_dir(self):
2157 d = self.POST(self.public_url, t="rename",
2158 from_name="foo", to_name='plunk')
2159 d.addCallback(lambda res:
2160 self.failIfNodeHasChild(self.public_root, u"foo"))
2161 d.addCallback(lambda res:
2162 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2163 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2164 d.addCallback(self.failUnlessIsFooJSON)
2167 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2168 """ If target is not None then the redirection has to go to target. If
2169 statuscode is not None then the redirection has to be accomplished with
2170 that HTTP status code."""
2171 if not isinstance(res, failure.Failure):
2172 to_where = (target is None) and "somewhere" or ("to " + target)
2173 self.fail("%s: we were expecting to get redirected %s, not get an"
2174 " actual page: %s" % (which, to_where, res))
2175 res.trap(error.PageRedirect)
2176 if statuscode is not None:
2177 self.failUnlessEqual(res.value.status, statuscode,
2178 "%s: not a redirect" % which)
2179 if target is not None:
2180 # the PageRedirect does not seem to capture the uri= query arg
2181 # properly, so we can't check for it.
2182 realtarget = self.webish_url + target
2183 self.failUnlessEqual(res.value.location, realtarget,
2184 "%s: wrong target" % which)
2185 return res.value.location
2187 def test_GET_URI_form(self):
2188 base = "/uri?uri=%s" % self._bar_txt_uri
2189 # this is supposed to give us a redirect to /uri/$URI, plus arguments
2190 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2192 d.addBoth(self.shouldRedirect, targetbase)
2193 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2194 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2195 d.addCallback(lambda res: self.GET(base+"&t=json"))
2196 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2197 d.addCallback(self.log, "about to get file by uri")
2198 d.addCallback(lambda res: self.GET(base, followRedirect=True))
2199 d.addCallback(self.failUnlessIsBarDotTxt)
2200 d.addCallback(self.log, "got file by uri, about to get dir by uri")
2201 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2202 followRedirect=True))
2203 d.addCallback(self.failUnlessIsFooJSON)
2204 d.addCallback(self.log, "got dir by uri")
2208 def test_GET_URI_form_bad(self):
2209 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2210 "400 Bad Request", "GET /uri requires uri=",
2214 def test_GET_rename_form(self):
2215 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2216 followRedirect=True)
2218 self.failUnless('name="when_done" value="."' in res, res)
2219 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2220 d.addCallback(_check)
2223 def log(self, res, msg):
2224 #print "MSG: %s RES: %s" % (msg, res)
2228 def test_GET_URI_URL(self):
2229 base = "/uri/%s" % self._bar_txt_uri
2231 d.addCallback(self.failUnlessIsBarDotTxt)
2232 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2233 d.addCallback(self.failUnlessIsBarDotTxt)
2234 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2235 d.addCallback(self.failUnlessIsBarDotTxt)
2238 def test_GET_URI_URL_dir(self):
2239 base = "/uri/%s?t=json" % self._foo_uri
2241 d.addCallback(self.failUnlessIsFooJSON)
2244 def test_GET_URI_URL_missing(self):
2245 base = "/uri/%s" % self._bad_file_uri
2247 d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2248 http.GONE, response_substring="NotEnoughSharesError")
2249 # TODO: how can we exercise both sides of WebDownloadTarget.fail
2250 # here? we must arrange for a download to fail after target.open()
2251 # has been called, and then inspect the response to see that it is
2252 # shorter than we expected.
2255 def test_PUT_DIRURL_uri(self):
2256 d = self.s.create_empty_dirnode()
2258 new_uri = dn.get_uri()
2259 # replace /foo with a new (empty) directory
2260 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2261 d.addCallback(lambda res:
2262 self.failUnlessEqual(res.strip(), new_uri))
2263 d.addCallback(lambda res:
2264 self.failUnlessChildURIIs(self.public_root,
2268 d.addCallback(_made_dir)
2271 def test_PUT_DIRURL_uri_noreplace(self):
2272 d = self.s.create_empty_dirnode()
2274 new_uri = dn.get_uri()
2275 # replace /foo with a new (empty) directory, but ask that
2276 # replace=false, so it should fail
2277 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2278 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2280 self.public_url + "/foo?t=uri&replace=false",
2282 d.addCallback(lambda res:
2283 self.failUnlessChildURIIs(self.public_root,
2287 d.addCallback(_made_dir)
2290 def test_PUT_DIRURL_bad_t(self):
2291 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2292 "400 Bad Request", "PUT to a directory",
2293 self.PUT, self.public_url + "/foo?t=BOGUS", "")
2294 d.addCallback(lambda res:
2295 self.failUnlessChildURIIs(self.public_root,
2300 def test_PUT_NEWFILEURL_uri(self):
2301 contents, n, new_uri = self.makefile(8)
2302 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2303 d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2304 d.addCallback(lambda res:
2305 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2309 def test_PUT_NEWFILEURL_uri_replace(self):
2310 contents, n, new_uri = self.makefile(8)
2311 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2312 d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2313 d.addCallback(lambda res:
2314 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2318 def test_PUT_NEWFILEURL_uri_no_replace(self):
2319 contents, n, new_uri = self.makefile(8)
2320 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2321 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2323 "There was already a child by that name, and you asked me "
2324 "to not replace it")
2327 def test_PUT_NEWFILE_URI(self):
2328 file_contents = "New file contents here\n"
2329 d = self.PUT("/uri", file_contents)
2331 assert isinstance(uri, str), uri
2332 self.failUnless(uri in FakeCHKFileNode.all_contents)
2333 self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2335 return self.GET("/uri/%s" % uri)
2336 d.addCallback(_check)
2338 self.failUnlessEqual(res, file_contents)
2339 d.addCallback(_check2)
2342 def test_PUT_NEWFILE_URI_only_PUT(self):
2343 d = self.PUT("/uri?t=bogus", "")
2344 d.addBoth(self.shouldFail, error.Error,
2345 "PUT_NEWFILE_URI_only_PUT",
2347 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2350 def test_PUT_NEWFILE_URI_mutable(self):
2351 file_contents = "New file contents here\n"
2352 d = self.PUT("/uri?mutable=true", file_contents)
2353 def _check_mutable(uri):
2356 self.failUnless(IMutableFileURI.providedBy(u))
2357 self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2358 n = self.s.create_node_from_uri(uri)
2359 return n.download_best_version()
2360 d.addCallback(_check_mutable)
2361 def _check2_mutable(data):
2362 self.failUnlessEqual(data, file_contents)
2363 d.addCallback(_check2_mutable)
2367 self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2368 self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2370 return self.GET("/uri/%s" % uri)
2371 d.addCallback(_check)
2373 self.failUnlessEqual(res, file_contents)
2374 d.addCallback(_check2)
2377 def test_PUT_mkdir(self):
2378 d = self.PUT("/uri?t=mkdir", "")
2380 n = self.s.create_node_from_uri(uri.strip())
2381 d2 = self.failUnlessNodeKeysAre(n, [])
2382 d2.addCallback(lambda res:
2383 self.GET("/uri/%s?t=json" % uri))
2385 d.addCallback(_check)
2386 d.addCallback(self.failUnlessIsEmptyJSON)
2389 def test_POST_check(self):
2390 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2392 # this returns a string form of the results, which are probably
2393 # None since we're using fake filenodes.
2394 # TODO: verify that the check actually happened, by changing
2395 # FakeCHKFileNode to count how many times .check() has been
2398 d.addCallback(_done)
2401 def test_bad_method(self):
2402 url = self.webish_url + self.public_url + "/foo/bar.txt"
2403 d = self.shouldHTTPError2("test_bad_method",
2404 501, "Not Implemented",
2405 "I don't know how to treat a BOGUS request.",
2406 client.getPage, url, method="BOGUS")
2409 def test_short_url(self):
2410 url = self.webish_url + "/uri"
2411 d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2412 "I don't know how to treat a DELETE request.",
2413 client.getPage, url, method="DELETE")
2416 def test_ophandle_bad(self):
2417 url = self.webish_url + "/operations/bogus?t=status"
2418 d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2419 "unknown/expired handle 'bogus'",
2420 client.getPage, url)
2423 def test_ophandle_cancel(self):
2424 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2425 followRedirect=True)
2426 d.addCallback(lambda ignored:
2427 self.GET("/operations/128?t=status&output=JSON"))
2429 data = simplejson.loads(res)
2430 self.failUnless("finished" in data, res)
2431 monitor = self.ws.root.child_operations.handles["128"][0]
2432 d = self.POST("/operations/128?t=cancel&output=JSON")
2434 data = simplejson.loads(res)
2435 self.failUnless("finished" in data, res)
2436 # t=cancel causes the handle to be forgotten
2437 self.failUnless(monitor.is_cancelled())
2438 d.addCallback(_check2)
2440 d.addCallback(_check1)
2441 d.addCallback(lambda ignored:
2442 self.shouldHTTPError2("test_ophandle_cancel",
2443 404, "404 Not Found",
2444 "unknown/expired handle '128'",
2446 "/operations/128?t=status&output=JSON"))
2449 def test_ophandle_retainfor(self):
2450 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2451 followRedirect=True)
2452 d.addCallback(lambda ignored:
2453 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2455 data = simplejson.loads(res)
2456 self.failUnless("finished" in data, res)
2457 d.addCallback(_check1)
2458 # the retain-for=0 will cause the handle to be expired very soon
2459 d.addCallback(self.stall, 2.0)
2460 d.addCallback(lambda ignored:
2461 self.shouldHTTPError2("test_ophandle_retainfor",
2462 404, "404 Not Found",
2463 "unknown/expired handle '129'",
2465 "/operations/129?t=status&output=JSON"))
2468 def test_ophandle_release_after_complete(self):
2469 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2470 followRedirect=True)
2471 d.addCallback(self.wait_for_operation, "130")
2472 d.addCallback(lambda ignored:
2473 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2474 # the release-after-complete=true will cause the handle to be expired
2475 d.addCallback(lambda ignored:
2476 self.shouldHTTPError2("test_ophandle_release_after_complete",
2477 404, "404 Not Found",
2478 "unknown/expired handle '130'",
2480 "/operations/130?t=status&output=JSON"))
2483 def test_incident(self):
2484 d = self.POST("/report_incident", details="eek")
2486 self.failUnless("Thank you for your report!" in res, res)
2487 d.addCallback(_done)
2490 def test_static(self):
2491 webdir = os.path.join(self.staticdir, "subdir")
2492 fileutil.make_dirs(webdir)
2493 f = open(os.path.join(webdir, "hello.txt"), "wb")
2497 d = self.GET("/static/subdir/hello.txt")
2499 self.failUnlessEqual(res, "hello")
2500 d.addCallback(_check)
2504 class Util(unittest.TestCase):
2505 def test_abbreviate_time(self):
2506 self.failUnlessEqual(common.abbreviate_time(None), "")
2507 self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2508 self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2509 self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2510 self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2512 def test_abbreviate_rate(self):
2513 self.failUnlessEqual(common.abbreviate_rate(None), "")
2514 self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2515 self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2516 self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2518 def test_abbreviate_size(self):
2519 self.failUnlessEqual(common.abbreviate_size(None), "")
2520 self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2521 self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2522 self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2523 self.failUnlessEqual(common.abbreviate_size(123), "123B")
2526 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase):
2528 def GET(self, urlpath, followRedirect=False, return_response=False,
2529 method="GET", **kwargs):
2530 # if return_response=True, this fires with (data, statuscode,
2531 # respheaders) instead of just data.
2532 assert not isinstance(urlpath, unicode)
2533 url = self.client_baseurls[0] + urlpath
2534 factory = HTTPClientGETFactory(url, method=method,
2535 followRedirect=followRedirect, **kwargs)
2536 reactor.connectTCP("localhost", self.client_webports[0], factory)
2537 d = factory.deferred
2538 def _got_data(data):
2539 return (data, factory.status, factory.response_headers)
2541 d.addCallback(_got_data)
2542 return factory.deferred
2544 def CHECK(self, ign, which, args):
2545 fileurl = self.fileurls[which]
2546 url = fileurl + "?" + args
2547 return self.GET(url, method="POST")
2549 def test_filecheck(self):
2550 self.basedir = "web/Grid/filecheck"
2552 c0 = self.g.clients[0]
2555 d = c0.upload(upload.Data(DATA, convergence=""))
2556 def _stash_uri(ur, which):
2557 self.uris[which] = ur.uri
2558 d.addCallback(_stash_uri, "good")
2559 d.addCallback(lambda ign:
2560 c0.upload(upload.Data(DATA+"1", convergence="")))
2561 d.addCallback(_stash_uri, "sick")
2562 d.addCallback(lambda ign:
2563 c0.upload(upload.Data(DATA+"2", convergence="")))
2564 d.addCallback(_stash_uri, "dead")
2565 def _stash_mutable_uri(n, which):
2566 self.uris[which] = n.get_uri()
2567 assert isinstance(self.uris[which], str)
2568 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2569 d.addCallback(_stash_mutable_uri, "corrupt")
2570 d.addCallback(lambda ign:
2571 c0.upload(upload.Data("literal", convergence="")))
2572 d.addCallback(_stash_uri, "small")
2574 def _compute_fileurls(ignored):
2576 for which in self.uris:
2577 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2578 d.addCallback(_compute_fileurls)
2580 def _clobber_shares(ignored):
2581 good_shares = self.find_shares(self.uris["good"])
2582 self.failUnlessEqual(len(good_shares), 10)
2583 sick_shares = self.find_shares(self.uris["sick"])
2584 os.unlink(sick_shares[0][2])
2585 dead_shares = self.find_shares(self.uris["dead"])
2586 for i in range(1, 10):
2587 os.unlink(dead_shares[i][2])
2588 c_shares = self.find_shares(self.uris["corrupt"])
2589 cso = CorruptShareOptions()
2590 cso.stdout = StringIO()
2591 cso.parseOptions([c_shares[0][2]])
2593 d.addCallback(_clobber_shares)
2595 d.addCallback(self.CHECK, "good", "t=check")
2596 def _got_html_good(res):
2597 self.failUnless("Healthy" in res, res)
2598 self.failIf("Not Healthy" in res, res)
2599 d.addCallback(_got_html_good)
2600 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2601 def _got_html_good_return_to(res):
2602 self.failUnless("Healthy" in res, res)
2603 self.failIf("Not Healthy" in res, res)
2604 self.failUnless('<a href="somewhere">Return to parent directory'
2606 d.addCallback(_got_html_good_return_to)
2607 d.addCallback(self.CHECK, "good", "t=check&output=json")
2608 def _got_json_good(res):
2609 r = simplejson.loads(res)
2610 self.failUnlessEqual(r["summary"], "Healthy")
2611 self.failUnless(r["results"]["healthy"])
2612 self.failIf(r["results"]["needs-rebalancing"])
2613 self.failUnless(r["results"]["recoverable"])
2614 d.addCallback(_got_json_good)
2616 d.addCallback(self.CHECK, "small", "t=check")
2617 def _got_html_small(res):
2618 self.failUnless("Literal files are always healthy" in res, res)
2619 self.failIf("Not Healthy" in res, res)
2620 d.addCallback(_got_html_small)
2621 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2622 def _got_html_small_return_to(res):
2623 self.failUnless("Literal files are always healthy" in res, res)
2624 self.failIf("Not Healthy" in res, res)
2625 self.failUnless('<a href="somewhere">Return to parent directory'
2627 d.addCallback(_got_html_small_return_to)
2628 d.addCallback(self.CHECK, "small", "t=check&output=json")
2629 def _got_json_small(res):
2630 r = simplejson.loads(res)
2631 self.failUnlessEqual(r["storage-index"], "")
2632 self.failUnless(r["results"]["healthy"])
2633 d.addCallback(_got_json_small)
2635 d.addCallback(self.CHECK, "sick", "t=check")
2636 def _got_html_sick(res):
2637 self.failUnless("Not Healthy" in res, res)
2638 d.addCallback(_got_html_sick)
2639 d.addCallback(self.CHECK, "sick", "t=check&output=json")
2640 def _got_json_sick(res):
2641 r = simplejson.loads(res)
2642 self.failUnlessEqual(r["summary"],
2643 "Not Healthy: 9 shares (enc 3-of-10)")
2644 self.failIf(r["results"]["healthy"])
2645 self.failIf(r["results"]["needs-rebalancing"])
2646 self.failUnless(r["results"]["recoverable"])
2647 d.addCallback(_got_json_sick)
2649 d.addCallback(self.CHECK, "dead", "t=check")
2650 def _got_html_dead(res):
2651 self.failUnless("Not Healthy" in res, res)
2652 d.addCallback(_got_html_dead)
2653 d.addCallback(self.CHECK, "dead", "t=check&output=json")
2654 def _got_json_dead(res):
2655 r = simplejson.loads(res)
2656 self.failUnlessEqual(r["summary"],
2657 "Not Healthy: 1 shares (enc 3-of-10)")
2658 self.failIf(r["results"]["healthy"])
2659 self.failIf(r["results"]["needs-rebalancing"])
2660 self.failIf(r["results"]["recoverable"])
2661 d.addCallback(_got_json_dead)
2663 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2664 def _got_html_corrupt(res):
2665 self.failUnless("Not Healthy! : Unhealthy" in res, res)
2666 d.addCallback(_got_html_corrupt)
2667 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2668 def _got_json_corrupt(res):
2669 r = simplejson.loads(res)
2670 self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2672 self.failIf(r["results"]["healthy"])
2673 self.failUnless(r["results"]["recoverable"])
2674 self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2675 self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2676 d.addCallback(_got_json_corrupt)
2678 d.addErrback(self.explain_web_error)
2681 def test_repair_html(self):
2682 self.basedir = "web/Grid/repair_html"
2684 c0 = self.g.clients[0]
2687 d = c0.upload(upload.Data(DATA, convergence=""))
2688 def _stash_uri(ur, which):
2689 self.uris[which] = ur.uri
2690 d.addCallback(_stash_uri, "good")
2691 d.addCallback(lambda ign:
2692 c0.upload(upload.Data(DATA+"1", convergence="")))
2693 d.addCallback(_stash_uri, "sick")
2694 d.addCallback(lambda ign:
2695 c0.upload(upload.Data(DATA+"2", convergence="")))
2696 d.addCallback(_stash_uri, "dead")
2697 def _stash_mutable_uri(n, which):
2698 self.uris[which] = n.get_uri()
2699 assert isinstance(self.uris[which], str)
2700 d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2701 d.addCallback(_stash_mutable_uri, "corrupt")
2703 def _compute_fileurls(ignored):
2705 for which in self.uris:
2706 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2707 d.addCallback(_compute_fileurls)
2709 def _clobber_shares(ignored):
2710 good_shares = self.find_shares(self.uris["good"])
2711 self.failUnlessEqual(len(good_shares), 10)
2712 sick_shares = self.find_shares(self.uris["sick"])
2713 os.unlink(sick_shares[0][2])
2714 dead_shares = self.find_shares(self.uris["dead"])
2715 for i in range(1, 10):
2716 os.unlink(dead_shares[i][2])
2717 c_shares = self.find_shares(self.uris["corrupt"])
2718 cso = CorruptShareOptions()
2719 cso.stdout = StringIO()
2720 cso.parseOptions([c_shares[0][2]])
2722 d.addCallback(_clobber_shares)
2724 d.addCallback(self.CHECK, "good", "t=check&repair=true")
2725 def _got_html_good(res):
2726 self.failUnless("Healthy" in res, res)
2727 self.failIf("Not Healthy" in res, res)
2728 self.failUnless("No repair necessary" in res, res)
2729 d.addCallback(_got_html_good)
2731 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2732 def _got_html_sick(res):
2733 self.failUnless("Healthy : healthy" in res, res)
2734 self.failIf("Not Healthy" in res, res)
2735 self.failUnless("Repair successful" in res, res)
2736 d.addCallback(_got_html_sick)
2738 # repair of a dead file will fail, of course, but it isn't yet
2739 # clear how this should be reported. Right now it shows up as
2742 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2743 #def _got_html_dead(res):
2745 # self.failUnless("Healthy : healthy" in res, res)
2746 # self.failIf("Not Healthy" in res, res)
2747 # self.failUnless("No repair necessary" in res, res)
2748 #d.addCallback(_got_html_dead)
2750 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2751 def _got_html_corrupt(res):
2752 self.failUnless("Healthy : Healthy" in res, res)
2753 self.failIf("Not Healthy" in res, res)
2754 self.failUnless("Repair successful" in res, res)
2755 d.addCallback(_got_html_corrupt)
2757 d.addErrback(self.explain_web_error)
2760 def test_repair_json(self):
2761 self.basedir = "web/Grid/repair_json"
2763 c0 = self.g.clients[0]
2766 d = c0.upload(upload.Data(DATA+"1", convergence=""))
2767 def _stash_uri(ur, which):
2768 self.uris[which] = ur.uri
2769 d.addCallback(_stash_uri, "sick")
2771 def _compute_fileurls(ignored):
2773 for which in self.uris:
2774 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2775 d.addCallback(_compute_fileurls)
2777 def _clobber_shares(ignored):
2778 sick_shares = self.find_shares(self.uris["sick"])
2779 os.unlink(sick_shares[0][2])
2780 d.addCallback(_clobber_shares)
2782 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
2783 def _got_json_sick(res):
2784 r = simplejson.loads(res)
2785 self.failUnlessEqual(r["repair-attempted"], True)
2786 self.failUnlessEqual(r["repair-successful"], True)
2787 self.failUnlessEqual(r["pre-repair-results"]["summary"],
2788 "Not Healthy: 9 shares (enc 3-of-10)")
2789 self.failIf(r["pre-repair-results"]["results"]["healthy"])
2790 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
2791 self.failUnless(r["post-repair-results"]["results"]["healthy"])
2792 d.addCallback(_got_json_sick)
2794 d.addErrback(self.explain_web_error)
2797 def test_deep_check(self):
2798 self.basedir = "web/Grid/deep_check"
2800 c0 = self.g.clients[0]
2804 d = c0.create_empty_dirnode()
2805 def _stash_root_and_create_file(n):
2807 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2808 return n.add_file(u"good", upload.Data(DATA, convergence=""))
2809 d.addCallback(_stash_root_and_create_file)
2810 def _stash_uri(fn, which):
2811 self.uris[which] = fn.get_uri()
2812 d.addCallback(_stash_uri, "good")
2813 d.addCallback(lambda ign:
2814 self.rootnode.add_file(u"small",
2815 upload.Data("literal",
2817 d.addCallback(_stash_uri, "small")
2819 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2821 units = [simplejson.loads(line)
2822 for line in res.splitlines()
2824 self.failUnlessEqual(len(units), 3+1)
2825 # should be parent-first
2827 self.failUnlessEqual(u0["path"], [])
2828 self.failUnlessEqual(u0["type"], "directory")
2829 self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2830 u0cr = u0["check-results"]
2831 self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
2833 ugood = [u for u in units
2834 if u["type"] == "file" and u["path"] == [u"good"]][0]
2835 self.failUnlessEqual(ugood["cap"], self.uris["good"])
2836 ugoodcr = ugood["check-results"]
2837 self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
2840 self.failUnlessEqual(stats["type"], "stats")
2842 self.failUnlessEqual(s["count-immutable-files"], 1)
2843 self.failUnlessEqual(s["count-literal-files"], 1)
2844 self.failUnlessEqual(s["count-directories"], 1)
2845 d.addCallback(_done)
2847 d.addErrback(self.explain_web_error)
2850 def test_deep_check_and_repair(self):
2851 self.basedir = "web/Grid/deep_check_and_repair"
2853 c0 = self.g.clients[0]
2857 d = c0.create_empty_dirnode()
2858 def _stash_root_and_create_file(n):
2860 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2861 return n.add_file(u"good", upload.Data(DATA, convergence=""))
2862 d.addCallback(_stash_root_and_create_file)
2863 def _stash_uri(fn, which):
2864 self.uris[which] = fn.get_uri()
2865 d.addCallback(_stash_uri, "good")
2866 d.addCallback(lambda ign:
2867 self.rootnode.add_file(u"small",
2868 upload.Data("literal",
2870 d.addCallback(_stash_uri, "small")
2871 d.addCallback(lambda ign:
2872 self.rootnode.add_file(u"sick",
2873 upload.Data(DATA+"1",
2875 d.addCallback(_stash_uri, "sick")
2876 #d.addCallback(lambda ign:
2877 # self.rootnode.add_file(u"dead",
2878 # upload.Data(DATA+"2",
2880 #d.addCallback(_stash_uri, "dead")
2882 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
2883 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
2884 #d.addCallback(_stash_uri, "corrupt")
2886 def _clobber_shares(ignored):
2887 good_shares = self.find_shares(self.uris["good"])
2888 self.failUnlessEqual(len(good_shares), 10)
2889 sick_shares = self.find_shares(self.uris["sick"])
2890 os.unlink(sick_shares[0][2])
2891 #dead_shares = self.find_shares(self.uris["dead"])
2892 #for i in range(1, 10):
2893 # os.unlink(dead_shares[i][2])
2895 #c_shares = self.find_shares(self.uris["corrupt"])
2896 #cso = CorruptShareOptions()
2897 #cso.stdout = StringIO()
2898 #cso.parseOptions([c_shares[0][2]])
2900 d.addCallback(_clobber_shares)
2902 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
2904 units = [simplejson.loads(line)
2905 for line in res.splitlines()
2907 self.failUnlessEqual(len(units), 4+1)
2908 # should be parent-first
2910 self.failUnlessEqual(u0["path"], [])
2911 self.failUnlessEqual(u0["type"], "directory")
2912 self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2913 u0crr = u0["check-and-repair-results"]
2914 self.failUnlessEqual(u0crr["repair-attempted"], False)
2915 self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2917 ugood = [u for u in units
2918 if u["type"] == "file" and u["path"] == [u"good"]][0]
2919 self.failUnlessEqual(ugood["cap"], self.uris["good"])
2920 ugoodcrr = ugood["check-and-repair-results"]
2921 self.failUnlessEqual(u0crr["repair-attempted"], False)
2922 self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2924 usick = [u for u in units
2925 if u["type"] == "file" and u["path"] == [u"sick"]][0]
2926 self.failUnlessEqual(usick["cap"], self.uris["sick"])
2927 usickcrr = usick["check-and-repair-results"]
2928 self.failUnlessEqual(usickcrr["repair-attempted"], True)
2929 self.failUnlessEqual(usickcrr["repair-successful"], True)
2930 self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
2931 self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
2934 self.failUnlessEqual(stats["type"], "stats")
2936 self.failUnlessEqual(s["count-immutable-files"], 2)
2937 self.failUnlessEqual(s["count-literal-files"], 1)
2938 self.failUnlessEqual(s["count-directories"], 1)
2939 d.addCallback(_done)
2941 d.addErrback(self.explain_web_error)