1 import os.path, re, urllib
3 from twisted.application import service
4 from twisted.trial import unittest
5 from twisted.internet import defer, reactor
6 from twisted.web import client, error, http
7 from twisted.python import failure, log
8 from allmydata import interfaces, provisioning, uri, webish
9 from allmydata.immutable import upload, download
10 from allmydata.web import status, common
11 from allmydata.util import fileutil, base32
12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
13 FakeMutableFileNode, create_chk_filenode
14 from allmydata.interfaces import IURI, INewDirectoryURI, \
15 IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
16 from allmydata.mutable import servermap, publish, retrieve
17 import common_util as testutil
19 # create a fake uploader/downloader, and a couple of fake dirnodes, then
20 # create a webserver that works against them
22 class FakeIntroducerClient:
23 def get_all_connectors(self):
25 def get_all_connections_for(self, service_name):
27 def get_all_peerids(self):
30 class FakeClient(service.MultiService):
31 nodeid = "fake_nodeid"
32 nickname = "fake_nickname"
33 basedir = "fake_basedir"
34 def get_versions(self):
35 return {'allmydata': "fake",
40 introducer_furl = "None"
41 introducer_client = FakeIntroducerClient()
42 _all_upload_status = [upload.UploadStatus()]
43 _all_download_status = [download.DownloadStatus()]
44 _all_mapupdate_statuses = [servermap.UpdateStatus()]
45 _all_publish_statuses = [publish.PublishStatus()]
46 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
47 convergence = "some random string"
49 def connected_to_introducer(self):
52 def get_nickname_for_peerid(self, peerid):
55 def create_node_from_uri(self, auri):
56 u = uri.from_string(auri)
57 if (INewDirectoryURI.providedBy(u)
58 or IReadonlyNewDirectoryURI.providedBy(u)):
59 return FakeDirectoryNode(self).init_from_uri(u)
60 if IFileURI.providedBy(u):
61 return FakeCHKFileNode(u, self)
62 assert IMutableFileURI.providedBy(u), u
63 return FakeMutableFileNode(self).init_from_uri(u)
65 def create_empty_dirnode(self):
66 n = FakeDirectoryNode(self)
68 d.addCallback(lambda res: n)
71 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
72 def create_mutable_file(self, contents=""):
73 n = FakeMutableFileNode(self)
74 return n.create(contents)
76 def upload(self, uploadable):
77 d = uploadable.get_size()
78 d.addCallback(lambda size: uploadable.read(size))
81 n = create_chk_filenode(self, data)
82 results = upload.UploadResults()
83 results.uri = n.get_uri()
85 d.addCallback(_got_data)
88 def list_all_upload_statuses(self):
89 return self._all_upload_status
90 def list_all_download_statuses(self):
91 return self._all_download_status
92 def list_all_mapupdate_statuses(self):
93 return self._all_mapupdate_statuses
94 def list_all_publish_statuses(self):
95 return self._all_publish_statuses
96 def list_all_retrieve_statuses(self):
97 return self._all_retrieve_statuses
98 def list_all_helper_statuses(self):
101 class MyGetter(client.HTTPPageGetter):
102 handleStatus_206 = lambda self: self.handleStatus_200()
104 class HTTPClientHEADFactory(client.HTTPClientFactory):
107 def noPage(self, reason):
108 # Twisted-2.5.0 and earlier had a bug, in which they would raise an
109 # exception when the response to a HEAD request had no body (when in
110 # fact they are defined to never have a body). This was fixed in
111 # Twisted-8.0 . To work around this, we catch the
112 # PartialDownloadError and make it disappear.
113 if (reason.check(client.PartialDownloadError)
114 and self.method.upper() == "HEAD"):
117 return client.HTTPClientFactory.noPage(self, reason)
119 class HTTPClientGETFactory(client.HTTPClientFactory):
122 class WebMixin(object):
124 self.s = FakeClient()
125 self.s.startService()
126 self.staticdir = self.mktemp()
127 self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
128 s.setServiceParent(self.s)
129 self.webish_port = port = s.listener._port.getHost().port
130 self.webish_url = "http://localhost:%d" % port
132 l = [ self.s.create_empty_dirnode() for x in range(6) ]
133 d = defer.DeferredList(l)
135 self.public_root = res[0][1]
136 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
137 self.public_url = "/uri/" + self.public_root.get_uri()
138 self.private_root = res[1][1]
142 self._foo_uri = foo.get_uri()
143 self._foo_readonly_uri = foo.get_readonly_uri()
144 # NOTE: we ignore the deferred on all set_uri() calls, because we
145 # know the fake nodes do these synchronously
146 self.public_root.set_uri(u"foo", foo.get_uri())
148 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
149 foo.set_uri(u"bar.txt", self._bar_txt_uri)
151 foo.set_uri(u"empty", res[3][1].get_uri())
152 sub_uri = res[4][1].get_uri()
153 self._sub_uri = sub_uri
154 foo.set_uri(u"sub", sub_uri)
155 sub = self.s.create_node_from_uri(sub_uri)
157 _ign, n, blocking_uri = self.makefile(1)
158 foo.set_uri(u"blockingfile", blocking_uri)
160 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
161 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
162 # still think of it as an umlaut
163 foo.set_uri(unicode_filename, self._bar_txt_uri)
165 _ign, n, baz_file = self.makefile(2)
166 sub.set_uri(u"baz.txt", baz_file)
168 _ign, n, self._bad_file_uri = self.makefile(3)
169 # this uri should not be downloadable
170 del FakeCHKFileNode.all_contents[self._bad_file_uri]
173 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
174 rodir.set_uri(u"nor", baz_file)
179 # public/foo/blockingfile
182 # public/foo/sub/baz.txt
184 # public/reedownlee/nor
185 self.NEWFILE_CONTENTS = "newfile contents\n"
187 return foo.get_metadata_for(u"bar.txt")
189 def _got_metadata(metadata):
190 self._bar_txt_metadata = metadata
191 d.addCallback(_got_metadata)
194 def makefile(self, number):
195 contents = "contents of file %s\n" % number
196 n = create_chk_filenode(self.s, contents)
197 return contents, n, n.get_uri()
200 return self.s.stopService()
202 def failUnlessIsBarDotTxt(self, res):
203 self.failUnlessEqual(res, self.BAR_CONTENTS, res)
205 def failUnlessIsBarJSON(self, res):
206 data = simplejson.loads(res)
207 self.failUnless(isinstance(data, list))
208 self.failUnlessEqual(data[0], u"filenode")
209 self.failUnless(isinstance(data[1], dict))
210 self.failIf(data[1]["mutable"])
211 self.failIf("rw_uri" in data[1]) # immutable
212 self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
213 self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
215 def failUnlessIsFooJSON(self, res):
216 data = simplejson.loads(res)
217 self.failUnless(isinstance(data, list))
218 self.failUnlessEqual(data[0], "dirnode", res)
219 self.failUnless(isinstance(data[1], dict))
220 self.failUnless(data[1]["mutable"])
221 self.failUnless("rw_uri" in data[1]) # mutable
222 self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
223 self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
225 kidnames = sorted([unicode(n) for n in data[1]["children"]])
226 self.failUnlessEqual(kidnames,
227 [u"bar.txt", u"blockingfile", u"empty",
228 u"n\u00fc.txt", u"sub"])
229 kids = dict( [(unicode(name),value)
231 in data[1]["children"].iteritems()] )
232 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
233 self.failUnless("metadata" in kids[u"sub"][1])
234 self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
235 self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
236 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
237 self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
238 self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
239 self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
240 self._bar_txt_metadata["ctime"])
241 self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
244 def GET(self, urlpath, followRedirect=False, return_response=False,
246 # if return_response=True, this fires with (data, statuscode,
247 # respheaders) instead of just data.
248 assert not isinstance(urlpath, unicode)
249 url = self.webish_url + urlpath
250 factory = HTTPClientGETFactory(url, method="GET",
251 followRedirect=followRedirect, **kwargs)
252 reactor.connectTCP("localhost", self.webish_port, factory)
255 return (data, factory.status, factory.response_headers)
257 d.addCallback(_got_data)
258 return factory.deferred
260 def HEAD(self, urlpath, return_response=False, **kwargs):
261 # this requires some surgery, because twisted.web.client doesn't want
262 # to give us back the response headers.
263 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
264 reactor.connectTCP("localhost", self.webish_port, factory)
267 return (data, factory.status, factory.response_headers)
269 d.addCallback(_got_data)
270 return factory.deferred
272 def PUT(self, urlpath, data, **kwargs):
273 url = self.webish_url + urlpath
274 return client.getPage(url, method="PUT", postdata=data, **kwargs)
276 def DELETE(self, urlpath):
277 url = self.webish_url + urlpath
278 return client.getPage(url, method="DELETE")
280 def POST(self, urlpath, followRedirect=False, **fields):
281 url = self.webish_url + urlpath
282 sepbase = "boogabooga"
286 form.append('Content-Disposition: form-data; name="_charset"')
290 for name, value in fields.iteritems():
291 if isinstance(value, tuple):
292 filename, value = value
293 form.append('Content-Disposition: form-data; name="%s"; '
294 'filename="%s"' % (name, filename.encode("utf-8")))
296 form.append('Content-Disposition: form-data; name="%s"' % name)
298 if isinstance(value, unicode):
299 value = value.encode("utf-8")
302 assert isinstance(value, str)
306 body = "\r\n".join(form) + "\r\n"
307 headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
309 return client.getPage(url, method="POST", postdata=body,
310 headers=headers, followRedirect=followRedirect)
312 def shouldFail(self, res, expected_failure, which,
313 substring=None, response_substring=None):
314 if isinstance(res, failure.Failure):
315 res.trap(expected_failure)
317 self.failUnless(substring in str(res),
318 "substring '%s' not in '%s'"
319 % (substring, str(res)))
320 if response_substring:
321 self.failUnless(response_substring in res.value.response,
322 "response substring '%s' not in '%s'"
323 % (response_substring, res.value.response))
325 self.fail("%s was supposed to raise %s, not get '%s'" %
326 (which, expected_failure, res))
328 def shouldFail2(self, expected_failure, which, substring,
330 callable, *args, **kwargs):
331 assert substring is None or isinstance(substring, str)
332 assert response_substring is None or isinstance(response_substring, str)
333 d = defer.maybeDeferred(callable, *args, **kwargs)
335 if isinstance(res, failure.Failure):
336 res.trap(expected_failure)
338 self.failUnless(substring in str(res),
339 "%s: substring '%s' not in '%s'"
340 % (which, substring, str(res)))
341 if response_substring:
342 self.failUnless(response_substring in res.value.response,
343 "%s: response substring '%s' not in '%s'"
345 response_substring, res.value.response))
347 self.fail("%s was supposed to raise %s, not get '%s'" %
348 (which, expected_failure, res))
352 def should404(self, res, which):
353 if isinstance(res, failure.Failure):
354 res.trap(error.Error)
355 self.failUnlessEqual(res.value.status, "404")
357 self.fail("%s was supposed to Error(404), not get '%s'" %
360 def shouldHTTPError(self, res, which, code=None, substring=None,
361 response_substring=None):
362 if isinstance(res, failure.Failure):
363 res.trap(error.Error)
365 self.failUnlessEqual(res.value.status, str(code))
367 self.failUnless(substring in str(res),
368 "substring '%s' not in '%s'"
369 % (substring, str(res)))
370 if response_substring:
371 self.failUnless(response_substring in res.value.response,
372 "response substring '%s' not in '%s'"
373 % (response_substring, res.value.response))
375 self.fail("%s was supposed to Error(%s), not get '%s'" %
378 def shouldHTTPError2(self, which,
379 code=None, substring=None, response_substring=None,
380 callable=None, *args, **kwargs):
381 assert substring is None or isinstance(substring, str)
383 d = defer.maybeDeferred(callable, *args, **kwargs)
384 d.addBoth(self.shouldHTTPError, which,
385 code, substring, response_substring)
389 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
390 def test_create(self):
393 def test_welcome(self):
396 self.failUnless('Welcome To AllMyData' in res)
397 self.failUnless('Tahoe' in res)
399 self.s.basedir = 'web/test_welcome'
400 fileutil.make_dirs("web/test_welcome")
401 fileutil.make_dirs("web/test_welcome/private")
403 d.addCallback(_check)
406 def test_provisioning_math(self):
407 self.failUnlessEqual(provisioning.binomial(10, 0), 1)
408 self.failUnlessEqual(provisioning.binomial(10, 1), 10)
409 self.failUnlessEqual(provisioning.binomial(10, 2), 45)
410 self.failUnlessEqual(provisioning.binomial(10, 9), 10)
411 self.failUnlessEqual(provisioning.binomial(10, 10), 1)
413 def test_provisioning(self):
414 d = self.GET("/provisioning/")
416 self.failUnless('Tahoe Provisioning Tool' in res)
417 fields = {'filled': True,
418 "num_users": int(50e3),
419 "files_per_user": 1000,
420 "space_per_user": int(1e9),
421 "sharing_ratio": 1.0,
422 "encoding_parameters": "3-of-10-5",
424 "ownership_mode": "A",
425 "download_rate": 100,
430 return self.POST("/provisioning/", **fields)
432 d.addCallback(_check)
434 self.failUnless('Tahoe Provisioning Tool' in res)
435 self.failUnless("Share space consumed: 167.01TB" in res)
437 fields = {'filled': True,
438 "num_users": int(50e6),
439 "files_per_user": 1000,
440 "space_per_user": int(5e9),
441 "sharing_ratio": 1.0,
442 "encoding_parameters": "25-of-100-50",
443 "num_servers": 30000,
444 "ownership_mode": "E",
445 "drive_failure_model": "U",
447 "download_rate": 1000,
452 return self.POST("/provisioning/", **fields)
453 d.addCallback(_check2)
455 self.failUnless("Share space consumed: huge!" in res)
456 fields = {'filled': True}
457 return self.POST("/provisioning/", **fields)
458 d.addCallback(_check3)
460 self.failUnless("Share space consumed:" in res)
461 d.addCallback(_check4)
464 def test_status(self):
465 dl_num = self.s.list_all_download_statuses()[0].get_counter()
466 ul_num = self.s.list_all_upload_statuses()[0].get_counter()
467 mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
468 pub_num = self.s.list_all_publish_statuses()[0].get_counter()
469 ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
470 d = self.GET("/status", followRedirect=True)
472 self.failUnless('Upload and Download Status' in res, res)
473 self.failUnless('"down-%d"' % dl_num in res, res)
474 self.failUnless('"up-%d"' % ul_num in res, res)
475 self.failUnless('"mapupdate-%d"' % mu_num in res, res)
476 self.failUnless('"publish-%d"' % pub_num in res, res)
477 self.failUnless('"retrieve-%d"' % ret_num in res, res)
478 d.addCallback(_check)
479 d.addCallback(lambda res: self.GET("/status/?t=json"))
480 def _check_json(res):
481 data = simplejson.loads(res)
482 self.failUnless(isinstance(data, dict))
483 active = data["active"]
484 # TODO: test more. We need a way to fake an active operation
486 d.addCallback(_check_json)
488 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
490 self.failUnless("File Download Status" in res, res)
491 d.addCallback(_check_dl)
492 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
494 self.failUnless("File Upload Status" in res, res)
495 d.addCallback(_check_ul)
496 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
497 def _check_mapupdate(res):
498 self.failUnless("Mutable File Servermap Update Status" in res, res)
499 d.addCallback(_check_mapupdate)
500 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
501 def _check_publish(res):
502 self.failUnless("Mutable File Publish Status" in res, res)
503 d.addCallback(_check_publish)
504 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
505 def _check_retrieve(res):
506 self.failUnless("Mutable File Retrieve Status" in res, res)
507 d.addCallback(_check_retrieve)
511 def test_status_numbers(self):
512 drrm = status.DownloadResultsRendererMixin()
513 self.failUnlessEqual(drrm.render_time(None, None), "")
514 self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
515 self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
516 self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
517 self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
518 self.failUnlessEqual(drrm.render_rate(None, None), "")
519 self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
520 self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
521 self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
523 urrm = status.UploadResultsRendererMixin()
524 self.failUnlessEqual(urrm.render_time(None, None), "")
525 self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
526 self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
527 self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
528 self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
529 self.failUnlessEqual(urrm.render_rate(None, None), "")
530 self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
531 self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
532 self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
534 def test_GET_FILEURL(self):
535 d = self.GET(self.public_url + "/foo/bar.txt")
536 d.addCallback(self.failUnlessIsBarDotTxt)
539 def test_GET_FILEURL_range(self):
540 headers = {"range": "bytes=1-10"}
541 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
542 return_response=True)
543 def _got((res, status, headers)):
544 self.failUnlessEqual(int(status), 206)
545 self.failUnless(headers.has_key("content-range"))
546 self.failUnlessEqual(headers["content-range"][0],
547 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
548 self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
552 def test_HEAD_FILEURL_range(self):
553 headers = {"range": "bytes=1-10"}
554 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
555 return_response=True)
556 def _got((res, status, headers)):
557 self.failUnlessEqual(res, "")
558 self.failUnlessEqual(int(status), 206)
559 self.failUnless(headers.has_key("content-range"))
560 self.failUnlessEqual(headers["content-range"][0],
561 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
565 def test_GET_FILEURL_range_bad(self):
566 headers = {"range": "BOGUS=fizbop-quarnak"}
567 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
569 "Syntactically invalid http range header",
570 self.GET, self.public_url + "/foo/bar.txt",
574 def test_HEAD_FILEURL(self):
575 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
576 def _got((res, status, headers)):
577 self.failUnlessEqual(res, "")
578 self.failUnlessEqual(headers["content-length"][0],
579 str(len(self.BAR_CONTENTS)))
580 self.failUnlessEqual(headers["content-type"], ["text/plain"])
584 def test_GET_FILEURL_named(self):
585 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
586 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
587 d = self.GET(base + "/@@name=/blah.txt")
588 d.addCallback(self.failUnlessIsBarDotTxt)
589 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
590 d.addCallback(self.failUnlessIsBarDotTxt)
591 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
592 d.addCallback(self.failUnlessIsBarDotTxt)
593 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
594 d.addCallback(self.failUnlessIsBarDotTxt)
595 save_url = base + "?save=true&filename=blah.txt"
596 d.addCallback(lambda res: self.GET(save_url))
597 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
598 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
599 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
600 u_url = base + "?save=true&filename=" + u_fn_e
601 d.addCallback(lambda res: self.GET(u_url))
602 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
605 def test_PUT_FILEURL_named_bad(self):
606 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
607 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
609 "/file can only be used with GET or HEAD",
610 self.PUT, base + "/@@name=/blah.txt", "")
613 def test_GET_DIRURL_named_bad(self):
614 base = "/file/%s" % urllib.quote(self._foo_uri)
615 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
618 self.GET, base + "/@@name=/blah.txt")
621 def test_GET_slash_file_bad(self):
622 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
624 "/file must be followed by a file-cap and a name",
628 def test_GET_unhandled_URI_named(self):
629 contents, n, newuri = self.makefile(12)
630 verifier_cap = n.get_verifier().to_string()
631 base = "/file/%s" % urllib.quote(verifier_cap)
632 # client.create_node_from_uri() can't handle verify-caps
633 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
635 "is not a valid file- or directory- cap",
639 def test_GET_unhandled_URI(self):
640 contents, n, newuri = self.makefile(12)
641 verifier_cap = n.get_verifier().to_string()
642 base = "/uri/%s" % urllib.quote(verifier_cap)
643 # client.create_node_from_uri() can't handle verify-caps
644 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
646 "is not a valid file- or directory- cap",
650 def test_GET_FILE_URI(self):
651 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
653 d.addCallback(self.failUnlessIsBarDotTxt)
656 def test_GET_FILE_URI_badchild(self):
657 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
658 errmsg = "Files have no children, certainly not named 'boguschild'"
659 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
660 "400 Bad Request", errmsg,
664 def test_PUT_FILE_URI_badchild(self):
665 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
666 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
667 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
668 "400 Bad Request", errmsg,
672 def test_GET_FILEURL_save(self):
673 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
674 # TODO: look at the headers, expect a Content-Disposition: attachment
676 d.addCallback(self.failUnlessIsBarDotTxt)
679 def test_GET_FILEURL_missing(self):
680 d = self.GET(self.public_url + "/foo/missing")
681 d.addBoth(self.should404, "test_GET_FILEURL_missing")
684 def test_PUT_NEWFILEURL(self):
685 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
686 # TODO: we lose the response code, so we can't check this
687 #self.failUnlessEqual(responsecode, 201)
688 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
689 d.addCallback(lambda res:
690 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
691 self.NEWFILE_CONTENTS))
694 def test_PUT_NEWFILEURL_range_bad(self):
695 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
696 target = self.public_url + "/foo/new.txt"
697 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
698 "501 Not Implemented",
699 "Content-Range in PUT not yet supported",
700 # (and certainly not for immutable files)
701 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
703 d.addCallback(lambda res:
704 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
707 def test_PUT_NEWFILEURL_mutable(self):
708 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
709 self.NEWFILE_CONTENTS)
710 # TODO: we lose the response code, so we can't check this
711 #self.failUnlessEqual(responsecode, 201)
713 u = uri.from_string_mutable_filenode(res)
714 self.failUnless(u.is_mutable())
715 self.failIf(u.is_readonly())
717 d.addCallback(_check_uri)
718 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
719 d.addCallback(lambda res:
720 self.failUnlessMutableChildContentsAre(self._foo_node,
722 self.NEWFILE_CONTENTS))
725 def test_PUT_NEWFILEURL_mutable_toobig(self):
726 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
727 "413 Request Entity Too Large",
728 "SDMF is limited to one segment, and 10001 > 10000",
730 self.public_url + "/foo/new.txt?mutable=true",
731 "b" * (self.s.MUTABLE_SIZELIMIT+1))
734 def test_PUT_NEWFILEURL_replace(self):
735 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
736 # TODO: we lose the response code, so we can't check this
737 #self.failUnlessEqual(responsecode, 200)
738 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
739 d.addCallback(lambda res:
740 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
741 self.NEWFILE_CONTENTS))
744 def test_PUT_NEWFILEURL_bad_t(self):
745 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
746 "PUT to a file: bad t=bogus",
747 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
751 def test_PUT_NEWFILEURL_no_replace(self):
752 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
753 self.NEWFILE_CONTENTS)
754 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
756 "There was already a child by that name, and you asked me "
760 def test_PUT_NEWFILEURL_mkdirs(self):
761 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
763 d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
764 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
765 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
766 d.addCallback(lambda res:
767 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
768 self.NEWFILE_CONTENTS))
771 def test_PUT_NEWFILEURL_blocked(self):
772 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
773 self.NEWFILE_CONTENTS)
774 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
776 "Unable to create directory 'blockingfile': a file was in the way")
779 def test_DELETE_FILEURL(self):
780 d = self.DELETE(self.public_url + "/foo/bar.txt")
781 d.addCallback(lambda res:
782 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
785 def test_DELETE_FILEURL_missing(self):
786 d = self.DELETE(self.public_url + "/foo/missing")
787 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
790 def test_DELETE_FILEURL_missing2(self):
791 d = self.DELETE(self.public_url + "/missing/missing")
792 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
795 def test_GET_FILEURL_json(self):
796 # twisted.web.http.parse_qs ignores any query args without an '=', so
797 # I can't do "GET /path?json", I have to do "GET /path/t=json"
798 # instead. This may make it tricky to emulate the S3 interface
800 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
801 d.addCallback(self.failUnlessIsBarJSON)
804 def test_GET_FILEURL_json_missing(self):
805 d = self.GET(self.public_url + "/foo/missing?json")
806 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
809 def test_GET_FILEURL_uri(self):
810 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
812 self.failUnlessEqual(res, self._bar_txt_uri)
813 d.addCallback(_check)
814 d.addCallback(lambda res:
815 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
817 # for now, for files, uris and readonly-uris are the same
818 self.failUnlessEqual(res, self._bar_txt_uri)
819 d.addCallback(_check2)
822 def test_GET_FILEURL_badtype(self):
823 d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
826 self.public_url + "/foo/bar.txt?t=bogus")
829 def test_GET_FILEURL_uri_missing(self):
830 d = self.GET(self.public_url + "/foo/missing?t=uri")
831 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
834 def test_GET_DIRURL(self):
835 # the addSlash means we get a redirect here
836 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
838 d = self.GET(self.public_url + "/foo", followRedirect=True)
840 self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
842 # the FILE reference points to a URI, but it should end in bar.txt
843 bar_url = ("%s/file/%s/@@named=/bar.txt" %
844 (ROOT, urllib.quote(self._bar_txt_uri)))
845 get_bar = "".join([r'<td>',
846 r'<a href="%s">bar.txt</a>' % bar_url,
849 r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
851 self.failUnless(re.search(get_bar, res), res)
852 for line in res.split("\n"):
853 # find the line that contains the delete button for bar.txt
854 if ("form action" in line and
855 'value="delete"' in line and
856 'value="bar.txt"' in line):
857 # the form target should use a relative URL
858 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
859 self.failUnless(('action="%s"' % foo_url) in line, line)
860 # and the when_done= should too
861 #done_url = urllib.quote(???)
862 #self.failUnless(('name="when_done" value="%s"' % done_url)
866 self.fail("unable to find delete-bar.txt line", res)
868 # the DIR reference just points to a URI
869 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
870 get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
871 + r'\s+<td>DIR</td>')
872 self.failUnless(re.search(get_sub, res), res)
873 d.addCallback(_check)
875 # look at a directory which is readonly
876 d.addCallback(lambda res:
877 self.GET(self.public_url + "/reedownlee", followRedirect=True))
879 self.failUnless("(readonly)" in res, res)
880 self.failIf("Upload a file" in res, res)
881 d.addCallback(_check2)
883 # and at a directory that contains a readonly directory
884 d.addCallback(lambda res:
885 self.GET(self.public_url, followRedirect=True))
887 self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
888 '</td>\s+<td>DIR-RO</td>', res))
889 d.addCallback(_check3)
893 def test_GET_DIRURL_badtype(self):
894 d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
898 self.public_url + "/foo?t=bogus")
901 def test_GET_DIRURL_json(self):
902 d = self.GET(self.public_url + "/foo?t=json")
903 d.addCallback(self.failUnlessIsFooJSON)
907 def test_POST_DIRURL_manifest_no_ophandle(self):
908 d = self.shouldFail2(error.Error,
909 "test_POST_DIRURL_manifest_no_ophandle",
911 "slow operation requires ophandle=",
912 self.POST, self.public_url, t="start-manifest")
915 def test_POST_DIRURL_manifest(self):
916 d = defer.succeed(None)
917 def getman(ignored, output):
918 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
920 d.addCallback(self.wait_for_operation, "125")
921 d.addCallback(self.get_operation_results, "125", output)
923 d.addCallback(getman, None)
924 def _got_html(manifest):
925 self.failUnless("Manifest of SI=" in manifest)
926 self.failUnless("<td>sub</td>" in manifest)
927 self.failUnless(self._sub_uri in manifest)
928 self.failUnless("<td>sub/baz.txt</td>" in manifest)
929 d.addCallback(_got_html)
931 # both t=status and unadorned GET should be identical
932 d.addCallback(lambda res: self.GET("/operations/125"))
933 d.addCallback(_got_html)
935 d.addCallback(getman, "html")
936 d.addCallback(_got_html)
937 d.addCallback(getman, "text")
938 def _got_text(manifest):
939 self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
940 self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
941 d.addCallback(_got_text)
942 d.addCallback(getman, "JSON")
943 def _got_json(manifest):
944 data = manifest["manifest"]
946 for (path_list, cap) in data:
947 got[tuple(path_list)] = cap
948 self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
949 self.failUnless((u"sub",u"baz.txt") in got)
950 d.addCallback(_got_json)
953 def test_POST_DIRURL_deepsize_no_ophandle(self):
954 d = self.shouldFail2(error.Error,
955 "test_POST_DIRURL_deepsize_no_ophandle",
957 "slow operation requires ophandle=",
958 self.POST, self.public_url, t="start-deep-size")
961 def test_POST_DIRURL_deepsize(self):
962 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
964 d.addCallback(self.wait_for_operation, "126")
965 d.addCallback(self.get_operation_results, "126", "json")
967 self.failUnlessEqual(data["finished"], True)
969 self.failUnless(size > 1000)
970 d.addCallback(_got_json)
971 d.addCallback(self.get_operation_results, "126", "text")
973 mo = re.search(r'^size: (\d+)$', res, re.M)
974 self.failUnless(mo, res)
975 size = int(mo.group(1))
976 # with directories, the size varies.
977 self.failUnless(size > 1000)
978 d.addCallback(_got_text)
981 def test_POST_DIRURL_deepstats_no_ophandle(self):
982 d = self.shouldFail2(error.Error,
983 "test_POST_DIRURL_deepstats_no_ophandle",
985 "slow operation requires ophandle=",
986 self.POST, self.public_url, t="start-deep-stats")
989 def test_POST_DIRURL_deepstats(self):
990 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
992 d.addCallback(self.wait_for_operation, "127")
993 d.addCallback(self.get_operation_results, "127", "json")
994 def _got_json(stats):
995 expected = {"count-immutable-files": 3,
996 "count-mutable-files": 0,
997 "count-literal-files": 0,
999 "count-directories": 3,
1000 "size-immutable-files": 57,
1001 "size-literal-files": 0,
1002 #"size-directories": 1912, # varies
1003 #"largest-directory": 1590,
1004 "largest-directory-children": 5,
1005 "largest-immutable-file": 19,
1007 for k,v in expected.iteritems():
1008 self.failUnlessEqual(stats[k], v,
1009 "stats[%s] was %s, not %s" %
1011 self.failUnlessEqual(stats["size-files-histogram"],
1013 d.addCallback(_got_json)
1016 def test_GET_DIRURL_uri(self):
1017 d = self.GET(self.public_url + "/foo?t=uri")
1019 self.failUnlessEqual(res, self._foo_uri)
1020 d.addCallback(_check)
1023 def test_GET_DIRURL_readonly_uri(self):
1024 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1026 self.failUnlessEqual(res, self._foo_readonly_uri)
1027 d.addCallback(_check)
1030 def test_PUT_NEWDIRURL(self):
1031 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1032 d.addCallback(lambda res:
1033 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1034 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1035 d.addCallback(self.failUnlessNodeKeysAre, [])
1038 def test_PUT_NEWDIRURL_exists(self):
1039 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1040 d.addCallback(lambda res:
1041 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1042 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1043 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1046 def test_PUT_NEWDIRURL_blocked(self):
1047 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1048 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1050 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1051 d.addCallback(lambda res:
1052 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1053 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1054 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1057 def test_PUT_NEWDIRURL_mkdir_p(self):
1058 d = defer.succeed(None)
1059 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1060 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1061 d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1062 def mkdir_p(mkpnode):
1063 url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1065 def made_subsub(ssuri):
1066 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1067 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1069 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1071 d.addCallback(made_subsub)
1073 d.addCallback(mkdir_p)
1076 def test_PUT_NEWDIRURL_mkdirs(self):
1077 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1078 d.addCallback(lambda res:
1079 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1080 d.addCallback(lambda res:
1081 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1082 d.addCallback(lambda res:
1083 self._foo_node.get_child_at_path(u"subdir/newdir"))
1084 d.addCallback(self.failUnlessNodeKeysAre, [])
1087 def test_DELETE_DIRURL(self):
1088 d = self.DELETE(self.public_url + "/foo")
1089 d.addCallback(lambda res:
1090 self.failIfNodeHasChild(self.public_root, u"foo"))
1093 def test_DELETE_DIRURL_missing(self):
1094 d = self.DELETE(self.public_url + "/foo/missing")
1095 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1096 d.addCallback(lambda res:
1097 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1100 def test_DELETE_DIRURL_missing2(self):
1101 d = self.DELETE(self.public_url + "/missing")
1102 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1105 def dump_root(self):
1107 w = webish.DirnodeWalkerMixin()
1108 def visitor(childpath, childnode, metadata):
1110 d = w.walk(self.public_root, visitor)
1113 def failUnlessNodeKeysAre(self, node, expected_keys):
1114 for k in expected_keys:
1115 assert isinstance(k, unicode)
1117 def _check(children):
1118 self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1119 d.addCallback(_check)
1121 def failUnlessNodeHasChild(self, node, name):
1122 assert isinstance(name, unicode)
1124 def _check(children):
1125 self.failUnless(name in children)
1126 d.addCallback(_check)
1128 def failIfNodeHasChild(self, node, name):
1129 assert isinstance(name, unicode)
1131 def _check(children):
1132 self.failIf(name in children)
1133 d.addCallback(_check)
1136 def failUnlessChildContentsAre(self, node, name, expected_contents):
1137 assert isinstance(name, unicode)
1138 d = node.get_child_at_path(name)
1139 d.addCallback(lambda node: node.download_to_data())
1140 def _check(contents):
1141 self.failUnlessEqual(contents, expected_contents)
1142 d.addCallback(_check)
1145 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1146 assert isinstance(name, unicode)
1147 d = node.get_child_at_path(name)
1148 d.addCallback(lambda node: node.download_best_version())
1149 def _check(contents):
1150 self.failUnlessEqual(contents, expected_contents)
1151 d.addCallback(_check)
1154 def failUnlessChildURIIs(self, node, name, expected_uri):
1155 assert isinstance(name, unicode)
1156 d = node.get_child_at_path(name)
1158 self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1159 d.addCallback(_check)
1162 def failUnlessURIMatchesChild(self, got_uri, node, name):
1163 assert isinstance(name, unicode)
1164 d = node.get_child_at_path(name)
1166 self.failUnlessEqual(got_uri.strip(), child.get_uri())
1167 d.addCallback(_check)
1170 def failUnlessCHKURIHasContents(self, got_uri, contents):
1171 self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1173 def test_POST_upload(self):
1174 d = self.POST(self.public_url + "/foo", t="upload",
1175 file=("new.txt", self.NEWFILE_CONTENTS))
1177 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1178 d.addCallback(lambda res:
1179 self.failUnlessChildContentsAre(fn, u"new.txt",
1180 self.NEWFILE_CONTENTS))
1183 def test_POST_upload_unicode(self):
1184 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1185 d = self.POST(self.public_url + "/foo", t="upload",
1186 file=(filename, self.NEWFILE_CONTENTS))
1188 d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1189 d.addCallback(lambda res:
1190 self.failUnlessChildContentsAre(fn, filename,
1191 self.NEWFILE_CONTENTS))
1192 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1193 d.addCallback(lambda res: self.GET(target_url))
1194 d.addCallback(lambda contents: self.failUnlessEqual(contents,
1195 self.NEWFILE_CONTENTS,
1199 def test_POST_upload_unicode_named(self):
1200 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1201 d = self.POST(self.public_url + "/foo", t="upload",
1203 file=("overridden", self.NEWFILE_CONTENTS))
1205 d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1206 d.addCallback(lambda res:
1207 self.failUnlessChildContentsAre(fn, filename,
1208 self.NEWFILE_CONTENTS))
1209 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1210 d.addCallback(lambda res: self.GET(target_url))
1211 d.addCallback(lambda contents: self.failUnlessEqual(contents,
1212 self.NEWFILE_CONTENTS,
1216 def test_POST_upload_no_link(self):
1217 d = self.POST("/uri", t="upload",
1218 file=("new.txt", self.NEWFILE_CONTENTS))
1219 def _check_upload_results(page):
1220 # this should be a page which describes the results of the upload
1221 # that just finished.
1222 self.failUnless("Upload Results:" in page)
1223 self.failUnless("URI:" in page)
1224 uri_re = re.compile("URI: <tt><span>(.*)</span>")
1225 mo = uri_re.search(page)
1226 self.failUnless(mo, page)
1227 new_uri = mo.group(1)
1229 d.addCallback(_check_upload_results)
1230 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1233 def test_POST_upload_no_link_whendone(self):
1234 d = self.POST("/uri", t="upload", when_done="/",
1235 file=("new.txt", self.NEWFILE_CONTENTS))
1236 d.addBoth(self.shouldRedirect, "/")
1239 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1240 d = defer.maybeDeferred(callable, *args, **kwargs)
1242 if isinstance(res, failure.Failure):
1243 res.trap(error.PageRedirect)
1244 statuscode = res.value.status
1245 target = res.value.location
1246 return checker(statuscode, target)
1247 self.fail("%s: callable was supposed to redirect, not return '%s'"
1252 def test_POST_upload_no_link_whendone_results(self):
1253 def check(statuscode, target):
1254 self.failUnlessEqual(statuscode, str(http.FOUND))
1255 self.failUnless(target.startswith(self.webish_url), target)
1256 return client.getPage(target, method="GET")
1257 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1259 self.POST, "/uri", t="upload",
1260 when_done="/uri/%(uri)s",
1261 file=("new.txt", self.NEWFILE_CONTENTS))
1262 d.addCallback(lambda res:
1263 self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1266 def test_POST_upload_no_link_mutable(self):
1267 d = self.POST("/uri", t="upload", mutable="true",
1268 file=("new.txt", self.NEWFILE_CONTENTS))
1269 def _check(new_uri):
1270 new_uri = new_uri.strip()
1271 self.new_uri = new_uri
1273 self.failUnless(IMutableFileURI.providedBy(u))
1274 self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1275 n = self.s.create_node_from_uri(new_uri)
1276 return n.download_best_version()
1277 d.addCallback(_check)
1279 self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1280 return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1281 d.addCallback(_check2)
1283 self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1284 return self.GET("/file/%s" % urllib.quote(self.new_uri))
1285 d.addCallback(_check3)
1287 self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1288 d.addCallback(_check4)
1291 def test_POST_upload_no_link_mutable_toobig(self):
1292 d = self.shouldFail2(error.Error,
1293 "test_POST_upload_no_link_mutable_toobig",
1294 "413 Request Entity Too Large",
1295 "SDMF is limited to one segment, and 10001 > 10000",
1297 "/uri", t="upload", mutable="true",
1299 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1302 def test_POST_upload_mutable(self):
1303 # this creates a mutable file
1304 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1305 file=("new.txt", self.NEWFILE_CONTENTS))
1307 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1308 d.addCallback(lambda res:
1309 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1310 self.NEWFILE_CONTENTS))
1311 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1313 self.failUnless(IMutableFileNode.providedBy(newnode))
1314 self.failUnless(newnode.is_mutable())
1315 self.failIf(newnode.is_readonly())
1316 self._mutable_node = newnode
1317 self._mutable_uri = newnode.get_uri()
1320 # now upload it again and make sure that the URI doesn't change
1321 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1322 d.addCallback(lambda res:
1323 self.POST(self.public_url + "/foo", t="upload",
1325 file=("new.txt", NEWER_CONTENTS)))
1326 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1327 d.addCallback(lambda res:
1328 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1330 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1332 self.failUnless(IMutableFileNode.providedBy(newnode))
1333 self.failUnless(newnode.is_mutable())
1334 self.failIf(newnode.is_readonly())
1335 self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1336 d.addCallback(_got2)
1338 # upload a second time, using PUT instead of POST
1339 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1340 d.addCallback(lambda res:
1341 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1342 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1343 d.addCallback(lambda res:
1344 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1347 # finally list the directory, since mutable files are displayed
1348 # slightly differently
1350 d.addCallback(lambda res:
1351 self.GET(self.public_url + "/foo/",
1352 followRedirect=True))
1353 def _check_page(res):
1354 # TODO: assert more about the contents
1355 self.failUnless("SSK" in res)
1357 d.addCallback(_check_page)
1359 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1361 self.failUnless(IMutableFileNode.providedBy(newnode))
1362 self.failUnless(newnode.is_mutable())
1363 self.failIf(newnode.is_readonly())
1364 self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1365 d.addCallback(_got3)
1367 # look at the JSON form of the enclosing directory
1368 d.addCallback(lambda res:
1369 self.GET(self.public_url + "/foo/?t=json",
1370 followRedirect=True))
1371 def _check_page_json(res):
1372 parsed = simplejson.loads(res)
1373 self.failUnlessEqual(parsed[0], "dirnode")
1374 children = dict( [(unicode(name),value)
1376 in parsed[1]["children"].iteritems()] )
1377 self.failUnless("new.txt" in children)
1378 new_json = children["new.txt"]
1379 self.failUnlessEqual(new_json[0], "filenode")
1380 self.failUnless(new_json[1]["mutable"])
1381 self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1382 ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1383 self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1384 d.addCallback(_check_page_json)
1386 # and the JSON form of the file
1387 d.addCallback(lambda res:
1388 self.GET(self.public_url + "/foo/new.txt?t=json"))
1389 def _check_file_json(res):
1390 parsed = simplejson.loads(res)
1391 self.failUnlessEqual(parsed[0], "filenode")
1392 self.failUnless(parsed[1]["mutable"])
1393 self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1394 ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1395 self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1396 d.addCallback(_check_file_json)
1398 # and look at t=uri and t=readonly-uri
1399 d.addCallback(lambda res:
1400 self.GET(self.public_url + "/foo/new.txt?t=uri"))
1401 d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1402 d.addCallback(lambda res:
1403 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1404 def _check_ro_uri(res):
1405 ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1406 self.failUnlessEqual(res, ro_uri)
1407 d.addCallback(_check_ro_uri)
1409 # make sure we can get to it from /uri/URI
1410 d.addCallback(lambda res:
1411 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1412 d.addCallback(lambda res:
1413 self.failUnlessEqual(res, NEW2_CONTENTS))
1415 # and that HEAD computes the size correctly
1416 d.addCallback(lambda res:
1417 self.HEAD(self.public_url + "/foo/new.txt",
1418 return_response=True))
1419 def _got_headers((res, status, headers)):
1420 self.failUnlessEqual(res, "")
1421 self.failUnlessEqual(headers["content-length"][0],
1422 str(len(NEW2_CONTENTS)))
1423 self.failUnlessEqual(headers["content-type"], ["text/plain"])
1424 d.addCallback(_got_headers)
1426 # make sure that size errors are displayed correctly for overwrite
1427 d.addCallback(lambda res:
1428 self.shouldFail2(error.Error,
1429 "test_POST_upload_mutable-toobig",
1430 "413 Request Entity Too Large",
1431 "SDMF is limited to one segment, and 10001 > 10000",
1433 self.public_url + "/foo", t="upload",
1436 "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1439 d.addErrback(self.dump_error)
1442 def test_POST_upload_mutable_toobig(self):
1443 d = self.shouldFail2(error.Error,
1444 "test_POST_upload_no_link_mutable_toobig",
1445 "413 Request Entity Too Large",
1446 "SDMF is limited to one segment, and 10001 > 10000",
1448 self.public_url + "/foo",
1449 t="upload", mutable="true",
1451 "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1454 def dump_error(self, f):
1455 # if the web server returns an error code (like 400 Bad Request),
1456 # web.client.getPage puts the HTTP response body into the .response
1457 # attribute of the exception object that it gives back. It does not
1458 # appear in the Failure's repr(), so the ERROR that trial displays
1459 # will be rather terse and unhelpful. addErrback this method to the
1460 # end of your chain to get more information out of these errors.
1461 if f.check(error.Error):
1462 print "web.error.Error:"
1464 print f.value.response
1467 def test_POST_upload_replace(self):
1468 d = self.POST(self.public_url + "/foo", t="upload",
1469 file=("bar.txt", self.NEWFILE_CONTENTS))
1471 d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1472 d.addCallback(lambda res:
1473 self.failUnlessChildContentsAre(fn, u"bar.txt",
1474 self.NEWFILE_CONTENTS))
1477 def test_POST_upload_no_replace_ok(self):
1478 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1479 file=("new.txt", self.NEWFILE_CONTENTS))
1480 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1481 d.addCallback(lambda res: self.failUnlessEqual(res,
1482 self.NEWFILE_CONTENTS))
1485 def test_POST_upload_no_replace_queryarg(self):
1486 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1487 file=("bar.txt", self.NEWFILE_CONTENTS))
1488 d.addBoth(self.shouldFail, error.Error,
1489 "POST_upload_no_replace_queryarg",
1491 "There was already a child by that name, and you asked me "
1492 "to not replace it")
1493 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1494 d.addCallback(self.failUnlessIsBarDotTxt)
1497 def test_POST_upload_no_replace_field(self):
1498 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1499 file=("bar.txt", self.NEWFILE_CONTENTS))
1500 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1502 "There was already a child by that name, and you asked me "
1503 "to not replace it")
1504 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1505 d.addCallback(self.failUnlessIsBarDotTxt)
1508 def test_POST_upload_whendone(self):
1509 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1510 file=("new.txt", self.NEWFILE_CONTENTS))
1511 d.addBoth(self.shouldRedirect, "/THERE")
1513 d.addCallback(lambda res:
1514 self.failUnlessChildContentsAre(fn, u"new.txt",
1515 self.NEWFILE_CONTENTS))
1518 def test_POST_upload_named(self):
1520 d = self.POST(self.public_url + "/foo", t="upload",
1521 name="new.txt", file=self.NEWFILE_CONTENTS)
1522 d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1523 d.addCallback(lambda res:
1524 self.failUnlessChildContentsAre(fn, u"new.txt",
1525 self.NEWFILE_CONTENTS))
1528 def test_POST_upload_named_badfilename(self):
1529 d = self.POST(self.public_url + "/foo", t="upload",
1530 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1531 d.addBoth(self.shouldFail, error.Error,
1532 "test_POST_upload_named_badfilename",
1534 "name= may not contain a slash",
1536 # make sure that nothing was added
1537 d.addCallback(lambda res:
1538 self.failUnlessNodeKeysAre(self._foo_node,
1539 [u"bar.txt", u"blockingfile",
1540 u"empty", u"n\u00fc.txt",
1544 def test_POST_FILEURL_check(self):
1545 bar_url = self.public_url + "/foo/bar.txt"
1546 d = self.POST(bar_url, t="check")
1548 self.failUnless("Healthy :" in res)
1549 d.addCallback(_check)
1550 redir_url = "http://allmydata.org/TARGET"
1551 def _check2(statuscode, target):
1552 self.failUnlessEqual(statuscode, str(http.FOUND))
1553 self.failUnlessEqual(target, redir_url)
1554 d.addCallback(lambda res:
1555 self.shouldRedirect2("test_POST_FILEURL_check",
1559 when_done=redir_url))
1560 d.addCallback(lambda res:
1561 self.POST(bar_url, t="check", return_to=redir_url))
1563 self.failUnless("Healthy :" in res)
1564 self.failUnless("Return to parent directory" in res)
1565 self.failUnless(redir_url in res)
1566 d.addCallback(_check3)
1568 d.addCallback(lambda res:
1569 self.POST(bar_url, t="check", output="JSON"))
1570 def _check_json(res):
1571 data = simplejson.loads(res)
1572 self.failUnless("storage-index" in data)
1573 self.failUnless(data["results"]["healthy"])
1574 d.addCallback(_check_json)
1578 def test_POST_FILEURL_check_and_repair(self):
1579 bar_url = self.public_url + "/foo/bar.txt"
1580 d = self.POST(bar_url, t="check", repair="true")
1582 self.failUnless("Healthy :" in res)
1583 d.addCallback(_check)
1584 redir_url = "http://allmydata.org/TARGET"
1585 def _check2(statuscode, target):
1586 self.failUnlessEqual(statuscode, str(http.FOUND))
1587 self.failUnlessEqual(target, redir_url)
1588 d.addCallback(lambda res:
1589 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1592 t="check", repair="true",
1593 when_done=redir_url))
1594 d.addCallback(lambda res:
1595 self.POST(bar_url, t="check", return_to=redir_url))
1597 self.failUnless("Healthy :" in res)
1598 self.failUnless("Return to parent directory" in res)
1599 self.failUnless(redir_url in res)
1600 d.addCallback(_check3)
1603 def test_POST_DIRURL_check(self):
1604 foo_url = self.public_url + "/foo/"
1605 d = self.POST(foo_url, t="check")
1607 self.failUnless("Healthy :" in res, res)
1608 d.addCallback(_check)
1609 redir_url = "http://allmydata.org/TARGET"
1610 def _check2(statuscode, target):
1611 self.failUnlessEqual(statuscode, str(http.FOUND))
1612 self.failUnlessEqual(target, redir_url)
1613 d.addCallback(lambda res:
1614 self.shouldRedirect2("test_POST_DIRURL_check",
1618 when_done=redir_url))
1619 d.addCallback(lambda res:
1620 self.POST(foo_url, t="check", return_to=redir_url))
1622 self.failUnless("Healthy :" in res, res)
1623 self.failUnless("Return to parent directory" in res)
1624 self.failUnless(redir_url in res)
1625 d.addCallback(_check3)
1627 d.addCallback(lambda res:
1628 self.POST(foo_url, t="check", output="JSON"))
1629 def _check_json(res):
1630 data = simplejson.loads(res)
1631 self.failUnless("storage-index" in data)
1632 self.failUnless(data["results"]["healthy"])
1633 d.addCallback(_check_json)
1637 def test_POST_DIRURL_check_and_repair(self):
1638 foo_url = self.public_url + "/foo/"
1639 d = self.POST(foo_url, t="check", repair="true")
1641 self.failUnless("Healthy :" in res, res)
1642 d.addCallback(_check)
1643 redir_url = "http://allmydata.org/TARGET"
1644 def _check2(statuscode, target):
1645 self.failUnlessEqual(statuscode, str(http.FOUND))
1646 self.failUnlessEqual(target, redir_url)
1647 d.addCallback(lambda res:
1648 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1651 t="check", repair="true",
1652 when_done=redir_url))
1653 d.addCallback(lambda res:
1654 self.POST(foo_url, t="check", return_to=redir_url))
1656 self.failUnless("Healthy :" in res)
1657 self.failUnless("Return to parent directory" in res)
1658 self.failUnless(redir_url in res)
1659 d.addCallback(_check3)
1662 def wait_for_operation(self, ignored, ophandle):
1663 url = "/operations/" + ophandle
1664 url += "?t=status&output=JSON"
1667 data = simplejson.loads(res)
1668 if not data["finished"]:
1669 d = self.stall(delay=1.0)
1670 d.addCallback(self.wait_for_operation, ophandle)
1676 def get_operation_results(self, ignored, ophandle, output=None):
1677 url = "/operations/" + ophandle
1680 url += "&output=" + output
1683 if output and output.lower() == "json":
1684 return simplejson.loads(res)
1689 def test_POST_DIRURL_deepcheck_no_ophandle(self):
1690 d = self.shouldFail2(error.Error,
1691 "test_POST_DIRURL_deepcheck_no_ophandle",
1693 "slow operation requires ophandle=",
1694 self.POST, self.public_url, t="start-deep-check")
1697 def test_POST_DIRURL_deepcheck(self):
1698 def _check_redirect(statuscode, target):
1699 self.failUnlessEqual(statuscode, str(http.FOUND))
1700 self.failUnless(target.endswith("/operations/123"))
1701 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1702 self.POST, self.public_url,
1703 t="start-deep-check", ophandle="123")
1704 d.addCallback(self.wait_for_operation, "123")
1705 def _check_json(data):
1706 self.failUnlessEqual(data["finished"], True)
1707 self.failUnlessEqual(data["count-objects-checked"], 8)
1708 self.failUnlessEqual(data["count-objects-healthy"], 8)
1709 d.addCallback(_check_json)
1710 d.addCallback(self.get_operation_results, "123", "html")
1711 def _check_html(res):
1712 self.failUnless("Objects Checked: <span>8</span>" in res)
1713 self.failUnless("Objects Healthy: <span>8</span>" in res)
1714 d.addCallback(_check_html)
1716 d.addCallback(lambda res:
1717 self.GET("/operations/123/"))
1718 d.addCallback(_check_html) # should be the same as without the slash
1720 d.addCallback(lambda res:
1721 self.shouldFail2(error.Error, "one", "404 Not Found",
1722 "No detailed results for SI bogus",
1723 self.GET, "/operations/123/bogus"))
1725 foo_si = self._foo_node.get_storage_index()
1726 foo_si_s = base32.b2a(foo_si)
1727 d.addCallback(lambda res:
1728 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1729 def _check_foo_json(res):
1730 data = simplejson.loads(res)
1731 self.failUnlessEqual(data["storage-index"], foo_si_s)
1732 self.failUnless(data["results"]["healthy"])
1733 d.addCallback(_check_foo_json)
1736 def test_POST_DIRURL_deepcheck_and_repair(self):
1737 d = self.POST(self.public_url, t="start-deep-check", repair="true",
1738 ophandle="124", output="json", followRedirect=True)
1739 d.addCallback(self.wait_for_operation, "124")
1740 def _check_json(data):
1741 self.failUnlessEqual(data["finished"], True)
1742 self.failUnlessEqual(data["count-objects-checked"], 8)
1743 self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1744 self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1745 self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1746 self.failUnlessEqual(data["count-repairs-attempted"], 0)
1747 self.failUnlessEqual(data["count-repairs-successful"], 0)
1748 self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1749 self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1750 self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1751 self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1752 d.addCallback(_check_json)
1753 d.addCallback(self.get_operation_results, "124", "html")
1754 def _check_html(res):
1755 self.failUnless("Objects Checked: <span>8</span>" in res)
1757 self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1758 self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1759 self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1761 self.failUnless("Repairs Attempted: <span>0</span>" in res)
1762 self.failUnless("Repairs Successful: <span>0</span>" in res)
1763 self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1765 self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1766 self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1767 self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1768 d.addCallback(_check_html)
1771 def test_POST_FILEURL_bad_t(self):
1772 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1773 "POST to file: bad t=bogus",
1774 self.POST, self.public_url + "/foo/bar.txt",
1778 def test_POST_mkdir(self): # return value?
1779 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1780 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1781 d.addCallback(self.failUnlessNodeKeysAre, [])
1784 def test_POST_mkdir_2(self):
1785 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1786 d.addCallback(lambda res:
1787 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1788 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1789 d.addCallback(self.failUnlessNodeKeysAre, [])
1792 def test_POST_mkdirs_2(self):
1793 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1794 d.addCallback(lambda res:
1795 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1796 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1797 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1798 d.addCallback(self.failUnlessNodeKeysAre, [])
1801 def test_POST_mkdir_no_parentdir_noredirect(self):
1802 d = self.POST("/uri?t=mkdir")
1803 def _after_mkdir(res):
1804 uri.NewDirectoryURI.init_from_string(res)
1805 d.addCallback(_after_mkdir)
1808 def test_POST_mkdir_no_parentdir_redirect(self):
1809 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1810 d.addBoth(self.shouldRedirect, None, statuscode='303')
1811 def _check_target(target):
1812 target = urllib.unquote(target)
1813 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1814 d.addCallback(_check_target)
1817 def test_POST_noparent_bad(self):
1818 d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1819 "/uri accepts only PUT, PUT?t=mkdir, "
1820 "POST?t=upload, and POST?t=mkdir",
1821 self.POST, "/uri?t=bogus")
1824 def test_welcome_page_mkdir_button(self):
1825 # Fetch the welcome page.
1827 def _after_get_welcome_page(res):
1828 MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1829 mo = MKDIR_BUTTON_RE.search(res)
1830 formaction = mo.group(1)
1832 formaname = mo.group(3)
1833 formavalue = mo.group(4)
1834 return (formaction, formt, formaname, formavalue)
1835 d.addCallback(_after_get_welcome_page)
1836 def _after_parse_form(res):
1837 (formaction, formt, formaname, formavalue) = res
1838 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1839 d.addCallback(_after_parse_form)
1840 d.addBoth(self.shouldRedirect, None, statuscode='303')
1843 def test_POST_mkdir_replace(self): # return value?
1844 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1845 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1846 d.addCallback(self.failUnlessNodeKeysAre, [])
1849 def test_POST_mkdir_no_replace_queryarg(self): # return value?
1850 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1851 d.addBoth(self.shouldFail, error.Error,
1852 "POST_mkdir_no_replace_queryarg",
1854 "There was already a child by that name, and you asked me "
1855 "to not replace it")
1856 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1857 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1860 def test_POST_mkdir_no_replace_field(self): # return value?
1861 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1863 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1865 "There was already a child by that name, and you asked me "
1866 "to not replace it")
1867 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1868 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1871 def test_POST_mkdir_whendone_field(self):
1872 d = self.POST(self.public_url + "/foo",
1873 t="mkdir", name="newdir", when_done="/THERE")
1874 d.addBoth(self.shouldRedirect, "/THERE")
1875 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1876 d.addCallback(self.failUnlessNodeKeysAre, [])
1879 def test_POST_mkdir_whendone_queryarg(self):
1880 d = self.POST(self.public_url + "/foo?when_done=/THERE",
1881 t="mkdir", name="newdir")
1882 d.addBoth(self.shouldRedirect, "/THERE")
1883 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1884 d.addCallback(self.failUnlessNodeKeysAre, [])
1887 def test_POST_bad_t(self):
1888 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1889 "POST to a directory with bad t=BOGUS",
1890 self.POST, self.public_url + "/foo", t="BOGUS")
1893 def test_POST_set_children(self):
1894 contents9, n9, newuri9 = self.makefile(9)
1895 contents10, n10, newuri10 = self.makefile(10)
1896 contents11, n11, newuri11 = self.makefile(11)
1899 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1902 "ctime": 1002777696.7564139,
1903 "mtime": 1002777696.7564139
1906 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1909 "ctime": 1002777696.7564139,
1910 "mtime": 1002777696.7564139
1913 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1916 "ctime": 1002777696.7564139,
1917 "mtime": 1002777696.7564139
1920 }""" % (newuri9, newuri10, newuri11)
1922 url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1924 d = client.getPage(url, method="POST", postdata=reqbody)
1926 self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1927 self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1928 self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1930 d.addCallback(_then)
1931 d.addErrback(self.dump_error)
1934 def test_POST_put_uri(self):
1935 contents, n, newuri = self.makefile(8)
1936 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1937 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1938 d.addCallback(lambda res:
1939 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1943 def test_POST_put_uri_replace(self):
1944 contents, n, newuri = self.makefile(8)
1945 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1946 d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1947 d.addCallback(lambda res:
1948 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1952 def test_POST_put_uri_no_replace_queryarg(self):
1953 contents, n, newuri = self.makefile(8)
1954 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1955 name="bar.txt", uri=newuri)
1956 d.addBoth(self.shouldFail, error.Error,
1957 "POST_put_uri_no_replace_queryarg",
1959 "There was already a child by that name, and you asked me "
1960 "to not replace it")
1961 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1962 d.addCallback(self.failUnlessIsBarDotTxt)
1965 def test_POST_put_uri_no_replace_field(self):
1966 contents, n, newuri = self.makefile(8)
1967 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1968 name="bar.txt", uri=newuri)
1969 d.addBoth(self.shouldFail, error.Error,
1970 "POST_put_uri_no_replace_field",
1972 "There was already a child by that name, and you asked me "
1973 "to not replace it")
1974 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1975 d.addCallback(self.failUnlessIsBarDotTxt)
1978 def test_POST_delete(self):
1979 d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1980 d.addCallback(lambda res: self._foo_node.list())
1981 def _check(children):
1982 self.failIf(u"bar.txt" in children)
1983 d.addCallback(_check)
1986 def test_POST_rename_file(self):
1987 d = self.POST(self.public_url + "/foo", t="rename",
1988 from_name="bar.txt", to_name='wibble.txt')
1989 d.addCallback(lambda res:
1990 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1991 d.addCallback(lambda res:
1992 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1993 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1994 d.addCallback(self.failUnlessIsBarDotTxt)
1995 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1996 d.addCallback(self.failUnlessIsBarJSON)
1999 def test_POST_rename_file_redundant(self):
2000 d = self.POST(self.public_url + "/foo", t="rename",
2001 from_name="bar.txt", to_name='bar.txt')
2002 d.addCallback(lambda res:
2003 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2004 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2005 d.addCallback(self.failUnlessIsBarDotTxt)
2006 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2007 d.addCallback(self.failUnlessIsBarJSON)
2010 def test_POST_rename_file_replace(self):
2011 # rename a file and replace a directory with it
2012 d = self.POST(self.public_url + "/foo", t="rename",
2013 from_name="bar.txt", to_name='empty')
2014 d.addCallback(lambda res:
2015 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2016 d.addCallback(lambda res:
2017 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2018 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2019 d.addCallback(self.failUnlessIsBarDotTxt)
2020 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2021 d.addCallback(self.failUnlessIsBarJSON)
2024 def test_POST_rename_file_no_replace_queryarg(self):
2025 # rename a file and replace a directory with it
2026 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2027 from_name="bar.txt", to_name='empty')
2028 d.addBoth(self.shouldFail, error.Error,
2029 "POST_rename_file_no_replace_queryarg",
2031 "There was already a child by that name, and you asked me "
2032 "to not replace it")
2033 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2034 d.addCallback(self.failUnlessIsEmptyJSON)
2037 def test_POST_rename_file_no_replace_field(self):
2038 # rename a file and replace a directory with it
2039 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2040 from_name="bar.txt", to_name='empty')
2041 d.addBoth(self.shouldFail, error.Error,
2042 "POST_rename_file_no_replace_field",
2044 "There was already a child by that name, and you asked me "
2045 "to not replace it")
2046 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2047 d.addCallback(self.failUnlessIsEmptyJSON)
2050 def failUnlessIsEmptyJSON(self, res):
2051 data = simplejson.loads(res)
2052 self.failUnlessEqual(data[0], "dirnode", data)
2053 self.failUnlessEqual(len(data[1]["children"]), 0)
2055 def test_POST_rename_file_slash_fail(self):
2056 d = self.POST(self.public_url + "/foo", t="rename",
2057 from_name="bar.txt", to_name='kirk/spock.txt')
2058 d.addBoth(self.shouldFail, error.Error,
2059 "test_POST_rename_file_slash_fail",
2061 "to_name= may not contain a slash",
2063 d.addCallback(lambda res:
2064 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2067 def test_POST_rename_dir(self):
2068 d = self.POST(self.public_url, t="rename",
2069 from_name="foo", to_name='plunk')
2070 d.addCallback(lambda res:
2071 self.failIfNodeHasChild(self.public_root, u"foo"))
2072 d.addCallback(lambda res:
2073 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2074 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2075 d.addCallback(self.failUnlessIsFooJSON)
2078 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2079 """ If target is not None then the redirection has to go to target. If
2080 statuscode is not None then the redirection has to be accomplished with
2081 that HTTP status code."""
2082 if not isinstance(res, failure.Failure):
2083 to_where = (target is None) and "somewhere" or ("to " + target)
2084 self.fail("%s: we were expecting to get redirected %s, not get an"
2085 " actual page: %s" % (which, to_where, res))
2086 res.trap(error.PageRedirect)
2087 if statuscode is not None:
2088 self.failUnlessEqual(res.value.status, statuscode,
2089 "%s: not a redirect" % which)
2090 if target is not None:
2091 # the PageRedirect does not seem to capture the uri= query arg
2092 # properly, so we can't check for it.
2093 realtarget = self.webish_url + target
2094 self.failUnlessEqual(res.value.location, realtarget,
2095 "%s: wrong target" % which)
2096 return res.value.location
2098 def test_GET_URI_form(self):
2099 base = "/uri?uri=%s" % self._bar_txt_uri
2100 # this is supposed to give us a redirect to /uri/$URI, plus arguments
2101 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2103 d.addBoth(self.shouldRedirect, targetbase)
2104 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2105 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2106 d.addCallback(lambda res: self.GET(base+"&t=json"))
2107 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2108 d.addCallback(self.log, "about to get file by uri")
2109 d.addCallback(lambda res: self.GET(base, followRedirect=True))
2110 d.addCallback(self.failUnlessIsBarDotTxt)
2111 d.addCallback(self.log, "got file by uri, about to get dir by uri")
2112 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2113 followRedirect=True))
2114 d.addCallback(self.failUnlessIsFooJSON)
2115 d.addCallback(self.log, "got dir by uri")
2119 def test_GET_URI_form_bad(self):
2120 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2121 "400 Bad Request", "GET /uri requires uri=",
2125 def test_GET_rename_form(self):
2126 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2127 followRedirect=True)
2129 self.failUnless('name="when_done" value="."' in res, res)
2130 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2131 d.addCallback(_check)
2134 def log(self, res, msg):
2135 #print "MSG: %s RES: %s" % (msg, res)
2139 def test_GET_URI_URL(self):
2140 base = "/uri/%s" % self._bar_txt_uri
2142 d.addCallback(self.failUnlessIsBarDotTxt)
2143 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2144 d.addCallback(self.failUnlessIsBarDotTxt)
2145 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2146 d.addCallback(self.failUnlessIsBarDotTxt)
2149 def test_GET_URI_URL_dir(self):
2150 base = "/uri/%s?t=json" % self._foo_uri
2152 d.addCallback(self.failUnlessIsFooJSON)
2155 def test_GET_URI_URL_missing(self):
2156 base = "/uri/%s" % self._bad_file_uri
2158 d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2159 http.GONE, response_substring="NotEnoughSharesError")
2160 # TODO: how can we exercise both sides of WebDownloadTarget.fail
2161 # here? we must arrange for a download to fail after target.open()
2162 # has been called, and then inspect the response to see that it is
2163 # shorter than we expected.
2166 def test_PUT_DIRURL_uri(self):
2167 d = self.s.create_empty_dirnode()
2169 new_uri = dn.get_uri()
2170 # replace /foo with a new (empty) directory
2171 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2172 d.addCallback(lambda res:
2173 self.failUnlessEqual(res.strip(), new_uri))
2174 d.addCallback(lambda res:
2175 self.failUnlessChildURIIs(self.public_root,
2179 d.addCallback(_made_dir)
2182 def test_PUT_DIRURL_uri_noreplace(self):
2183 d = self.s.create_empty_dirnode()
2185 new_uri = dn.get_uri()
2186 # replace /foo with a new (empty) directory, but ask that
2187 # replace=false, so it should fail
2188 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2189 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2191 self.public_url + "/foo?t=uri&replace=false",
2193 d.addCallback(lambda res:
2194 self.failUnlessChildURIIs(self.public_root,
2198 d.addCallback(_made_dir)
2201 def test_PUT_DIRURL_bad_t(self):
2202 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2203 "400 Bad Request", "PUT to a directory",
2204 self.PUT, self.public_url + "/foo?t=BOGUS", "")
2205 d.addCallback(lambda res:
2206 self.failUnlessChildURIIs(self.public_root,
2211 def test_PUT_NEWFILEURL_uri(self):
2212 contents, n, new_uri = self.makefile(8)
2213 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2214 d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2215 d.addCallback(lambda res:
2216 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2220 def test_PUT_NEWFILEURL_uri_replace(self):
2221 contents, n, new_uri = self.makefile(8)
2222 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2223 d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2224 d.addCallback(lambda res:
2225 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2229 def test_PUT_NEWFILEURL_uri_no_replace(self):
2230 contents, n, new_uri = self.makefile(8)
2231 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2232 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2234 "There was already a child by that name, and you asked me "
2235 "to not replace it")
2238 def test_PUT_NEWFILE_URI(self):
2239 file_contents = "New file contents here\n"
2240 d = self.PUT("/uri", file_contents)
2242 self.failUnless(uri in FakeCHKFileNode.all_contents)
2243 self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2245 return self.GET("/uri/%s" % uri)
2246 d.addCallback(_check)
2248 self.failUnlessEqual(res, file_contents)
2249 d.addCallback(_check2)
2252 def test_PUT_NEWFILE_URI_only_PUT(self):
2253 d = self.PUT("/uri?t=bogus", "")
2254 d.addBoth(self.shouldFail, error.Error,
2255 "PUT_NEWFILE_URI_only_PUT",
2257 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2260 def test_PUT_NEWFILE_URI_mutable(self):
2261 file_contents = "New file contents here\n"
2262 d = self.PUT("/uri?mutable=true", file_contents)
2263 def _check_mutable(uri):
2266 self.failUnless(IMutableFileURI.providedBy(u))
2267 self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2268 n = self.s.create_node_from_uri(uri)
2269 return n.download_best_version()
2270 d.addCallback(_check_mutable)
2271 def _check2_mutable(data):
2272 self.failUnlessEqual(data, file_contents)
2273 d.addCallback(_check2_mutable)
2277 self.failUnless(uri in FakeCHKFileNode.all_contents)
2278 self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2280 return self.GET("/uri/%s" % uri)
2281 d.addCallback(_check)
2283 self.failUnlessEqual(res, file_contents)
2284 d.addCallback(_check2)
2287 def test_PUT_mkdir(self):
2288 d = self.PUT("/uri?t=mkdir", "")
2290 n = self.s.create_node_from_uri(uri.strip())
2291 d2 = self.failUnlessNodeKeysAre(n, [])
2292 d2.addCallback(lambda res:
2293 self.GET("/uri/%s?t=json" % uri))
2295 d.addCallback(_check)
2296 d.addCallback(self.failUnlessIsEmptyJSON)
2299 def test_POST_check(self):
2300 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2302 # this returns a string form of the results, which are probably
2303 # None since we're using fake filenodes.
2304 # TODO: verify that the check actually happened, by changing
2305 # FakeCHKFileNode to count how many times .check() has been
2308 d.addCallback(_done)
2311 def test_bad_method(self):
2312 url = self.webish_url + self.public_url + "/foo/bar.txt"
2313 d = self.shouldHTTPError2("test_bad_method",
2314 501, "Not Implemented",
2315 "I don't know how to treat a BOGUS request.",
2316 client.getPage, url, method="BOGUS")
2319 def test_short_url(self):
2320 url = self.webish_url + "/uri"
2321 d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2322 "I don't know how to treat a DELETE request.",
2323 client.getPage, url, method="DELETE")
2326 def test_ophandle_bad(self):
2327 url = self.webish_url + "/operations/bogus?t=status"
2328 d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2329 "unknown/expired handle 'bogus'",
2330 client.getPage, url)
2333 def test_ophandle_cancel(self):
2334 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2335 followRedirect=True)
2336 d.addCallback(lambda ignored:
2337 self.GET("/operations/128?t=status&output=JSON"))
2339 data = simplejson.loads(res)
2340 self.failUnless("finished" in data, res)
2341 monitor = self.ws.root.child_operations.handles["128"][0]
2342 d = self.POST("/operations/128?t=cancel&output=JSON")
2344 data = simplejson.loads(res)
2345 self.failUnless("finished" in data, res)
2346 # t=cancel causes the handle to be forgotten
2347 self.failUnless(monitor.is_cancelled())
2348 d.addCallback(_check2)
2350 d.addCallback(_check1)
2351 d.addCallback(lambda ignored:
2352 self.shouldHTTPError2("test_ophandle_cancel",
2353 404, "404 Not Found",
2354 "unknown/expired handle '128'",
2356 "/operations/128?t=status&output=JSON"))
2359 def test_ophandle_retainfor(self):
2360 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2361 followRedirect=True)
2362 d.addCallback(lambda ignored:
2363 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2365 data = simplejson.loads(res)
2366 self.failUnless("finished" in data, res)
2367 d.addCallback(_check1)
2368 # the retain-for=0 will cause the handle to be expired very soon
2369 d.addCallback(self.stall, 2.0)
2370 d.addCallback(lambda ignored:
2371 self.shouldHTTPError2("test_ophandle_retainfor",
2372 404, "404 Not Found",
2373 "unknown/expired handle '129'",
2375 "/operations/129?t=status&output=JSON"))
2378 def test_ophandle_release_after_complete(self):
2379 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2380 followRedirect=True)
2381 d.addCallback(self.wait_for_operation, "130")
2382 d.addCallback(lambda ignored:
2383 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2384 # the release-after-complete=true will cause the handle to be expired
2385 d.addCallback(lambda ignored:
2386 self.shouldHTTPError2("test_ophandle_release_after_complete",
2387 404, "404 Not Found",
2388 "unknown/expired handle '130'",
2390 "/operations/130?t=status&output=JSON"))
2393 def test_incident(self):
2394 d = self.POST("/report_incident", details="eek")
2396 self.failUnless("Thank you for your report!" in res, res)
2397 d.addCallback(_done)
2400 def test_static(self):
2401 webdir = os.path.join(self.staticdir, "subdir")
2402 fileutil.make_dirs(webdir)
2403 f = open(os.path.join(webdir, "hello.txt"), "wb")
2407 d = self.GET("/static/subdir/hello.txt")
2409 self.failUnlessEqual(res, "hello")
2410 d.addCallback(_check)
2414 class Util(unittest.TestCase):
2415 def test_abbreviate_time(self):
2416 self.failUnlessEqual(common.abbreviate_time(None), "")
2417 self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2418 self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2419 self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2420 self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2422 def test_abbreviate_rate(self):
2423 self.failUnlessEqual(common.abbreviate_rate(None), "")
2424 self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2425 self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2426 self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2428 def test_abbreviate_size(self):
2429 self.failUnlessEqual(common.abbreviate_size(None), "")
2430 self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2431 self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2432 self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2433 self.failUnlessEqual(common.abbreviate_size(123), "123B")