-from base64 import b32encode
-import os, sys, time, simplejson
+
+import os, re, sys, time, simplejson
from cStringIO import StringIO
+
from twisted.trial import unittest
from twisted.internet import defer
from twisted.internet import threads # CLI tests use deferToThread
from allmydata.immutable.filenode import ImmutableFileNode
from allmydata.util import idlib, mathutil
from allmydata.util import log, base32
-from allmydata.util.encodingutil import quote_output, unicode_to_argv, get_filesystem_encoding
+from allmydata.util.verlib import NormalizedVersion
+from allmydata.util.encodingutil import quote_output, unicode_to_argv
from allmydata.util.fileutil import abspath_expanduser_unicode
from allmydata.util.consumer import MemoryConsumer, download_to_data
from allmydata.scripts import runner
from allmydata.monitor import Monitor
from allmydata.mutable.common import NotWriteableError
from allmydata.mutable import layout as mutable_layout
-from foolscap.api import DeadReferenceError
+from allmydata.mutable.publish import MutableData
+
+import foolscap
+from foolscap.api import DeadReferenceError, fireEventually
from twisted.python.failure import Failure
from twisted.web.client import getPage
from twisted.web.error import Error
all_peerids = c.get_storage_broker().get_all_serverids()
self.failUnlessEqual(len(all_peerids), self.numclients+1)
sb = c.storage_broker
- permuted_peers = sb.get_servers_for_index("a")
+ permuted_peers = sb.get_servers_for_psi("a")
self.failUnlessEqual(len(permuted_peers), self.numclients+1)
d.addCallback(_check)
d = self.set_up_nodes()
def _check_connections(res):
for c in self.clients:
- c.DEFAULT_ENCODING_PARAMETERS['happy'] = 5
+ c.encoding_params['happy'] = 5
all_peerids = c.get_storage_broker().get_all_serverids()
self.failUnlessEqual(len(all_peerids), self.numclients)
sb = c.storage_broker
- permuted_peers = sb.get_servers_for_index("a")
+ permuted_peers = sb.get_servers_for_psi("a")
self.failUnlessEqual(len(permuted_peers), self.numclients)
d.addCallback(_check_connections)
return d1
d.addCallback(_do_upload)
def _upload_done(results):
- theuri = results.uri
+ theuri = results.get_uri()
log.msg("upload finished: uri is %s" % (theuri,))
self.uri = theuri
assert isinstance(self.uri, str), self.uri
facility="tahoe.tests")
d1 = download_to_data(badnode)
def _baduri_should_fail(res):
- log.msg("finished downloading non-existend URI",
+ log.msg("finished downloading non-existent URI",
level=log.UNUSUAL, facility="tahoe.tests")
self.failUnless(isinstance(res, Failure))
self.failUnless(res.check(NoSharesError),
add_to_sparent=True))
def _added(extra_node):
self.extra_node = extra_node
- self.extra_node.DEFAULT_ENCODING_PARAMETERS['happy'] = 5
+ self.extra_node.encoding_params['happy'] = 5
d.addCallback(_added)
+ def _has_helper():
+ uploader = self.extra_node.getServiceNamed("uploader")
+ furl, connected = uploader.get_helper_info()
+ return connected
+ d.addCallback(lambda ign: self.poll(_has_helper))
+
HELPER_DATA = "Data that needs help to upload" * 1000
def _upload_with_helper(res):
u = upload.Data(HELPER_DATA, convergence=convergence)
d = self.extra_node.upload(u)
def _uploaded(results):
- n = self.clients[1].create_node_from_uri(results.uri)
+ n = self.clients[1].create_node_from_uri(results.get_uri())
return download_to_data(n)
d.addCallback(_uploaded)
def _check(newdata):
u.debug_stash_RemoteEncryptedUploadable = True
d = self.extra_node.upload(u)
def _uploaded(results):
- n = self.clients[1].create_node_from_uri(results.uri)
+ n = self.clients[1].create_node_from_uri(results.get_uri())
return download_to_data(n)
d.addCallback(_uploaded)
def _check(newdata):
if convergence is not None:
d.addCallback(_upload_duplicate_with_helper)
+ d.addCallback(fireEventually)
+
def _upload_resumable(res):
DATA = "Data that needs help to upload and gets interrupted" * 1000
u1 = CountingDataUploadable(DATA, convergence=convergence)
u2 = CountingDataUploadable(DATA, convergence=convergence)
# we interrupt the connection after about 5kB by shutting down
- # the helper, then restartingit.
+ # the helper, then restarting it.
u1.interrupt_after = 5000
u1.interrupt_after_d = defer.Deferred()
- u1.interrupt_after_d.addCallback(lambda res:
- self.bounce_client(0))
+ bounced_d = defer.Deferred()
+ def _do_bounce(res):
+ d = self.bounce_client(0)
+ d.addBoth(bounced_d.callback)
+ u1.interrupt_after_d.addCallback(_do_bounce)
# sneak into the helper and reduce its chunk size, so that our
# debug_interrupt will sever the connection on about the fifth
# this same test run, but I'm not currently worried about it.
offloaded.CHKCiphertextFetcher.CHUNK_SIZE = 1000
- d = self.extra_node.upload(u1)
-
- def _should_not_finish(res):
- self.fail("interrupted upload should have failed, not finished"
- " with result %s" % (res,))
- def _interrupted(f):
- f.trap(DeadReferenceError)
-
- # make sure we actually interrupted it before finishing the
- # file
- self.failUnless(u1.bytes_read < len(DATA),
- "read %d out of %d total" % (u1.bytes_read,
- len(DATA)))
-
- log.msg("waiting for reconnect", level=log.NOISY,
- facility="tahoe.test.test_system")
- # now, we need to give the nodes a chance to notice that this
- # connection has gone away. When this happens, the storage
- # servers will be told to abort their uploads, removing the
- # partial shares. Unfortunately this involves TCP messages
- # going through the loopback interface, and we can't easily
- # predict how long that will take. If it were all local, we
- # could use fireEventually() to stall. Since we don't have
- # the right introduction hooks, the best we can do is use a
- # fixed delay. TODO: this is fragile.
- u1.interrupt_after_d.addCallback(self.stall, 2.0)
- return u1.interrupt_after_d
- d.addCallbacks(_should_not_finish, _interrupted)
+ upload_d = self.extra_node.upload(u1)
+ # The upload will start, and bounce_client() will be called after
+ # about 5kB. bounced_d will fire after bounce_client() finishes
+ # shutting down and restarting the node.
+ d = bounced_d
+ def _bounced(ign):
+ # By this point, the upload should have failed because of the
+ # interruption. upload_d will fire in a moment
+ def _should_not_finish(res):
+ self.fail("interrupted upload should have failed, not"
+ " finished with result %s" % (res,))
+ def _interrupted(f):
+ f.trap(DeadReferenceError)
+ # make sure we actually interrupted it before finishing
+ # the file
+ self.failUnless(u1.bytes_read < len(DATA),
+ "read %d out of %d total" %
+ (u1.bytes_read, len(DATA)))
+ upload_d.addCallbacks(_should_not_finish, _interrupted)
+ return upload_d
+ d.addCallback(_bounced)
def _disconnected(res):
# check to make sure the storage servers aren't still hanging
self.failIf(os.path.exists(incdir) and os.listdir(incdir))
d.addCallback(_disconnected)
- # then we need to give the reconnector a chance to
- # reestablish the connection to the helper.
d.addCallback(lambda res:
- log.msg("wait_for_connections", level=log.NOISY,
+ log.msg("wait_for_helper", level=log.NOISY,
facility="tahoe.test.test_system"))
- d.addCallback(lambda res: self.wait_for_connections())
-
+ # then we need to wait for the extra node to reestablish its
+ # connection to the helper.
+ d.addCallback(lambda ign: self.poll(_has_helper))
d.addCallback(lambda res:
log.msg("uploading again", level=log.NOISY,
d.addCallback(lambda res: self.extra_node.upload(u2))
def _uploaded(results):
- cap = results.uri
+ cap = results.get_uri()
log.msg("Second upload complete", level=log.NOISY,
facility="tahoe.test.test_system")
# this is really bytes received rather than sent, but it's
# convenient and basically measures the same thing
- bytes_sent = results.ciphertext_fetched
+ bytes_sent = results.get_ciphertext_fetched()
self.failUnless(isinstance(bytes_sent, (int, long)), bytes_sent)
# We currently don't support resumption of upload if the data is
def test_mutable(self):
self.basedir = "system/SystemTest/test_mutable"
DATA = "initial contents go here." # 25 bytes % 3 != 0
+ DATA_uploadable = MutableData(DATA)
NEWDATA = "new contents yay"
+ NEWDATA_uploadable = MutableData(NEWDATA)
NEWERDATA = "this is getting old"
+ NEWERDATA_uploadable = MutableData(NEWERDATA)
d = self.set_up_nodes(use_key_generator=True)
def _create_mutable(res):
c = self.clients[0]
log.msg("starting create_mutable_file")
- d1 = c.create_mutable_file(DATA)
+ d1 = c.create_mutable_file(DATA_uploadable)
def _done(res):
log.msg("DONE: %s" % (res,))
self._mutable_node_1 = res
self.failUnlessEqual(res, DATA)
# replace the data
log.msg("starting replace1")
- d1 = newnode.overwrite(NEWDATA)
+ d1 = newnode.overwrite(NEWDATA_uploadable)
d1.addCallback(lambda res: newnode.download_best_version())
return d1
d.addCallback(_check_download_3)
newnode2 = self.clients[3].create_node_from_uri(uri)
self._newnode3 = self.clients[3].create_node_from_uri(uri)
log.msg("starting replace2")
- d1 = newnode1.overwrite(NEWERDATA)
+ d1 = newnode1.overwrite(NEWERDATA_uploadable)
d1.addCallback(lambda res: newnode2.download_best_version())
return d1
d.addCallback(_check_download_4)
def _check_empty_file(res):
# make sure we can create empty files, this usually screws up the
# segsize math
- d1 = self.clients[2].create_mutable_file("")
+ d1 = self.clients[2].create_mutable_file(MutableData(""))
d1.addCallback(lambda newnode: newnode.download_best_version())
d1.addCallback(lambda res: self.failUnlessEqual("", res))
return d1
self.key_generator_svc.key_generator.pool_size + size_delta)
d.addCallback(check_kg_poolsize, 0)
- d.addCallback(lambda junk: self.clients[3].create_mutable_file('hello, world'))
+ d.addCallback(lambda junk:
+ self.clients[3].create_mutable_file(MutableData('hello, world')))
d.addCallback(check_kg_poolsize, -1)
d.addCallback(lambda junk: self.clients[3].create_dirnode())
d.addCallback(check_kg_poolsize, -2)
d = self.set_up_nodes(use_stats_gatherer=True)
def _new_happy_semantics(ign):
for c in self.clients:
- c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ c.encoding_params['happy'] = 1
d.addCallback(_new_happy_semantics)
d.addCallback(self._test_introweb)
d.addCallback(self.log, "starting publish")
newappverstr = "%s: %s" % (allmydata.__appname__, altverstr)
self.failUnless((appverstr in res) or (newappverstr in res), (appverstr, newappverstr, res))
- self.failUnless("Announcement Summary: storage: 5, stub_client: 5" in res)
+ self.failUnless("Announcement Summary: storage: 5" in res)
self.failUnless("Subscription Summary: storage: 5" in res)
+ self.failUnless("tahoe.css" in res)
except unittest.FailTest:
print
print "GET %s output was:" % self.introweb_url
print res
raise
d.addCallback(_check)
+ # make sure it serves the CSS too
+ d.addCallback(lambda res:
+ getPage(self.introweb_url+"tahoe.css", method="GET"))
d.addCallback(lambda res:
getPage(self.introweb_url + "?t=json",
method="GET", followRedirect=True))
self.failUnlessEqual(data["subscription_summary"],
{"storage": 5})
self.failUnlessEqual(data["announcement_summary"],
- {"storage": 5, "stub_client": 5})
- self.failUnlessEqual(data["announcement_distinct_hosts"],
- {"storage": 1, "stub_client": 1})
+ {"storage": 5})
except unittest.FailTest:
print
print "GET %s?t=json output was:" % self.introweb_url
public = "uri/" + self._root_directory_uri
d = getPage(base)
def _got_welcome(page):
- # XXX This test is oversensitive to formatting
- expected = "Connected to <span>%d</span>\n of <span>%d</span> known storage servers:" % (self.numclients, self.numclients)
- self.failUnless(expected in page,
- "I didn't see the right 'connected storage servers'"
- " message in: %s" % page
- )
- expected = "<th>My nodeid:</th> <td class=\"nodeid mine data-chars\">%s</td>" % (b32encode(self.clients[0].nodeid).lower(),)
- self.failUnless(expected in page,
- "I didn't see the right 'My nodeid' message "
- "in: %s" % page)
+ html = page.replace('\n', ' ')
+ connected_re = r'Connected to <span>%d</span>\s*of <span>%d</span> known storage servers' % (self.numclients, self.numclients)
+ self.failUnless(re.search(connected_re, html),
+ "I didn't see the right '%s' message in:\n%s" % (connected_re, page))
+ # nodeids/tubids don't have any regexp-special characters
+ nodeid_re = r'<th>Node ID:</th>\s*<td title="TubID: %s">%s</td>' % (
+ self.clients[0].get_long_tubid(), self.clients[0].get_long_nodeid())
+ self.failUnless(re.search(nodeid_re, html),
+ "I didn't see the right '%s' message in:\n%s" % (nodeid_re, page))
self.failUnless("Helper: 0 active uploads" in page)
d.addCallback(_got_welcome)
d.addCallback(self.log, "done with _got_welcome")
# get the welcome page from the node that uses the helper too
d.addCallback(lambda res: getPage(self.helper_webish_url))
def _got_welcome_helper(page):
- self.failUnless("Connected to helper?: <span>yes</span>" in page,
- page)
- self.failUnless("Not running helper" in page)
+ html = page.replace('\n', ' ')
+ self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), page)
+ self.failUnlessIn("Not running helper", page)
d.addCallback(_got_welcome_helper)
d.addCallback(lambda res: getPage(base + public))
d.addCallback(lambda res: getPage(base + public + "/subdir1"))
def _got_subdir1(page):
# there ought to be an href for our file
- self.failUnless(("<td>%d</td>" % len(self.data)) in page)
+ self.failUnlessIn('<td align="right">%d</td>' % len(self.data), page)
self.failUnless(">mydata567</a>" in page)
d.addCallback(_got_subdir1)
d.addCallback(self.log, "done with _got_subdir1")
def _new_happy_semantics(ign):
for c in self.clients:
# these get reset somewhere? Whatever.
- c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ c.encoding_params['happy'] = 1
d.addCallback(_new_happy_semantics)
d.addCallback(lambda res: self.PUT(public + "/subdir3/big.txt",
"big" * 500000)) # 1.5MB
return self.GET("status/publish-%d" % self._publish_status)
d.addCallback(_got_update)
def _got_publish(res):
+ self.failUnlessIn("Publish Results", res)
return self.GET("status/retrieve-%d" % self._retrieve_status)
d.addCallback(_got_publish)
+ def _got_retrieve(res):
+ self.failUnlessIn("Retrieve Results", res)
+ d.addCallback(_got_retrieve)
# check that the helper status page exists
d.addCallback(lambda res:
# itself) doesn't explode when you ask for its status
d.addCallback(lambda res: getPage(self.helper_webish_url + "status/"))
def _got_non_helper_status(res):
- self.failUnless("Upload and Download Status" in res)
+ self.failUnlessIn("Recent and Active Operations", res)
d.addCallback(_got_non_helper_status)
# or for helper status with t=json
# see if the statistics page exists
d.addCallback(lambda res: self.GET("statistics"))
def _got_stats(res):
- self.failUnless("Node Statistics" in res)
- self.failUnless(" 'downloader.files_downloaded': 5," in res, res)
+ self.failUnlessIn("Operational Statistics", res)
+ self.failUnlessIn(" 'downloader.files_downloaded': 5,", res)
d.addCallback(_got_stats)
d.addCallback(lambda res: self.GET("statistics?t=json"))
def _got_stats_json(res):
d.addCallback(self._test_control2, control_furl_file)
return d
def _test_control2(self, rref, filename):
- d = rref.callRemote("upload_from_file_to_uri",
- filename.encode(get_filesystem_encoding()), convergence=None)
- downfile = os.path.join(self.basedir, "control.downfile").encode(get_filesystem_encoding())
- d.addCallback(lambda uri:
- rref.callRemote("download_from_uri_to_file",
- uri, downfile))
- def _check(res):
- self.failUnlessEqual(res, downfile)
- data = open(downfile, "r").read()
- expected_data = open(filename, "r").read()
- self.failUnlessEqual(data, expected_data)
- d.addCallback(_check)
+ d = defer.succeed(None)
d.addCallback(lambda res: rref.callRemote("speed_test", 1, 200, False))
- if sys.platform == "linux2":
+ if sys.platform in ("linux2", "linux3"):
d.addCallback(lambda res: rref.callRemote("get_memory_usage"))
d.addCallback(lambda res: rref.callRemote("measure_peer_response_time"))
return d
def run(ignored, verb, *args, **kwargs):
stdin = kwargs.get("stdin", "")
- newargs = [verb] + nodeargs + list(args)
+ newargs = nodeargs + [verb] + list(args)
return self._run_cli(newargs, stdin=stdin)
def _check_ls((out,err), expected_children, unexpected_children=[]):
d.addCallback(run, "put", files[1], "subdir/tahoe-file1")
# tahoe put bar tahoe:FOO
d.addCallback(run, "put", files[2], "tahoe:file2")
- d.addCallback(run, "put", "--mutable", files[3], "tahoe:file3")
+ d.addCallback(run, "put", "--format=SDMF", files[3], "tahoe:file3")
def _check_put_mutable((out,err)):
self._mutable_file3_uri = out.strip()
d.addCallback(_check_put_mutable)
open(os.path.join(sdn2, "rfile5"), "wb").write("rfile5")
# from disk into tahoe
- d.addCallback(run, "cp", "-r", dn, "tahoe:dir1")
+ d.addCallback(run, "cp", "-r", dn, "tahoe:")
d.addCallback(run, "ls")
d.addCallback(_check_ls, ["dir1"])
d.addCallback(run, "ls", "dir1")
def _check_cp_r_out((out,err)):
def _cmp(name):
old = open(os.path.join(dn, name), "rb").read()
- newfn = os.path.join(dn_copy, name)
+ newfn = os.path.join(dn_copy, "dir1", name)
self.failUnless(os.path.exists(newfn))
new = open(newfn, "rb").read()
self.failUnlessEqual(old, new)
d.addCallback(run, "cp", "-r", "--caps-only", "tahoe:dir1", dn_copy2)
def _check_capsonly((out,err)):
# these should all be LITs
- x = open(os.path.join(dn_copy2, "subdir2", "rfile4")).read()
+ x = open(os.path.join(dn_copy2, "dir1", "subdir2", "rfile4")).read()
y = uri.from_string_filenode(x)
self.failUnlessEqual(y.data, "rfile4")
d.addCallback(_check_capsonly)
d.addCallback(run, "cp", "-r", "tahoe:dir1", "tahoe:dir1-copy")
d.addCallback(run, "ls")
d.addCallback(_check_ls, ["dir1", "dir1-copy"])
- d.addCallback(run, "ls", "dir1-copy")
+ d.addCallback(run, "ls", "dir1-copy/dir1")
d.addCallback(_check_ls, ["rfile1", "rfile2", "rfile3", "subdir2"],
["rfile4", "rfile5"])
- d.addCallback(run, "ls", "tahoe:dir1-copy/subdir2")
+ d.addCallback(run, "ls", "tahoe:dir1-copy/dir1/subdir2")
d.addCallback(_check_ls, ["rfile4", "rfile5"],
["rfile1", "rfile2", "rfile3"])
- d.addCallback(run, "get", "dir1-copy/subdir2/rfile4")
+ d.addCallback(run, "get", "dir1-copy/dir1/subdir2/rfile4")
d.addCallback(_check_stdout_against, data="rfile4")
# and copy it a second time, which ought to overwrite the same files
# tahoe_ls doesn't currently handle the error correctly: it tries to
# JSON-parse a traceback.
## def _ls_missing(res):
-## argv = ["ls"] + nodeargs + ["bogus"]
+## argv = nodeargs + ["ls", "bogus"]
## return self._run_cli(argv)
## d.addCallback(_ls_missing)
## def _check_ls_missing((out,err)):
d = self.set_up_nodes()
def _new_happy_semantics(ign):
for c in self.clients:
- c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ c.encoding_params['happy'] = 1
d.addCallback(_new_happy_semantics)
def _run_in_subprocess(ignored, verb, *args, **kwargs):
stdin = kwargs.get("stdin")
env = kwargs.get("env")
- newargs = [verb, "--node-directory", self.getdir("client0")] + list(args)
+ newargs = ["--node-directory", self.getdir("client0"), verb] + list(args)
return self.run_bintahoe(newargs, stdin=stdin, env=env)
def _check_succeeded(res, check_stderr=True):
return d
d.addCallback(_got_lit_filenode)
return d
+
+
+class Connections(SystemTestMixin, unittest.TestCase):
+ def test_rref(self):
+ if NormalizedVersion(foolscap.__version__) < NormalizedVersion('0.6.4'):
+ raise unittest.SkipTest("skipped due to http://foolscap.lothar.com/trac/ticket/196 "
+ "(which does not affect normal usage of Tahoe-LAFS)")
+
+ self.basedir = "system/Connections/rref"
+ d = self.set_up_nodes(2)
+ def _start(ign):
+ self.c0 = self.clients[0]
+ nonclients = [s for s in self.c0.storage_broker.get_connected_servers()
+ if s.get_serverid() != self.c0.nodeid]
+ self.failUnlessEqual(len(nonclients), 1)
+
+ self.s1 = nonclients[0] # s1 is the server, not c0
+ self.s1_rref = self.s1.get_rref()
+ self.failIfEqual(self.s1_rref, None)
+ self.failUnless(self.s1.is_connected())
+ d.addCallback(_start)
+
+ # now shut down the server
+ d.addCallback(lambda ign: self.clients[1].disownServiceParent())
+ # and wait for the client to notice
+ def _poll():
+ return len(self.c0.storage_broker.get_connected_servers()) < 2
+ d.addCallback(lambda ign: self.poll(_poll))
+
+ def _down(ign):
+ self.failIf(self.s1.is_connected())
+ rref = self.s1.get_rref()
+ self.failUnless(rref)
+ self.failUnlessIdentical(rref, self.s1_rref)
+ d.addCallback(_down)
+ return d
projects / tahoe-lafs / tahoe-lafs.git / blobdiff