from twisted.trial import unittest
from twisted.internet import defer
from twisted.internet import threads # CLI tests use deferToThread
+
import allmydata
from allmydata import uri
from allmydata.storage.mutable import MutableShareFile
from allmydata.storage.server import si_a2b
from allmydata.immutable import offloaded, upload
-from allmydata.immutable.filenode import ImmutableFileNode, LiteralFileNode
+from allmydata.immutable.literal import LiteralFileNode
+from allmydata.immutable.filenode import ImmutableFileNode
from allmydata.util import idlib, mathutil
from allmydata.util import log, base32
+from allmydata.util.encodingutil import quote_output, unicode_to_argv, get_filesystem_encoding
+from allmydata.util.fileutil import abspath_expanduser_unicode
from allmydata.util.consumer import MemoryConsumer, download_to_data
from allmydata.scripts import runner
from allmydata.interfaces import IDirectoryNode, IFileNode, \
from allmydata.monitor import Monitor
from allmydata.mutable.common import NotWriteableError
from allmydata.mutable import layout as mutable_layout
+from allmydata.mutable.publish import MutableData
from foolscap.api import DeadReferenceError
from twisted.python.failure import Failure
from twisted.web.client import getPage
from allmydata.test.common import SystemTestMixin
+# TODO: move this to common or common_util
+from allmydata.test.test_runner import RunBinTahoeMixin
+
LARGE_DATA = """
This is some data to publish to the remote grid.., which needs to be large
enough to not fit inside a LIT uri.
self.interrupt_after_d.callback(self)
return upload.Data.read(self, length)
-class SystemTest(SystemTestMixin, unittest.TestCase):
+class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
timeout = 3600 # It takes longer than 960 seconds on Zandr's ARM box.
def test_connections(self):
all_peerids = c.get_storage_broker().get_all_serverids()
self.failUnlessEqual(len(all_peerids), self.numclients+1)
sb = c.storage_broker
- permuted_peers = sb.get_servers_for_index("a")
+ permuted_peers = sb.get_servers_for_psi("a")
self.failUnlessEqual(len(permuted_peers), self.numclients+1)
d.addCallback(_check)
d = self.set_up_nodes()
def _check_connections(res):
for c in self.clients:
+ c.DEFAULT_ENCODING_PARAMETERS['happy'] = 5
all_peerids = c.get_storage_broker().get_all_serverids()
self.failUnlessEqual(len(all_peerids), self.numclients)
sb = c.storage_broker
- permuted_peers = sb.get_servers_for_index("a")
+ permuted_peers = sb.get_servers_for_psi("a")
self.failUnlessEqual(len(permuted_peers), self.numclients)
d.addCallback(_check_connections)
add_to_sparent=True))
def _added(extra_node):
self.extra_node = extra_node
+ self.extra_node.DEFAULT_ENCODING_PARAMETERS['happy'] = 5
d.addCallback(_added)
HELPER_DATA = "Data that needs help to upload" * 1000
# this is really bytes received rather than sent, but it's
# convenient and basically measures the same thing
bytes_sent = results.ciphertext_fetched
+ self.failUnless(isinstance(bytes_sent, (int, long)), bytes_sent)
# We currently don't support resumption of upload if the data is
# encrypted with a random key. (Because that would require us
# Make sure we did not have to read the whole file the
# second time around .
self.failUnless(bytes_sent < len(DATA),
- "resumption didn't save us any work:"
- " read %d bytes out of %d total" %
- (bytes_sent, len(DATA)))
+ "resumption didn't save us any work:"
+ " read %r bytes out of %r total" %
+ (bytes_sent, len(DATA)))
else:
# Make sure we did have to read the whole file the second
# time around -- because the one that we partially uploaded
# earlier was encrypted with a different random key.
self.failIf(bytes_sent < len(DATA),
"resumption saved us some work even though we were using random keys:"
- " read %d bytes out of %d total" %
+ " read %r bytes out of %r total" %
(bytes_sent, len(DATA)))
n = self.clients[1].create_node_from_uri(cap)
return download_to_data(n)
return d
- def _find_shares(self, basedir):
+ def _find_all_shares(self, basedir):
shares = []
for (dirpath, dirnames, filenames) in os.walk(basedir):
if "storage" not in dirpath:
def test_mutable(self):
self.basedir = "system/SystemTest/test_mutable"
DATA = "initial contents go here." # 25 bytes % 3 != 0
+ DATA_uploadable = MutableData(DATA)
NEWDATA = "new contents yay"
+ NEWDATA_uploadable = MutableData(NEWDATA)
NEWERDATA = "this is getting old"
+ NEWERDATA_uploadable = MutableData(NEWERDATA)
d = self.set_up_nodes(use_key_generator=True)
def _create_mutable(res):
c = self.clients[0]
log.msg("starting create_mutable_file")
- d1 = c.create_mutable_file(DATA)
+ d1 = c.create_mutable_file(DATA_uploadable)
def _done(res):
log.msg("DONE: %s" % (res,))
self._mutable_node_1 = res
def _test_debug(res):
# find a share. It is important to run this while there is only
# one slot in the grid.
- shares = self._find_shares(self.basedir)
+ shares = self._find_all_shares(self.basedir)
(client_num, storage_index, filename, shnum) = shares[0]
log.msg("test_system.SystemTest.test_mutable._test_debug using %s"
% filename)
self.failUnlessEqual(res, DATA)
# replace the data
log.msg("starting replace1")
- d1 = newnode.overwrite(NEWDATA)
+ d1 = newnode.overwrite(NEWDATA_uploadable)
d1.addCallback(lambda res: newnode.download_best_version())
return d1
d.addCallback(_check_download_3)
newnode2 = self.clients[3].create_node_from_uri(uri)
self._newnode3 = self.clients[3].create_node_from_uri(uri)
log.msg("starting replace2")
- d1 = newnode1.overwrite(NEWERDATA)
+ d1 = newnode1.overwrite(NEWERDATA_uploadable)
d1.addCallback(lambda res: newnode2.download_best_version())
return d1
d.addCallback(_check_download_4)
def _corrupt_shares(res):
# run around and flip bits in all but k of the shares, to test
# the hash checks
- shares = self._find_shares(self.basedir)
+ shares = self._find_all_shares(self.basedir)
## sort by share number
#shares.sort( lambda a,b: cmp(a[3], b[3]) )
where = dict([ (shnum, filename)
def _check_empty_file(res):
# make sure we can create empty files, this usually screws up the
# segsize math
- d1 = self.clients[2].create_mutable_file("")
+ d1 = self.clients[2].create_mutable_file(MutableData(""))
d1.addCallback(lambda newnode: newnode.download_best_version())
d1.addCallback(lambda res: self.failUnlessEqual("", res))
return d1
self.key_generator_svc.key_generator.pool_size + size_delta)
d.addCallback(check_kg_poolsize, 0)
- d.addCallback(lambda junk: self.clients[3].create_mutable_file('hello, world'))
+ d.addCallback(lambda junk:
+ self.clients[3].create_mutable_file(MutableData('hello, world')))
d.addCallback(check_kg_poolsize, -1)
d.addCallback(lambda junk: self.clients[3].create_dirnode())
d.addCallback(check_kg_poolsize, -2)
self.basedir = "system/SystemTest/test_filesystem"
self.data = LARGE_DATA
d = self.set_up_nodes(use_stats_gatherer=True)
+ def _new_happy_semantics(ign):
+ for c in self.clients:
+ c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ d.addCallback(_new_happy_semantics)
d.addCallback(self._test_introweb)
d.addCallback(self.log, "starting publish")
d.addCallback(self._do_publish1)
d = getPage(self.introweb_url, method="GET", followRedirect=True)
def _check(res):
try:
- self.failUnless("allmydata-tahoe: %s" % str(allmydata.__version__)
- in res)
+ self.failUnless("%s: %s" % (allmydata.__appname__, allmydata.__version__) in res)
+ verstr = str(allmydata.__version__)
+
+ # The Python "rational version numbering" convention
+ # disallows "-r$REV" but allows ".post$REV"
+ # instead. Eventually we'll probably move to
+ # that. When we do, this test won't go red:
+ ix = verstr.rfind('-r')
+ if ix != -1:
+ altverstr = verstr[:ix] + '.post' + verstr[ix+2:]
+ else:
+ ix = verstr.rfind('.post')
+ if ix != -1:
+ altverstr = verstr[:ix] + '-r' + verstr[ix+5:]
+ else:
+ altverstr = verstr
+
+ appverstr = "%s: %s" % (allmydata.__appname__, verstr)
+ newappverstr = "%s: %s" % (allmydata.__appname__, altverstr)
+
+ self.failUnless((appverstr in res) or (newappverstr in res), (appverstr, newappverstr, res))
self.failUnless("Announcement Summary: storage: 5, stub_client: 5" in res)
self.failUnless("Subscription Summary: storage: 5" in res)
except unittest.FailTest:
d.addCallback(lambda res: getPage(base + public + "/subdir1"))
def _got_subdir1(page):
# there ought to be an href for our file
- self.failUnless(("<td>%d</td>" % len(self.data)) in page)
+ self.failUnlessIn('<td align="right">%d</td>' % len(self.data), page)
self.failUnless(">mydata567</a>" in page)
d.addCallback(_got_subdir1)
d.addCallback(self.log, "done with _got_subdir1")
d.addCallback(self.failUnlessEqual, "new.txt contents")
# and again with something large enough to use multiple segments,
# and hopefully trigger pauseProducing too
+ def _new_happy_semantics(ign):
+ for c in self.clients:
+ # these get reset somewhere? Whatever.
+ c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ d.addCallback(_new_happy_semantics)
d.addCallback(lambda res: self.PUT(public + "/subdir3/big.txt",
"big" * 500000)) # 1.5MB
d.addCallback(lambda res: self.GET(public + "/subdir3/big.txt"))
# exercise some of the diagnostic tools in runner.py
# find a share
- for (dirpath, dirnames, filenames) in os.walk(self.basedir):
+ for (dirpath, dirnames, filenames) in os.walk(unicode(self.basedir)):
if "storage" not in dirpath:
continue
if not filenames:
if magic == '\x00\x00\x00\x01':
break
else:
- self.fail("unable to find any uri_extension files in %s"
+ self.fail("unable to find any uri_extension files in %r"
% self.basedir)
- log.msg("test_system.SystemTest._test_runner using %s" % filename)
+ log.msg("test_system.SystemTest._test_runner using %r" % filename)
out,err = StringIO(), StringIO()
rc = runner.runner(["debug", "dump-share", "--offsets",
- filename],
+ unicode_to_argv(filename)],
stdout=out, stderr=err)
output = out.getvalue()
self.failUnlessEqual(rc, 0)
# we only upload a single file, so we can assert some things about
# its size and shares.
- self.failUnless(("share filename: %s" % filename) in output)
- self.failUnless("size: %d\n" % len(self.data) in output)
- self.failUnless("num_segments: 1\n" in output)
+ self.failUnlessIn("share filename: %s" % quote_output(abspath_expanduser_unicode(filename)), output)
+ self.failUnlessIn("size: %d\n" % len(self.data), output)
+ self.failUnlessIn("num_segments: 1\n", output)
# segment_size is always a multiple of needed_shares
- self.failUnless("segment_size: %d\n" % mathutil.next_multiple(len(self.data), 3) in output)
- self.failUnless("total_shares: 10\n" in output)
+ self.failUnlessIn("segment_size: %d\n" % mathutil.next_multiple(len(self.data), 3), output)
+ self.failUnlessIn("total_shares: 10\n", output)
# keys which are supposed to be present
for key in ("size", "num_segments", "segment_size",
"needed_shares", "total_shares",
#"plaintext_hash", "plaintext_root_hash",
"crypttext_hash", "crypttext_root_hash",
"share_root_hash", "UEB_hash"):
- self.failUnless("%s: " % key in output, key)
- self.failUnless(" verify-cap: URI:CHK-Verifier:" in output)
+ self.failUnlessIn("%s: " % key, output)
+ self.failUnlessIn(" verify-cap: URI:CHK-Verifier:", output)
# now use its storage index to find the other shares using the
# 'find-shares' tool
sharedir, shnum = os.path.split(filename)
storagedir, storage_index_s = os.path.split(sharedir)
+ storage_index_s = str(storage_index_s)
out,err = StringIO(), StringIO()
nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
cmd = ["debug", "find-shares", storage_index_s] + nodedirs
d.addCallback(self._test_control2, control_furl_file)
return d
def _test_control2(self, rref, filename):
- d = rref.callRemote("upload_from_file_to_uri", filename, convergence=None)
- downfile = os.path.join(self.basedir, "control.downfile")
+ d = rref.callRemote("upload_from_file_to_uri",
+ filename.encode(get_filesystem_encoding()), convergence=None)
+ downfile = os.path.join(self.basedir, "control.downfile").encode(get_filesystem_encoding())
d.addCallback(lambda uri:
rref.callRemote("download_from_uri_to_file",
uri, downfile))
self.failUnlessEqual(data, expected_data)
d.addCallback(_check)
d.addCallback(lambda res: rref.callRemote("speed_test", 1, 200, False))
- if sys.platform == "linux2":
+ if sys.platform in ("linux2", "linux3"):
d.addCallback(lambda res: rref.callRemote("get_memory_usage"))
d.addCallback(lambda res: rref.callRemote("measure_peer_response_time"))
return d
d.addCallback(run, "list-aliases")
def _check_aliases_1((out,err)):
self.failUnlessEqual(err, "")
- self.failUnlessEqual(out, "tahoe: %s\n" % private_uri)
+ self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % private_uri)
d.addCallback(_check_aliases_1)
# now that that's out of the way, remove root_dir.cap and work with
return d
+ def test_filesystem_with_cli_in_subprocess(self):
+ # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
+
+ self.basedir = "system/SystemTest/test_filesystem_with_cli_in_subprocess"
+ d = self.set_up_nodes()
+ def _new_happy_semantics(ign):
+ for c in self.clients:
+ c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ d.addCallback(_new_happy_semantics)
+
+ def _run_in_subprocess(ignored, verb, *args, **kwargs):
+ stdin = kwargs.get("stdin")
+ env = kwargs.get("env")
+ newargs = [verb, "--node-directory", self.getdir("client0")] + list(args)
+ return self.run_bintahoe(newargs, stdin=stdin, env=env)
+
+ def _check_succeeded(res, check_stderr=True):
+ out, err, rc_or_sig = res
+ self.failUnlessEqual(rc_or_sig, 0, str(res))
+ if check_stderr:
+ self.failUnlessEqual(err, "")
+
+ d.addCallback(_run_in_subprocess, "create-alias", "newalias")
+ d.addCallback(_check_succeeded)
+
+ STDIN_DATA = "This is the file to upload from stdin."
+ d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA)
+ d.addCallback(_check_succeeded, check_stderr=False)
+
+ def _mv_with_http_proxy(ign):
+ env = os.environ
+ env['http_proxy'] = env['HTTP_PROXY'] = "http://127.0.0.0:12345" # invalid address
+ return _run_in_subprocess(None, "mv", "newalias:tahoe-file", "newalias:tahoe-moved", env=env)
+ d.addCallback(_mv_with_http_proxy)
+ d.addCallback(_check_succeeded)
+
+ d.addCallback(_run_in_subprocess, "ls", "newalias:")
+ def _check_ls(res):
+ out, err, rc_or_sig = res
+ self.failUnlessEqual(rc_or_sig, 0, str(res))
+ self.failUnlessEqual(err, "", str(res))
+ self.failUnlessIn("tahoe-moved", out)
+ self.failIfIn("tahoe-file", out)
+ d.addCallback(_check_ls)
+ return d
+
+ def test_debug_trial(self):
+ def _check_for_line(lines, result, test):
+ for l in lines:
+ if result in l and test in l:
+ return
+ self.fail("output (prefixed with '##') does not have a line containing both %r and %r:\n## %s"
+ % (result, test, "\n## ".join(lines)))
+
+ def _check_for_outcome(lines, out, outcome):
+ self.failUnlessIn(outcome, out, "output (prefixed with '##') does not contain %r:\n## %s"
+ % (outcome, "\n## ".join(lines)))
+
+ d = self.run_bintahoe(['debug', 'trial', '--reporter=verbose',
+ 'allmydata.test.trialtest'])
+ def _check_failure( (out, err, rc) ):
+ self.failUnlessEqual(rc, 1)
+ lines = out.split('\n')
+ _check_for_line(lines, "[SKIPPED]", "test_skip")
+ _check_for_line(lines, "[TODO]", "test_todo")
+ _check_for_line(lines, "[FAIL]", "test_fail")
+ _check_for_line(lines, "[ERROR]", "test_deferred_error")
+ _check_for_line(lines, "[ERROR]", "test_error")
+ _check_for_outcome(lines, out, "FAILED")
+ d.addCallback(_check_failure)
+
+ # the --quiet argument regression-tests a problem in finding which arguments to pass to trial
+ d.addCallback(lambda ign: self.run_bintahoe(['--quiet', 'debug', 'trial', '--reporter=verbose',
+ 'allmydata.test.trialtest.Success']))
+ def _check_success( (out, err, rc) ):
+ self.failUnlessEqual(rc, 0)
+ lines = out.split('\n')
+ _check_for_line(lines, "[SKIPPED]", "test_skip")
+ _check_for_line(lines, "[TODO]", "test_todo")
+ _check_for_outcome(lines, out, "PASSED")
+ d.addCallback(_check_success)
+ return d
+
def _run_cli(self, argv, stdin=""):
#print "CLI:", argv
stdout, stderr = StringIO(), StringIO()
return d
d.addCallback(_got_lit_filenode)
return d
-
projects / tahoe-lafs / tahoe-lafs.git / blobdiff