from twisted.trial import unittest
from twisted.python import usage, runtime
-from twisted.internet import utils
+from twisted.internet import threads
+
import os.path, re, sys, subprocess
from cStringIO import StringIO
from allmydata.util import fileutil, pollmixin
bintahoe = alt_bintahoe
-class SkipMixin:
+class RunBinTahoeMixin:
def skip_if_cannot_run_bintahoe(self):
- if "cygwin" in sys.platform.lower():
- raise unittest.SkipTest("We don't know how to make this test work on cygwin: spawnProcess seems to hang forever. We don't know if 'bin/tahoe start' can be run on cygwin.")
if not os.path.exists(bintahoe):
raise unittest.SkipTest("The bin/tahoe script isn't to be found in the expected location (%s), and I don't want to test a 'tahoe' executable that I find somewhere else, in case it isn't the right executable for this version of Tahoe. Perhaps running 'setup.py build' again will help." % (bintahoe,))
# twistd on windows doesn't daemonize. cygwin should work normally.
raise unittest.SkipTest("twistd does not fork under windows")
+ def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
+ self.skip_if_cannot_run_bintahoe()
+ command = [sys.executable] + python_options + [bintahoe] + args
+ if stdin is None:
+ stdin_stream = None
+ else:
+ stdin_stream = subprocess.PIPE
+
+ def _run():
+ p = subprocess.Popen(command, stdin=stdin_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ (out, err) = p.communicate(stdin)
+ return (out, err, p.returncode)
+ return threads.deferToThread(_run)
+
-class BinTahoe(common_util.SignalMixin, unittest.TestCase, SkipMixin):
+class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin):
def _check_right_code(self, file_to_check):
root_to_check = get_root_from_file(file_to_check)
cwd = os.path.normcase(os.path.realpath("."))
self._check_right_code(srcfile)
def test_import_in_repl(self):
- self.skip_if_cannot_run_bintahoe()
-
- p = subprocess.Popen([sys.executable, bintahoe, "debug", "repl"],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (out, err) = p.communicate("import allmydata; print; print allmydata.__file__")
-
- self.failUnlessEqual(p.returncode, 0)
- lines = out.splitlines()
- self.failUnlessIn('>>>', lines[0], (out, err))
- self._check_right_code(lines[1])
+ d = self.run_bintahoe(["debug", "repl"],
+ stdin="import allmydata; print; print allmydata.__file__")
+ def _cb(res):
+ out, err, rc_or_sig = res
+ self.failUnlessEqual(rc_or_sig, 0, str(res))
+ lines = out.splitlines()
+ self.failUnlessIn('>>>', lines[0], str(res))
+ self._check_right_code(lines[1])
+ d.addCallback(_cb)
+ return d
def test_path(self):
- self.skip_if_cannot_run_bintahoe()
- d = utils.getProcessOutputAndValue(bintahoe, args=["--version-and-path"], env=os.environ)
+ d = self.run_bintahoe(["--version-and-path"])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0, str(res))
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII argument/output could not be encoded on this platform.")
- d = utils.getProcessOutputAndValue(bintahoe, args=[tricky_arg], env=os.environ)
+ d = self.run_bintahoe([tricky_arg])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 1, str(res))
self.skip_if_cannot_run_bintahoe()
# -t is a harmless option that warns about tabs.
- d = utils.getProcessOutputAndValue(sys.executable, args=['-t', bintahoe, '--version'],
- env=os.environ)
+ d = self.run_bintahoe(["--version"], python_options=["-t"])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0, str(res))
except pkg_resources.VersionConflict:
raise unittest.SkipTest("We pass this test only with Twisted >= v9.0.0")
- d = utils.getProcessOutputAndValue(bintahoe, args=["--version"], env=os.environ)
+ d = self.run_bintahoe(["--version"])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0, str(res))
class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
- SkipMixin):
+ RunBinTahoeMixin):
# exercise "tahoe start", for both introducer, client node, and
# key-generator, by spawning "tahoe start" as a subprocess. This doesn't
# get us figleaf-based line-level coverage, but it does a better job of
TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
INTRODUCER_FURL_FILE = os.path.join(c1, "introducer.furl")
- d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-introducer", "--basedir", c1], env=os.environ)
+ d = self.run_bintahoe(["--quiet", "create-introducer", "--basedir", c1])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0)
d.addCallback(_cb)
def _then_start_the_node(res):
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "start", c1])
d.addCallback(_then_start_the_node)
def _cb2(res):
self.failUnless(os.path.exists(TWISTD_PID_FILE))
# rm this so we can detect when the second incarnation is ready
os.unlink(INTRODUCER_FURL_FILE)
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "restart", c1])
d.addCallback(_started)
def _then(res):
open(HOTLINE_FILE, "w").write("")
self.failUnless(os.path.exists(TWISTD_PID_FILE))
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "stop", c1])
d.addCallback(_stop)
def _after_stopping(res):
TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
PORTNUMFILE = os.path.join(c1, "client.port")
- d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-client", "--basedir", c1, "--webport", "0"], env=os.environ)
+ d = self.run_bintahoe(["--quiet", "create-client", "--basedir", c1, "--webport", "0"])
def _cb(res):
out, err, rc_or_sig = res
errstr = "cc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
d.addCallback(_cb)
def _start(res):
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "start", c1])
d.addCallback(_start)
def _cb2(res):
# 'tahoe stop' command takes a while.
def _stop(res):
self.failUnless(os.path.exists(TWISTD_PID_FILE), (TWISTD_PID_FILE, os.listdir(os.path.dirname(TWISTD_PID_FILE))))
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "stop", c1])
d.addCallback(_stop)
return d
TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
PORTNUMFILE = os.path.join(c1, "client.port")
- d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-node", "--basedir", c1, "--webport", "0"], env=os.environ)
+ d = self.run_bintahoe(["--quiet", "create-node", "--basedir", c1, "--webport", "0"])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0)
d.addCallback(_cb)
def _start(res):
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "start", c1])
d.addCallback(_start)
def _cb2(res):
# rm this so we can detect when the second incarnation is ready
os.unlink(PORTNUMFILE)
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "restart", c1])
d.addCallback(_started)
def _cb3(res):
def _stop(res):
open(HOTLINE_FILE, "w").write("")
self.failUnless(os.path.exists(TWISTD_PID_FILE), (TWISTD_PID_FILE, os.listdir(os.path.dirname(TWISTD_PID_FILE))))
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "stop", c1])
d.addCallback(_stop)
def _cb4(res):
basedir = self.workdir("test_baddir")
fileutil.make_dirs(basedir)
- d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", "--basedir", basedir], env=os.environ)
+ d = self.run_bintahoe(["--quiet", "start", "--basedir", basedir])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 1)
d.addCallback(_cb)
def _then_stop_it(res):
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", "--basedir", basedir], env=os.environ)
+ return self.run_bintahoe(["--quiet", "stop", "--basedir", basedir])
d.addCallback(_then_stop_it)
def _cb2(res):
def _then_start_in_bogus_basedir(res):
not_a_dir = os.path.join(basedir, "bogus")
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", "--basedir", not_a_dir], env=os.environ)
+ return self.run_bintahoe(["--quiet", "start", "--basedir", not_a_dir])
d.addCallback(_then_start_in_bogus_basedir)
def _cb3(res):
TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
KEYGEN_FURL_FILE = os.path.join(c1, "key_generator.furl")
- d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-key-generator", "--basedir", c1], env=os.environ)
+ d = self.run_bintahoe(["--quiet", "create-key-generator", "--basedir", c1])
def _cb(res):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0)
d.addCallback(_cb)
def _start(res):
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "start", c1])
d.addCallback(_start)
def _cb2(res):
self.failUnless(os.path.exists(TWISTD_PID_FILE))
# rm this so we can detect when the second incarnation is ready
os.unlink(KEYGEN_FURL_FILE)
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "restart", c1])
d.addCallback(_started)
def _cb3(res):
# 'tahoe stop' command takes a while.
def _stop(res):
self.failUnless(os.path.exists(TWISTD_PID_FILE))
- return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
+ return self.run_bintahoe(["--quiet", "stop", c1])
d.addCallback(_stop)
def _cb4(res):
from twisted.trial import unittest
from twisted.internet import defer
from twisted.internet import threads # CLI tests use deferToThread
-from twisted.internet import utils
import allmydata
from allmydata import uri
from allmydata.test.common import SystemTestMixin
# TODO: move these to common or common_util
-from allmydata.test.test_runner import bintahoe, SkipMixin
+from allmydata.test.test_runner import RunBinTahoeMixin
LARGE_DATA = """
This is some data to publish to the remote grid.., which needs to be large
self.interrupt_after_d.callback(self)
return upload.Data.read(self, length)
-class SystemTest(SystemTestMixin, SkipMixin, unittest.TestCase):
+class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
timeout = 3600 # It takes longer than 960 seconds on Zandr's ARM box.
def test_connections(self):
d.addCallback(_check_ls_rouri)
- d.addCallback(run, "mv", "tahoe-file-stdin", "tahoe-moved-first-time")
+ d.addCallback(run, "mv", "tahoe-file-stdin", "tahoe-moved")
d.addCallback(run, "ls")
- d.addCallback(_check_ls, ["tahoe-moved-first-time"], ["tahoe-file-stdin"])
-
- def _mv_with_http_proxy(ign):
- env = os.environ
- env['http_proxy'] = env['HTTP_PROXY'] = "http://127.0.0.0:12345" # invalid address
- return self._run_cli_in_subprocess(["mv"] + nodeargs + ["tahoe-moved-first-time", "tahoe-moved"], env=env)
- d.addCallback(_mv_with_http_proxy)
-
- def _check_mv_with_http_proxy(res):
- out, err, rc_or_sig = res
- self.failUnlessEqual(rc_or_sig, 0, str(res))
- d.addCallback(_check_mv_with_http_proxy)
-
- d.addCallback(run, "ls")
- d.addCallback(_check_ls, ["tahoe-moved"], ["tahoe-moved-firsttime"])
+ d.addCallback(_check_ls, ["tahoe-moved"], ["tahoe-file-stdin"])
d.addCallback(run, "ln", "tahoe-moved", "newlink")
d.addCallback(run, "ls")
return d
+ def test_filesystem_with_cli_in_subprocess(self):
+ # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
+
+ self.basedir = "system/SystemTest/test_filesystem_with_cli_in_subprocess"
+ d = self.set_up_nodes()
+ def _new_happy_semantics(ign):
+ for c in self.clients:
+ c.DEFAULT_ENCODING_PARAMETERS['happy'] = 1
+ d.addCallback(_new_happy_semantics)
+
+ def _run_in_subprocess(ignored, verb, *args, **kwargs):
+ stdin = kwargs.get("stdin")
+ env = kwargs.get("env")
+ newargs = [verb, "--node-directory", self.getdir("client0")] + list(args)
+ return self.run_bintahoe(newargs, stdin=stdin, env=env)
+
+ def _check_succeeded(res, check_stderr=True):
+ out, err, rc_or_sig = res
+ self.failUnlessEqual(rc_or_sig, 0, str(res))
+ if check_stderr:
+ self.failUnlessEqual(err, "")
+
+ d.addCallback(_run_in_subprocess, "create-alias", "newalias")
+ d.addCallback(_check_succeeded)
+
+ STDIN_DATA = "This is the file to upload from stdin."
+ d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA)
+ d.addCallback(_check_succeeded, check_stderr=False)
+
+ def _mv_with_http_proxy(ign):
+ env = os.environ
+ env['http_proxy'] = env['HTTP_PROXY'] = "http://127.0.0.0:12345" # invalid address
+ return _run_in_subprocess(None, "mv", "newalias:tahoe-file", "newalias:tahoe-moved", env=env)
+ d.addCallback(_mv_with_http_proxy)
+ d.addCallback(_check_succeeded)
+
+ d.addCallback(_run_in_subprocess, "ls", "newalias:")
+ def _check_ls(res):
+ out, err, rc_or_sig = res
+ self.failUnlessEqual(rc_or_sig, 0, str(res))
+ self.failUnlessEqual(err, "", str(res))
+ self.failUnlessIn("tahoe-moved", out)
+ self.failIfIn("tahoe-file", out)
+ d.addCallback(_check_ls)
+ return d
+
def test_debug_trial(self):
def _check_for_line(lines, result, test):
for l in lines:
self.failUnlessIn(outcome, out, "output (prefixed with '##') does not contain %r:\n## %s"
% (outcome, "\n## ".join(lines)))
- d = self._run_cli_in_subprocess(['debug', 'trial', '--reporter=verbose',
- 'allmydata.test.trialtest'])
+ d = self.run_bintahoe(['debug', 'trial', '--reporter=verbose',
+ 'allmydata.test.trialtest'])
def _check_failure( (out, err, rc) ):
self.failUnlessEqual(rc, 1)
lines = out.split('\n')
d.addCallback(_check_failure)
# the --quiet argument regression-tests a problem in finding which arguments to pass to trial
- d.addCallback(lambda ign: self._run_cli_in_subprocess(['--quiet', 'debug', 'trial', '--reporter=verbose',
- 'allmydata.test.trialtest.Success']))
+ d.addCallback(lambda ign: self.run_bintahoe(['--quiet', 'debug', 'trial', '--reporter=verbose',
+ 'allmydata.test.trialtest.Success']))
def _check_success( (out, err, rc) ):
self.failUnlessEqual(rc, 0)
lines = out.split('\n')
d.addCallback(_done)
return d
- def _run_cli_in_subprocess(self, argv, env=None):
- self.skip_if_cannot_run_bintahoe()
-
- if env is None:
- env = os.environ
- d = utils.getProcessOutputAndValue(sys.executable, args=[bintahoe] + argv,
- env=env)
- return d
-
def _test_checker(self, res):
ut = upload.Data("too big to be literal" * 200, convergence=None)
d = self._personal_node.add_file(u"big file", ut)
# from the Python Standard Library
-import os, re, socket, sys
+import os, re, socket, sys, subprocess
# from Twisted
-from twisted.internet import defer
-from twisted.internet import reactor
+from twisted.internet import defer, threads, reactor
from twisted.internet.protocol import DatagramProtocol
-from twisted.internet.utils import getProcessOutput
from twisted.python.procutils import which
from twisted.python import log
-from allmydata.util import observer
-
try:
import resource
def increase_rlimits():
# Solaris 2.x
_sunos_path = '/usr/sbin/ifconfig'
-class SequentialTrier(object):
- """ I hold a list of executables to try and try each one in turn
- until one gives me a list of IP addresses."""
-
- def __init__(self, exebasename, args, regex):
- assert not os.path.isabs(exebasename)
- self.exes_left_to_try = which(exebasename)
- self.exes_left_to_try.reverse()
- self.args = args
- self.regex = regex
- self.o = observer.OneShotObserverList()
- self._try_next()
-
- def _try_next(self):
- if not self.exes_left_to_try:
- self.o.fire(None)
- else:
- exe = self.exes_left_to_try.pop()
- d2 = _query(exe, self.args, self.regex)
-
- def cb(res):
- if res:
- self.o.fire(res)
- else:
- self._try_next()
-
- def eb(why):
- self._try_next()
-
- d2.addCallbacks(cb, eb)
-
- def when_tried(self):
- return self.o.when_fired()
# k: platform string as provided in the value of _platform_map
# v: tuple of (path_to_tool, args, regex,)
"irix": (_irix_path, _netbsd_args, _netbsd_re,),
"sunos": (_sunos_path, _netbsd_args, _netbsd_re,),
}
+
def _find_addresses_via_config():
+ return threads.deferToThread(_synchronously_find_addresses_via_config)
+
+def _synchronously_find_addresses_via_config():
# originally by Greg Smith, hacked by Zooko to conform to Brian's API
platform = _platform_map.get(sys.platform)
if os.path.isabs(pathtotool):
return _query(pathtotool, args, regex)
else:
- return SequentialTrier(pathtotool, args, regex).when_tried()
+ exes_to_try = which(pathtotool)
+ for exe in exes_to_try:
+ try:
+ addresses = _query(exe, args, regex)
+ except Exception:
+ addresses = []
+ if addresses:
+ return addresses
+ return []
def _query(path, args, regex):
- d = getProcessOutput(path, args)
- def _parse(output):
- addresses = []
- outputsplit = output.split('\n')
- for outline in outputsplit:
- m = regex.match(outline)
- if m:
- addr = m.groupdict()['address']
- if addr not in addresses:
- addresses.append(addr)
+ p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (output, err) = p.communicate()
- return addresses
- d.addCallback(_parse)
- return d
+ addresses = []
+ outputsplit = output.split('\n')
+ for outline in outputsplit:
+ m = regex.match(outline)
+ if m:
+ addr = m.groupdict()['address']
+ if addr not in addresses:
+ addresses.append(addr)
+
+ return addresses
def _cygwin_hack_find_addresses(target):
addresses = []
projects / tahoe-lafs / tahoe-lafs.git / commitdiff