src/allmydata/test/test_runner.py

   1 # -*- coding: utf-8 -*-
   2
   3 from twisted.trial import unittest
   4
   5 from twisted.python import usage, runtime
   6 from twisted.internet import utils
   7 import os.path, re, sys
   8 from cStringIO import StringIO
   9 from allmydata.util import fileutil, pollmixin
  10 from allmydata.scripts import runner
  11
  12 from allmydata.test import common_util
  13 import allmydata
  14
  15 bintahoe = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(allmydata.__file__))), 'bin', 'tahoe')
  16 if sys.platform == "win32":  # TODO: should this include cygwin?
  17     bintahoe += ".exe"
  18
  19
  20 class SkipMixin:
  21     def skip_if_cannot_run_bintahoe(self):
  22         if "cygwin" in sys.platform.lower():
  23             raise unittest.SkipTest("We don't know how to make this test work on cygwin: spawnProcess seems to hang forever. We don't know if 'bin/tahoe start' can be run on cygwin.")
  24         if not os.path.exists(bintahoe):
  25             raise unittest.SkipTest("The bin/tahoe script isn't to be found in the expected location (%s), and I don't want to test a 'tahoe' executable that I find somewhere else, in case it isn't the right executable for this version of Tahoe. Perhaps running 'setup.py build' again will help." % (bintahoe,))
  26
  27     def skip_if_cannot_daemonize(self):
  28         self.skip_if_cannot_run_bintahoe()
  29         if runtime.platformType == "win32":
  30             # twistd on windows doesn't daemonize. cygwin should work normally.
  31             raise unittest.SkipTest("twistd does not fork under windows")
  32
  33
  34 class TheRightCode(common_util.SignalMixin, unittest.TestCase, SkipMixin):
  35     def test_path(self):
  36         self.skip_if_cannot_run_bintahoe()
  37         d = utils.getProcessOutputAndValue(bintahoe, args=["--version-and-path"], env=os.environ)
  38         def _cb(res):
  39             out, err, rc_or_sig = res
  40             self.failUnlessEqual(rc_or_sig, 0)
  41
  42             # Fail unless the allmydata-tahoe package is *this* version *and*
  43             # was loaded from *this* source directory.
  44             ad = os.path.dirname(os.path.dirname(os.path.realpath(allmydata.__file__)))
  45             required_ver_and_path = "allmydata-tahoe: %s (%s)" % (allmydata.__version__, ad)
  46             self.failUnless(out.startswith(required_ver_and_path),
  47                             (out, err, rc_or_sig, required_ver_and_path))
  48         d.addCallback(_cb)
  49         return d
  50
  51
  52 class CreateNode(unittest.TestCase):
  53     # exercise "tahoe create-node", create-introducer,
  54     # create-key-generator, and create-stats-gatherer, by calling the
  55     # corresponding code as a subroutine.
  56
  57     def workdir(self, name):
  58         basedir = os.path.join("test_runner", "CreateNode", name)
  59         fileutil.make_dirs(basedir)
  60         return basedir
  61
  62     def run_tahoe(self, argv):
  63         out,err = StringIO(), StringIO()
  64         rc = runner.runner(argv, stdout=out, stderr=err)
  65         return rc, out.getvalue(), err.getvalue()
  66
  67     def do_create(self, command, basedir):
  68         c1 = os.path.join(basedir, command + "-c1")
  69         argv = ["--quiet", command, "--basedir", c1]
  70         rc, out, err = self.run_tahoe(argv)
  71         self.failUnlessEqual(err, "")
  72         self.failUnlessEqual(out, "")
  73         self.failUnlessEqual(rc, 0)
  74         self.failUnless(os.path.exists(c1))
  75         self.failUnless(os.path.exists(os.path.join(c1, "tahoe-client.tac")))
  76
  77         # tahoe.cfg should exist, and should have storage enabled for
  78         # 'create-node', and disabled for 'create-client'.
  79         tahoe_cfg = os.path.join(c1, "tahoe.cfg")
  80         self.failUnless(os.path.exists(tahoe_cfg))
  81         content = open(tahoe_cfg).read()
  82         if command == "create-client":
  83             self.failUnless("\n[storage]\nenabled = false\n" in content)
  84         else:
  85             self.failUnless("\n[storage]\nenabled = true\n" in content)
  86
  87         # creating the client a second time should be rejected
  88         rc, out, err = self.run_tahoe(argv)
  89         self.failIfEqual(rc, 0, str((out, err, rc)))
  90         self.failUnlessEqual(out, "")
  91         self.failUnless("is not empty." in err)
  92
  93         # Fail if there is a non-empty line that doesn't end with a
  94         # punctuation mark.
  95         for line in err.splitlines():
  96             self.failIf(re.search("[\S][^\.!?]$", line), (line,))
  97
  98         # test that the non --basedir form works too
  99         c2 = os.path.join(basedir, command + "c2")
 100         argv = ["--quiet", command, c2]
 101         rc, out, err = self.run_tahoe(argv)
 102         self.failUnless(os.path.exists(c2))
 103         self.failUnless(os.path.exists(os.path.join(c2, "tahoe-client.tac")))
 104
 105         # make sure it rejects too many arguments
 106         argv = [command, "basedir", "extraarg"]
 107         self.failUnlessRaises(usage.UsageError,
 108                               runner.runner, argv,
 109                               run_by_human=False)
 110
 111     def test_node(self):
 112         basedir = self.workdir("test_node")
 113         self.do_create("create-node", basedir)
 114
 115     def test_client(self):
 116         # create-client should behave like create-node --no-storage.
 117         basedir = self.workdir("test_client")
 118         self.do_create("create-client", basedir)
 119
 120     def test_introducer(self):
 121         basedir = self.workdir("test_introducer")
 122         c1 = os.path.join(basedir, "c1")
 123         argv = ["--quiet", "create-introducer", "--basedir", c1]
 124         rc, out, err = self.run_tahoe(argv)
 125         self.failUnlessEqual(err, "", err)
 126         self.failUnlessEqual(out, "")
 127         self.failUnlessEqual(rc, 0)
 128         self.failUnless(os.path.exists(c1))
 129         self.failUnless(os.path.exists(os.path.join(c1,"tahoe-introducer.tac")))
 130
 131         # creating the introducer a second time should be rejected
 132         rc, out, err = self.run_tahoe(argv)
 133         self.failIfEqual(rc, 0)
 134         self.failUnlessEqual(out, "")
 135         self.failUnless("is not empty" in err)
 136
 137         # Fail if there is a non-empty line that doesn't end with a
 138         # punctuation mark.
 139         for line in err.splitlines():
 140             self.failIf(re.search("[\S][^\.!?]$", line), (line,))
 141
 142         # test the non --basedir form
 143         c2 = os.path.join(basedir, "c2")
 144         argv = ["--quiet", "create-introducer", c2]
 145         rc, out, err = self.run_tahoe(argv)
 146         self.failUnlessEqual(err, "", err)
 147         self.failUnlessEqual(out, "")
 148         self.failUnlessEqual(rc, 0)
 149         self.failUnless(os.path.exists(c2))
 150         self.failUnless(os.path.exists(os.path.join(c2,"tahoe-introducer.tac")))
 151
 152         # reject extra arguments
 153         argv = ["create-introducer", "basedir", "extraarg"]
 154         self.failUnlessRaises(usage.UsageError,
 155                               runner.runner, argv,
 156                               run_by_human=False)
 157         # and require basedir to be provided in some form
 158         argv = ["create-introducer"]
 159         self.failUnlessRaises(usage.UsageError,
 160                               runner.runner, argv,
 161                               run_by_human=False)
 162
 163     def test_key_generator(self):
 164         basedir = self.workdir("test_key_generator")
 165         kg1 = os.path.join(basedir, "kg1")
 166         argv = ["--quiet", "create-key-generator", "--basedir", kg1]
 167         rc, out, err = self.run_tahoe(argv)
 168         self.failUnlessEqual(err, "")
 169         self.failUnlessEqual(out, "")
 170         self.failUnlessEqual(rc, 0)
 171         self.failUnless(os.path.exists(kg1))
 172         self.failUnless(os.path.exists(os.path.join(kg1, "tahoe-key-generator.tac")))
 173
 174         # creating it a second time should be rejected
 175         rc, out, err = self.run_tahoe(argv)
 176         self.failIfEqual(rc, 0, str((out, err, rc)))
 177         self.failUnlessEqual(out, "")
 178         self.failUnless("is not empty." in err)
 179
 180         # make sure it rejects too many arguments
 181         argv = ["create-key-generator", "basedir", "extraarg"]
 182         self.failUnlessRaises(usage.UsageError,
 183                               runner.runner, argv,
 184                               run_by_human=False)
 185
 186         # make sure it rejects a missing basedir specification
 187         argv = ["create-key-generator"]
 188         rc, out, err = self.run_tahoe(argv)
 189         self.failIfEqual(rc, 0, str((out, err, rc)))
 190         self.failUnlessEqual(out, "")
 191         self.failUnless("a basedir was not provided" in err)
 192
 193     def test_stats_gatherer(self):
 194         basedir = self.workdir("test_stats_gatherer")
 195         sg1 = os.path.join(basedir, "sg1")
 196         argv = ["--quiet", "create-stats-gatherer", "--basedir", sg1]
 197         rc, out, err = self.run_tahoe(argv)
 198         self.failUnlessEqual(err, "")
 199         self.failUnlessEqual(out, "")
 200         self.failUnlessEqual(rc, 0)
 201         self.failUnless(os.path.exists(sg1))
 202         self.failUnless(os.path.exists(os.path.join(sg1, "tahoe-stats-gatherer.tac")))
 203
 204         # creating it a second time should be rejected
 205         rc, out, err = self.run_tahoe(argv)
 206         self.failIfEqual(rc, 0, str((out, err, rc)))
 207         self.failUnlessEqual(out, "")
 208         self.failUnless("is not empty." in err)
 209
 210         # test the non --basedir form
 211         kg2 = os.path.join(basedir, "kg2")
 212         argv = ["--quiet", "create-stats-gatherer", kg2]
 213         rc, out, err = self.run_tahoe(argv)
 214         self.failUnlessEqual(err, "", err)
 215         self.failUnlessEqual(out, "")
 216         self.failUnlessEqual(rc, 0)
 217         self.failUnless(os.path.exists(kg2))
 218         self.failUnless(os.path.exists(os.path.join(kg2,"tahoe-stats-gatherer.tac")))
 219
 220         # make sure it rejects too many arguments
 221         argv = ["create-stats-gatherer", "basedir", "extraarg"]
 222         self.failUnlessRaises(usage.UsageError,
 223                               runner.runner, argv,
 224                               run_by_human=False)
 225
 226         # make sure it rejects a missing basedir specification
 227         argv = ["create-stats-gatherer"]
 228         rc, out, err = self.run_tahoe(argv)
 229         self.failIfEqual(rc, 0, str((out, err, rc)))
 230         self.failUnlessEqual(out, "")
 231         self.failUnless("a basedir was not provided" in err)
 232
 233     def test_subcommands(self):
 234         # no arguments should trigger a command listing, via UsageError
 235         self.failUnlessRaises(usage.UsageError,
 236                               runner.runner,
 237                               [],
 238                               run_by_human=False)
 239
 240
 241 class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
 242               SkipMixin):
 243     # exercise "tahoe start", for both introducer, client node, and
 244     # key-generator, by spawning "tahoe start" as a subprocess. This doesn't
 245     # get us figleaf-based line-level coverage, but it does a better job of
 246     # confirming that the user can actually run "./bin/tahoe start" and
 247     # expect it to work. This verifies that bin/tahoe sets up PYTHONPATH and
 248     # the like correctly.
 249
 250     # This doesn't work on cygwin (it hangs forever), so we skip this test
 251     # when we're on cygwin. It is likely that "tahoe start" itself doesn't
 252     # work on cygwin: twisted seems unable to provide a version of
 253     # spawnProcess which really works there.
 254
 255     def workdir(self, name):
 256         basedir = os.path.join("test_runner", "RunNode", name)
 257         fileutil.make_dirs(basedir)
 258         return basedir
 259
 260     def test_introducer(self):
 261         self.skip_if_cannot_daemonize()
 262         basedir = self.workdir("test_introducer")
 263         c1 = os.path.join(basedir, "c1")
 264         HOTLINE_FILE = os.path.join(c1, "suicide_prevention_hotline")
 265         TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
 266         INTRODUCER_FURL_FILE = os.path.join(c1, "introducer.furl")
 267
 268         d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-introducer", "--basedir", c1], env=os.environ)
 269         def _cb(res):
 270             out, err, rc_or_sig = res
 271             self.failUnlessEqual(rc_or_sig, 0)
 272             # by writing this file, we get ten seconds before the node will
 273             # exit. This insures that even if the test fails (and the 'stop'
 274             # command doesn't work), the client should still terminate.
 275             open(HOTLINE_FILE, "w").write("")
 276             # now it's safe to start the node
 277         d.addCallback(_cb)
 278
 279         def _then_start_the_node(res):
 280             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
 281         d.addCallback(_then_start_the_node)
 282
 283         def _cb2(res):
 284             out, err, rc_or_sig = res
 285
 286             open(HOTLINE_FILE, "w").write("")
 287             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 288             self.failUnlessEqual(rc_or_sig, 0, errstr)
 289             self.failUnlessEqual(out, "", errstr)
 290             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 291
 292             # the parent (twistd) has exited. However, twistd writes the pid
 293             # from the child, not the parent, so we can't expect twistd.pid
 294             # to exist quite yet.
 295
 296             # the node is running, but it might not have made it past the
 297             # first reactor turn yet, and if we kill it too early, it won't
 298             # remove the twistd.pid file. So wait until it does something
 299             # that we know it won't do until after the first turn.
 300         d.addCallback(_cb2)
 301
 302         def _node_has_started():
 303             return os.path.exists(INTRODUCER_FURL_FILE)
 304         d.addCallback(lambda res: self.poll(_node_has_started))
 305
 306         def _started(res):
 307             open(HOTLINE_FILE, "w").write("")
 308             self.failUnless(os.path.exists(TWISTD_PID_FILE))
 309             # rm this so we can detect when the second incarnation is ready
 310             os.unlink(INTRODUCER_FURL_FILE)
 311             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
 312         d.addCallback(_started)
 313
 314         def _then(res):
 315             out, err, rc_or_sig = res
 316             open(HOTLINE_FILE, "w").write("")
 317             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 318             self.failUnlessEqual(rc_or_sig, 0, errstr)
 319             self.failUnlessEqual(out, "", errstr)
 320             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 321         d.addCallback(_then)
 322
 323         # again, the second incarnation of the node might not be ready yet,
 324         # so poll until it is
 325         d.addCallback(lambda res: self.poll(_node_has_started))
 326
 327         # now we can kill it. TODO: On a slow machine, the node might kill
 328         # itself before we get a chance too, especially if spawning the
 329         # 'tahoe stop' command takes a while.
 330         def _stop(res):
 331             open(HOTLINE_FILE, "w").write("")
 332             self.failUnless(os.path.exists(TWISTD_PID_FILE))
 333
 334             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
 335         d.addCallback(_stop)
 336
 337         def _after_stopping(res):
 338             out, err, rc_or_sig = res
 339             open(HOTLINE_FILE, "w").write("")
 340             # the parent has exited by now
 341             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 342             self.failUnlessEqual(rc_or_sig, 0, errstr)
 343             self.failUnlessEqual(out, "", errstr)
 344             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 345             # the parent was supposed to poll and wait until it sees
 346             # twistd.pid go away before it exits, so twistd.pid should be
 347             # gone by now.
 348             self.failIf(os.path.exists(TWISTD_PID_FILE))
 349         d.addCallback(_after_stopping)
 350
 351         def _remove_hotline(res):
 352             os.unlink(HOTLINE_FILE)
 353             return res
 354         d.addBoth(_remove_hotline)
 355         return d
 356     test_introducer.timeout = 480 # This hit the 120-second timeout on "François Lenny-armv5tel", then it hit a 240-second timeout on our feisty2.5 buildslave: http://allmydata.org/buildbot/builders/feisty2.5/builds/2381/steps/test/logs/test.log
 357
 358     def test_client(self):
 359         self.skip_if_cannot_daemonize()
 360         basedir = self.workdir("test_client")
 361         c1 = os.path.join(basedir, "c1")
 362         HOTLINE_FILE = os.path.join(c1, "suicide_prevention_hotline")
 363         TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
 364         PORTNUMFILE = os.path.join(c1, "client.port")
 365
 366         d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-node", "--basedir", c1, "--webport", "0"], env=os.environ)
 367         def _cb(res):
 368             out, err, rc_or_sig = res
 369             self.failUnlessEqual(rc_or_sig, 0)
 370             # By writing this file, we get sixty seconds before the client will exit. This insures
 371             # that even if the 'stop' command doesn't work (and the test fails), the client should
 372             # still terminate.
 373             open(HOTLINE_FILE, "w").write("")
 374             open(os.path.join(c1, "introducer.furl"), "w").write("pb://xrndsskn2zuuian5ltnxrte7lnuqdrkz@127.0.0.1:55617/introducer\n")
 375             # now it's safe to start the node
 376         d.addCallback(_cb)
 377
 378         def _start(res):
 379             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
 380         d.addCallback(_start)
 381
 382         def _cb2(res):
 383             out, err, rc_or_sig = res
 384             open(HOTLINE_FILE, "w").write("")
 385             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 386             self.failUnlessEqual(rc_or_sig, 0, errstr)
 387             self.failUnlessEqual(out, "", errstr)
 388             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 389
 390             # the parent (twistd) has exited. However, twistd writes the pid
 391             # from the child, not the parent, so we can't expect twistd.pid
 392             # to exist quite yet.
 393
 394             # the node is running, but it might not have made it past the
 395             # first reactor turn yet, and if we kill it too early, it won't
 396             # remove the twistd.pid file. So wait until it does something
 397             # that we know it won't do until after the first turn.
 398         d.addCallback(_cb2)
 399
 400         def _node_has_started():
 401             return os.path.exists(PORTNUMFILE)
 402         d.addCallback(lambda res: self.poll(_node_has_started))
 403
 404         def _started(res):
 405             open(HOTLINE_FILE, "w").write("")
 406             self.failUnless(os.path.exists(TWISTD_PID_FILE))
 407             # rm this so we can detect when the second incarnation is ready
 408             os.unlink(PORTNUMFILE)
 409
 410             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
 411         d.addCallback(_started)
 412
 413         def _cb3(res):
 414             out, err, rc_or_sig = res
 415
 416             open(HOTLINE_FILE, "w").write("")
 417             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 418             self.failUnlessEqual(rc_or_sig, 0, errstr)
 419             self.failUnlessEqual(out, "", errstr)
 420             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 421         d.addCallback(_cb3)
 422
 423         # again, the second incarnation of the node might not be ready yet,
 424         # so poll until it is
 425         d.addCallback(lambda res: self.poll(_node_has_started))
 426
 427         # now we can kill it. TODO: On a slow machine, the node might kill
 428         # itself before we get a chance too, especially if spawning the
 429         # 'tahoe stop' command takes a while.
 430         def _stop(res):
 431             open(HOTLINE_FILE, "w").write("")
 432             self.failUnless(os.path.exists(TWISTD_PID_FILE), (TWISTD_PID_FILE, os.listdir(os.path.dirname(TWISTD_PID_FILE))))
 433             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
 434         d.addCallback(_stop)
 435
 436         def _cb4(res):
 437             out, err, rc_or_sig = res
 438
 439             open(HOTLINE_FILE, "w").write("")
 440             # the parent has exited by now
 441             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 442             self.failUnlessEqual(rc_or_sig, 0, errstr)
 443             self.failUnlessEqual(out, "", errstr)
 444             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 445             # the parent was supposed to poll and wait until it sees
 446             # twistd.pid go away before it exits, so twistd.pid should be
 447             # gone by now.
 448             self.failIf(os.path.exists(TWISTD_PID_FILE))
 449         d.addCallback(_cb4)
 450         def _remove_hotline(res):
 451             os.unlink(HOTLINE_FILE)
 452             return res
 453         d.addBoth(_remove_hotline)
 454         return d
 455
 456     def test_baddir(self):
 457         self.skip_if_cannot_daemonize()
 458         basedir = self.workdir("test_baddir")
 459         fileutil.make_dirs(basedir)
 460
 461         d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", "--basedir", basedir], env=os.environ)
 462         def _cb(res):
 463             out, err, rc_or_sig = res
 464             self.failUnlessEqual(rc_or_sig, 1)
 465             self.failUnless("does not look like a node directory" in err)
 466         d.addCallback(_cb)
 467
 468         def _then_stop_it(res):
 469             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", "--basedir", basedir], env=os.environ)
 470         d.addCallback(_then_stop_it)
 471
 472         def _cb2(res):
 473             out, err, rc_or_sig = res
 474             self.failUnlessEqual(rc_or_sig, 2)
 475             self.failUnless("does not look like a running node directory" in err)
 476         d.addCallback(_cb2)
 477
 478         def _then_start_in_bogus_basedir(res):
 479             not_a_dir = os.path.join(basedir, "bogus")
 480             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", "--basedir", not_a_dir], env=os.environ)
 481         d.addCallback(_then_start_in_bogus_basedir)
 482
 483         def _cb3(res):
 484             out, err, rc_or_sig = res
 485             self.failUnlessEqual(rc_or_sig, 1)
 486             self.failUnless("does not look like a directory at all" in err, err)
 487         d.addCallback(_cb3)
 488         return d
 489
 490     def test_keygen(self):
 491         self.skip_if_cannot_daemonize()
 492         basedir = self.workdir("test_keygen")
 493         c1 = os.path.join(basedir, "c1")
 494         TWISTD_PID_FILE = os.path.join(c1, "twistd.pid")
 495         KEYGEN_FURL_FILE = os.path.join(c1, "key_generator.furl")
 496
 497         d = utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "create-key-generator", "--basedir", c1], env=os.environ)
 498         def _cb(res):
 499             out, err, rc_or_sig = res
 500             self.failUnlessEqual(rc_or_sig, 0)
 501         d.addCallback(_cb)
 502
 503         def _start(res):
 504             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "start", c1], env=os.environ)
 505         d.addCallback(_start)
 506
 507         def _cb2(res):
 508             out, err, rc_or_sig = res
 509             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 510             self.failUnlessEqual(rc_or_sig, 0, errstr)
 511             self.failUnlessEqual(out, "", errstr)
 512             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 513
 514             # the parent (twistd) has exited. However, twistd writes the pid
 515             # from the child, not the parent, so we can't expect twistd.pid
 516             # to exist quite yet.
 517
 518             # the node is running, but it might not have made it past the
 519             # first reactor turn yet, and if we kill it too early, it won't
 520             # remove the twistd.pid file. So wait until it does something
 521             # that we know it won't do until after the first turn.
 522         d.addCallback(_cb2)
 523
 524         def _node_has_started():
 525             return os.path.exists(KEYGEN_FURL_FILE)
 526         d.addCallback(lambda res: self.poll(_node_has_started))
 527
 528         def _started(res):
 529             self.failUnless(os.path.exists(TWISTD_PID_FILE))
 530             # rm this so we can detect when the second incarnation is ready
 531             os.unlink(KEYGEN_FURL_FILE)
 532             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "restart", c1], env=os.environ)
 533         d.addCallback(_started)
 534
 535         def _cb3(res):
 536             out, err, rc_or_sig = res
 537             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 538             self.failUnlessEqual(rc_or_sig, 0, errstr)
 539             self.failUnlessEqual(out, "", errstr)
 540             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 541         d.addCallback(_cb3)
 542
 543         # again, the second incarnation of the node might not be ready yet,
 544         # so poll until it is
 545         d.addCallback(lambda res: self.poll(_node_has_started))
 546
 547         # now we can kill it. TODO: On a slow machine, the node might kill
 548         # itself before we get a chance too, especially if spawning the
 549         # 'tahoe stop' command takes a while.
 550         def _stop(res):
 551             self.failUnless(os.path.exists(TWISTD_PID_FILE))
 552             return utils.getProcessOutputAndValue(bintahoe, args=["--quiet", "stop", c1], env=os.environ)
 553         d.addCallback(_stop)
 554
 555         def _cb4(res):
 556             out, err, rc_or_sig = res
 557             # the parent has exited by now
 558             errstr = "rc=%d, OUT: '%s', ERR: '%s'" % (rc_or_sig, out, err)
 559             self.failUnlessEqual(rc_or_sig, 0, errstr)
 560             self.failUnlessEqual(out, "", errstr)
 561             # self.failUnlessEqual(err, "", errstr) # See test_client_no_noise -- for now we ignore noise.
 562             # the parent was supposed to poll and wait until it sees
 563             # twistd.pid go away before it exits, so twistd.pid should be
 564             # gone by now.
 565             self.failIf(os.path.exists(TWISTD_PID_FILE))
 566         d.addCallback(_cb4)
 567         return d