From: Brian Warner Date: Wed, 6 Dec 2006 22:26:12 +0000 (-0700) Subject: tests: add support for figleaf code-coverage gathering X-Git-Tag: tahoe_v0.1.0-0-UNSTABLE~444 X-Git-Url: https://git.rkrishnan.org/%5B/%5D%20/uri/flags/%3C?a=commitdiff_plain;h=e5ba7f15d96a7e14af426942788c56adc2962b6e;p=tahoe-lafs%2Ftahoe-lafs.git tests: add support for figleaf code-coverage gathering --- diff --git a/.darcs-boringfile b/.darcs-boringfile index 74176fa8..6f5979c1 100644 --- a/.darcs-boringfile +++ b/.darcs-boringfile @@ -46,3 +46,6 @@ ^(dapper|sid)/debian/files$ ^build($|/) ^build-stamp$ +^\.figleaf$ +^coverage-html($|/) +^twisted/plugins/dropin\.cache$ diff --git a/Makefile b/Makefile index 52cf30aa..4bf376cd 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,11 @@ run-client3: test: trial allmydata +test-figleaf: + trial --reporter=bwverbose-figleaf allmydata + figleaf2html -d coverage-html -x allmydata/test/figleaf.excludes +# after doing test-figleaf, point your browser at coverage-html/index.html + create_dirs: mkdir -p queen-basedir mkdir -p client-basedir diff --git a/allmydata/test/figleaf.excludes b/allmydata/test/figleaf.excludes new file mode 100644 index 00000000..0b779a91 --- /dev/null +++ b/allmydata/test/figleaf.excludes @@ -0,0 +1,2 @@ +^/home/warner/stuff/python/twisted/Twisted/ +^/var/lib diff --git a/allmydata/test/trial_figleaf.py b/allmydata/test/trial_figleaf.py new file mode 100644 index 00000000..05780476 --- /dev/null +++ b/allmydata/test/trial_figleaf.py @@ -0,0 +1,127 @@ + +"""A Trial IReporter plugin that gathers figleaf code-coverage information. + +Once this plugin is installed, trial can be invoked with one of two new +--reporter options: + + trial --reporter=verbose-figleaf ARGS + trial --reporter-bwverbose-figleaf ARGS + +Once such a test run has finished, there will be a .figleaf file in the +top-level directory. This file can be turned into a directory of .html files +(with index.html as the starting point) by running: + + figleaf2html -d OUTPUTDIR [-x EXCLUDEFILE] + +Figleaf thinks of everyting in terms of absolute filenames rather than +modules. The EXCLUDEFILE may be necessary to keep it from providing reports +on non-Code-Under-Test files that live in unusual locations. In particular, +if you use extra PYTHONPATH arguments to point at some alternate version of +an upstream library (like Twisted), or if something like debian's +python-support puts symlinks to .py files in sys.path but not the .py files +themselves, figleaf will present coverage information on both of these. The +EXCLUDEFILE option might help to inhibit these. + +Other figleaf problems: + + the annotated code files are written to BASENAME(file).html, which results + in collisions between similarly-named source files. + + The line-wise coverage information isn't quite right. Blank lines are + counted as unreached code, lambdas aren't quite right, and some multiline + comments (docstrings?) aren't quite right. + +""" + +# TODO: pull some of figleaf into our tree so we can customize it more +# easily. + +from twisted.trial.reporter import TreeReporter, VerboseTextReporter + +# These plugins are registered via twisted/plugins/allmydata_trial.py . See +# the notes there for an explanation of how that works. + + + +# Reporters don't really get told about the suite starting and stopping. + +# The Reporter class is imported before the test classes are. + +# The test classes are imported before the Reporter is created. To get +# control earlier than that requires modifying twisted/scripts/trial.py . + +# Then Reporter.__init__ is called. + +# Then tests run, calling things like write() and addSuccess(). Each test is +# framed by a startTest/stopTest call. + +# Then the results are emitted, calling things like printErrors, +# printSummary, and wasSuccessful. + +# So for code-coverage (not including import), start in __init__ and finish +# in printSummary. To include import, we have to start in our own import and +# finish in printSummary. + +import figleaf +figleaf.start() + +class FigleafReporter(TreeReporter): + def __init__(self, *args, **kwargs): + TreeReporter.__init__(self, *args, **kwargs) + + def printSummary(self): + figleaf.stop() + figleaf.write_coverage(".figleaf") + print "Figleaf results written to .figleaf" + return TreeReporter.printSummary(self) + +class FigleafTextReporter(VerboseTextReporter): + def __init__(self, *args, **kwargs): + VerboseTextReporter.__init__(self, *args, **kwargs) + + def printSummary(self): + figleaf.stop() + figleaf.write_coverage(".figleaf") + print "Figleaf results written to .figleaf" + return VerboseTextReporter.printSummary(self) + +class not_FigleafReporter(object): + # this class, used as a reporter on a fully-passing test suite, doesn't + # trigger exceptions. So it is a guide to what methods are invoked on a + # Reporter. + def __init__(self, *args, **kwargs): + print "FIGLEAF HERE" + self.r = TreeReporter(*args, **kwargs) + self.shouldStop = self.r.shouldStop + self.separator = self.r.separator + self.testsRun = self.r.testsRun + self._starting2 = False + + def write(self, *args): + if not self._starting2: + self._starting2 = True + print "FIRST WRITE" + return self.r.write(*args) + + def startTest(self, *args, **kwargs): + return self.r.startTest(*args, **kwargs) + + def stopTest(self, *args, **kwargs): + return self.r.stopTest(*args, **kwargs) + + def addSuccess(self, *args, **kwargs): + return self.r.addSuccess(*args, **kwargs) + + def printErrors(self, *args, **kwargs): + return self.r.printErrors(*args, **kwargs) + + def writeln(self, *args, **kwargs): + return self.r.writeln(*args, **kwargs) + + def printSummary(self, *args, **kwargs): + print "PRINT SUMMARY" + return self.r.printSummary(*args, **kwargs) + + def wasSuccessful(self, *args, **kwargs): + return self.r.wasSuccessful(*args, **kwargs) + diff --git a/twisted/plugins/allmydata_trial.py b/twisted/plugins/allmydata_trial.py new file mode 100644 index 00000000..08fd3759 --- /dev/null +++ b/twisted/plugins/allmydata_trial.py @@ -0,0 +1,48 @@ +#! /usr/bin/python + +from zope.interface import implements +from twisted.trial.itrial import IReporter +from twisted.plugin import IPlugin + +# register a plugin that can create our FigleafReporter. The reporter itself +# lives in a separate place + +# note that this .py file is *not* in a package: there is no __init__.py in +# our parent directory. This is important, because otherwise ours would fight +# with Twisted's. When trial looks for plugins, it merely executes all the +# *.py files it finds in and twisted/plugins/ subdirectories of anything on +# sys.path . The namespace that results from executing these .py files is +# examined for instances which provide both IPlugin and the target interface +# (in this case, trial is looking for IReporter instances). Each such +# instance tells the application how to create a plugin by naming the module +# and class that should be instantiated. + +# When installing our package via setup.py, arrange for this file to be +# installed to the system-wide twisted/plugins/ directory. + +class _Reporter(object): + implements(IPlugin, IReporter) + + def __init__(self, name, module, description, longOpt, shortOpt, klass): + self.name = name + self.module = module + self.description = description + self.longOpt = longOpt + self.shortOpt = shortOpt + self.klass = klass + + +fig = _Reporter("Figleaf Code-Coverage Reporter", + "allmydata.test.trial_figleaf", + description="verbose color output (with figleaf coverage)", + longOpt="verbose-figleaf", + shortOpt="f", + klass="FigleafReporter") + +bwfig = _Reporter("Figleaf Code-Coverage Reporter (colorless)", + "allmydata.test.trial_figleaf", + description="Colorless verbose output (with figleaf coverage)", + longOpt="bwverbose-figleaf", + shortOpt=None, + klass="FigleafTextReporter") +