From: Brian Warner Date: Thu, 28 Jan 2010 17:39:04 +0000 (-0800) Subject: code coverage: replace figleaf with coverage.py, should work on py2.6 now. X-Git-Tag: allmydata-tahoe-1.6.1~33 X-Git-Url: https://git.rkrishnan.org/pf/content/%22news.html/frontends/provisioning?a=commitdiff_plain;h=880f824103357172fba805a4206dccf9e0da1f14;p=tahoe-lafs%2Ftahoe-lafs.git code coverage: replace figleaf with coverage.py, should work on py2.6 now. It still lacks the right HTML report (the builtin report is very pretty, but lacks the "lines uncovered" numbers that I want), and the half-finished delta-from-last-run measurements. --- diff --git a/.darcs-boringfile b/.darcs-boringfile index 09913bd0..908d6c08 100644 --- a/.darcs-boringfile +++ b/.darcs-boringfile @@ -52,10 +52,10 @@ ^build($|/) ^build-stamp$ ^python-build-stamp-2.[45]$ -^\.figleaf$ +^\.coverage$ ^coverage-html($|/) ^twisted/plugins/dropin\.cache$ -^\.figleaf\.el$ +^\.coverage\.el$ ^_test_memory($|/) # _version.py is generated at build time, and never checked in diff --git a/Makefile b/Makefile index e220a518..0b29820c 100644 --- a/Makefile +++ b/Makefile @@ -81,8 +81,8 @@ endif # TESTING -.PHONY: signal-error-deps test test-figleaf quicktest quicktest-figleaf -.PHONY: figleaf-output get-old-figleaf-coverage figleaf-delta-output +.PHONY: signal-error-deps test test-coverage quicktest quicktest-coverage +.PHONY: coverage-output get-old-coverage-coverage coverage-delta-output signal-error-deps: @@ -114,41 +114,55 @@ test: build src/allmydata/_version.py fuse-test: .built .checked-deps $(RUNPP) -d contrib/fuse -p -c runtests.py -test-figleaf: build src/allmydata/_version.py - rm -f .figleaf - $(PYTHON) setup.py trial --reporter=bwverbose-figleaf -s $(TEST) +test-coverage: build src/allmydata/_version.py + rm -f .coverage + $(PYTHON) setup.py trial --reporter=bwverbose-coverage -s $(TEST) quicktest: $(PYTHON) misc/run-with-pythonpath.py trial $(TRIALARGS) $(TEST) -quicktest-figleaf: - rm -f .figleaf - $(PYTHON) misc/run-with-pythonpath.py trial --reporter=bwverbose-figleaf $(TEST) +# code-coverage: install the "coverage" package from PyPI, do "make +# quicktest-coverage" to do a unit test run with coverage-gathering enabled, +# then use "make coverate-output-text" for a brief report, or "make +# coverage-output" for a pretty HTML report. Also see "make .coverage.el" and +# misc/coverage.el for emacs integration. -figleaf-output: - $(RUNPP) -p -c "misc/figleaf2html -d coverage-html -r src -x misc/figleaf.excludes" - cp .figleaf coverage-html/figleaf.pickle - @echo "now point your browser at coverage-html/index.html" +quicktest-coverage: + rm -f .coverage + $(PYTHON) misc/run-with-pythonpath.py trial --reporter=bwverbose-coverage $(TEST) +# on my laptop, "quicktest" takes 239s, "quicktest-coverage" takes 304s + +COVERAGE_OMIT = --omit /System,/Library,/usr/lib,src/allmydata/test,support -# use these two targets to compare this coverage against the previous run. -# The deltas only work if the old test was run in the same directory, since -# it compares absolute filenames. -get-old-figleaf-coverage: - wget --progress=dot -O old.figleaf http://allmydata.org/tahoe-figleaf/current/figleaf.pickle +# this is like 'coverage report', but includes lines-uncovered +coverage-output-text: + $(PYTHON) misc/coverage2text.py -figleaf-delta-output: - $(RUNPP) -p -c "misc/figleaf2html -d coverage-html -r src -x misc/figleaf.excludes -o old.figleaf" - cp .figleaf coverage-html/figleaf.pickle +coverage-output: + rm -rf coverage-html + coverage html -d coverage-html $(COVERAGE_OMIT) + cp .coverage coverage-html/coverage.data @echo "now point your browser at coverage-html/index.html" -# after doing test-figleaf and figleaf-output, point your browser at -# coverage-html/index.html +## use these two targets to compare this coverage against the previous run. +## The deltas only work if the old test was run in the same directory, since +## it compares absolute filenames. +#get-old-figleaf-coverage: +# wget --progress=dot -O old.figleaf http://allmydata.org/tahoe-figleaf/current/figleaf.pickle +# +#figleaf-delta-output: +# $(RUNPP) -p -c "misc/figleaf2html -d coverage-html -r src -x misc/figleaf.excludes -o old.figleaf" +# cp .figleaf coverage-html/figleaf.pickle +# @echo "now point your browser at coverage-html/index.html" -.PHONY: upload-figleaf .figleaf.el pyflakes count-lines +.PHONY: upload-coverage .coverage.el pyflakes count-lines .PHONY: check-memory check-memory-once check-speed check-grid .PHONY: repl test-darcs-boringfile test-clean clean find-trailing-spaces -# 'upload-figleaf' is meant to be run with an UPLOAD_TARGET=host:/dir setting +.coverage.el: .coverage + $(PYTHON) misc/coverage2el.py + +# 'upload-coverage' is meant to be run with an UPLOAD_TARGET=host:/dir setting ifdef UPLOAD_TARGET ifndef UPLOAD_HOST @@ -158,17 +172,15 @@ ifndef COVERAGEDIR $(error COVERAGEDIR must be set when using UPLOAD_TARGET) endif -upload-figleaf: +upload-coverage: rsync -a coverage-html/ $(UPLOAD_TARGET) - ssh $(UPLOAD_HOST) make update-tahoe-figleaf COVERAGEDIR=$(COVERAGEDIR) + ssh $(UPLOAD_HOST) make update-tahoe-coverage COVERAGEDIR=$(COVERAGEDIR) else -upload-figleaf: +upload-coverage: echo "this target is meant to be run with UPLOAD_TARGET=host:/path/" false endif -.figleaf.el: .figleaf - $(RUNPP) -p -c "misc/figleaf2el.py .figleaf src" pyflakes: $(PYTHON) -OOu `which pyflakes` src/allmydata |sort |uniq diff --git a/misc/coverage.el b/misc/coverage.el new file mode 100644 index 00000000..64e7134e --- /dev/null +++ b/misc/coverage.el @@ -0,0 +1,120 @@ + +(defvar coverage-annotation-file ".coverage.el") +(defvar coverage-annotations nil) + +(defun find-coverage-annotation-file () + (let ((dir (file-name-directory buffer-file-name)) + (olddir "/")) + (while (and (not (equal dir olddir)) + (not (file-regular-p (concat dir coverage-annotation-file)))) + (setq olddir dir + dir (file-name-directory (directory-file-name dir)))) + (and (not (equal dir olddir)) (concat dir coverage-annotation-file)) +)) + +(defun load-coverage-annotations () + (let* ((annotation-file (find-coverage-annotation-file)) + (coverage + (with-temp-buffer + (insert-file-contents annotation-file) + (let ((form (read (current-buffer)))) + (eval form))))) + (setq coverage-annotations coverage) + coverage + )) + +(defun coverage-unannotate () + (save-excursion + (dolist (ov (overlays-in (point-min) (point-max))) + (delete-overlay ov)) + (setq coverage-this-buffer-is-annotated nil) + (message "Removed annotations") +)) + +;; in emacs22, it will be possible to put the annotations in the fringe. Set +;; a display property for one of the characters in the line, using +;; (right-fringe BITMAP FACE), where BITMAP should probably be right-triangle +;; or so, and FACE should probably be '(:foreground "red"). We can also +;; create new bitmaps, with faces. To do tartans will require a lot of +;; bitmaps, and you've only got about 8 pixels to work with. + +;; unfortunately emacs21 gives us less control over the fringe. We can use +;; overlays to put letters on the left or right margins (in the text area, +;; overriding actual program text), and to modify the text being displayed +;; (by changing its background color, or adding a box around each word). + +(defun coverage-annotate (show-code) + (let ((allcoverage (load-coverage-annotations)) + (filename-key buffer-file-truename) + thiscoverage code-lines covered-lines uncovered-code-lines + ) + (while (and (not (gethash filename-key allcoverage nil)) + (string-match "/" filename-key)) + ;; eat everything up to and including the first slash, then look again + (setq filename-key (substring filename-key + (+ 1 (string-match "/" filename-key))))) + (setq thiscoverage (gethash filename-key allcoverage nil)) + (if thiscoverage + (progn + (setq coverage-this-buffer-is-annotated t) + (setq code-lines (nth 0 thiscoverage) + covered-lines (nth 1 thiscoverage) + uncovered-code-lines (nth 2 thiscoverage) + ) + + (save-excursion + (dolist (ov (overlays-in (point-min) (point-max))) + (delete-overlay ov)) + (if show-code + (dolist (line code-lines) + (goto-line line) + ;;(add-text-properties (point) (line-end-position) '(face bold) ) + (overlay-put (make-overlay (point) (line-end-position)) + ;'before-string "C" + ;'face '(background-color . "green") + 'face '(:background "dark green") + ) + )) + (dolist (line uncovered-code-lines) + (goto-line line) + (overlay-put (make-overlay (point) (line-end-position)) + ;'before-string "D" + ;'face '(:background "blue") + ;'face '(:underline "blue") + 'face '(:box "red") + ) + ) + (message "Added annotations") + ) + ) + (message "unable to find coverage for this file")) +)) + +(defun coverage-toggle-annotations (show-code) + (interactive "P") + (if coverage-this-buffer-is-annotated + (coverage-unannotate) + (coverage-annotate show-code)) +) + + +(setq coverage-this-buffer-is-annotated nil) +(make-variable-buffer-local 'coverage-this-buffer-is-annotated) + +(define-minor-mode coverage-annotation-minor-mode + "Minor mode to annotate code-coverage information" + nil + " CA" + '( + ("\C-c\C-a" . coverage-toggle-annotations) + ) + + () ; forms run on mode entry/exit +) + +(defun maybe-enable-coverage-mode () + (if (string-match "/src/allmydata/" (buffer-file-name)) + (coverage-annotation-minor-mode t) + )) + +(add-hook 'python-mode-hook 'maybe-enable-coverage-mode) diff --git a/misc/coverage2el.py b/misc/coverage2el.py new file mode 100755 index 00000000..ed94bd0f --- /dev/null +++ b/misc/coverage2el.py @@ -0,0 +1,45 @@ + +from coverage import coverage, summary + +class ElispReporter(summary.SummaryReporter): + def report(self): + self.find_code_units(None, ["/System", "/Library", "/usr/lib", + "support/lib", "src/allmydata/test"]) + + out = open(".coverage.el", "w") + out.write(""" +;; This is an elisp-readable form of the figleaf coverage data. It defines a +;; single top-level hash table in which the key is an asolute pathname, and +;; the value is a three-element list. The first element of this list is a +;; list of line numbers that represent actual code statements. The second is +;; a list of line numbers for lines which got used during the unit test. The +;; third is a list of line numbers for code lines that were not covered +;; (since 'code' and 'covered' start as sets, this last list is equal to +;; 'code - covered'). + + """) + out.write("(let ((results (make-hash-table :test 'equal)))\n") + for cu in self.code_units: + f = cu.filename + (fn, executable, missing, mf) = self.coverage.analysis(cu) + code_linenumbers = executable + uncovered_code = missing + covered_linenumbers = sorted(set(executable) - set(missing)) + out.write(" (puthash \"%s\" '((%s) (%s) (%s)) results)\n" + % (f, + " ".join([str(ln) for ln in sorted(code_linenumbers)]), + " ".join([str(ln) for ln in sorted(covered_linenumbers)]), + " ".join([str(ln) for ln in sorted(uncovered_code)]), + )) + out.write(" results)\n") + out.close() + +def main(): + c = coverage() + c.load() + ElispReporter(c).report() + +if __name__ == '__main__': + main() + + diff --git a/misc/coverage2text.py b/misc/coverage2text.py new file mode 100755 index 00000000..f91e25b5 --- /dev/null +++ b/misc/coverage2text.py @@ -0,0 +1,116 @@ + +import sys +from coverage import coverage +from coverage.results import Numbers +from coverage.summary import SummaryReporter +from twisted.python import usage + +# this is an adaptation of the code behind "coverage report", modified to +# display+sortby "lines uncovered", which (IMHO) is more important of a +# metric than lines covered or percentage covered. Concentrating on the files +# with the most uncovered lines encourages getting the tree and test suite +# into a state that provides full line-coverage on all files. + +# much of this code was adapted from coverage/summary.py in the 'coverage' +# distribution, and is used under their BSD license. + +class Options(usage.Options): + optParameters = [ + ("sortby", "s", "uncovered", "how to sort: uncovered, covered, name"), + ] + +class MyReporter(SummaryReporter): + def report(self, outfile=None, sortby="uncovered"): + self.find_code_units(None, ["/System", "/Library", "/usr/lib", + "support/lib", "src/allmydata/test"]) + + # Prepare the formatting strings + max_name = max([len(cu.name) for cu in self.code_units] + [5]) + fmt_name = "%%- %ds " % max_name + fmt_err = "%s %s: %s\n" + header1 = (fmt_name % "" ) + " Statements " + header2 = (fmt_name % "Name") + " Uncovered Covered" + fmt_coverage = fmt_name + "%9d %7d " + if self.branches: + header1 += " Branches " + header2 += " Found Excutd" + fmt_coverage += " %6d %6d" + header1 += " Percent" + header2 += " Covered" + fmt_coverage += " %7d%%" + if self.show_missing: + header1 += " " + header2 += " Missing" + fmt_coverage += " %s" + rule = "-" * len(header1) + "\n" + header1 += "\n" + header2 += "\n" + fmt_coverage += "\n" + + if not outfile: + outfile = sys.stdout + + # Write the header + outfile.write(header1) + outfile.write(header2) + outfile.write(rule) + + total = Numbers() + total_uncovered = 0 + + lines = [] + for cu in self.code_units: + try: + analysis = self.coverage._analyze(cu) + nums = analysis.numbers + uncovered = nums.n_statements - nums.n_executed + total_uncovered += uncovered + args = (cu.name, uncovered, nums.n_executed) + if self.branches: + args += (nums.n_branches, nums.n_executed_branches) + args += (nums.pc_covered,) + if self.show_missing: + args += (analysis.missing_formatted(),) + if sortby == "covered": + sortkey = nums.pc_covered + elif sortby == "uncovered": + sortkey = uncovered + else: + sortkey = cu.name + lines.append((sortkey, fmt_coverage % args)) + total += nums + except KeyboardInterrupt: # pragma: no cover + raise + except: + if not self.ignore_errors: + typ, msg = sys.exc_info()[:2] + outfile.write(fmt_err % (cu.name, typ.__name__, msg)) + lines.sort() + if sortby in ("uncovered", "covered"): + lines.reverse() + for sortkey,line in lines: + outfile.write(line) + + if total.n_files > 1: + outfile.write(rule) + args = ("TOTAL", total_uncovered, total.n_executed) + if self.branches: + args += (total.n_branches, total.n_executed_branches) + args += (total.pc_covered,) + if self.show_missing: + args += ("",) + outfile.write(fmt_coverage % args) + +def report(o): + c = coverage() + c.load() + r = MyReporter(c, show_missing=False, ignore_errors=False) + r.report(sortby=o['sortby']) + +if __name__ == '__main__': + o = Options() + o.parseOptions() + report(o) + + + diff --git a/misc/figleaf.el b/misc/figleaf.el deleted file mode 100644 index fef42e08..00000000 --- a/misc/figleaf.el +++ /dev/null @@ -1,140 +0,0 @@ - -;(require 'gnus-start) - -; (defun gnus-load (file) -; "Load FILE, but in such a way that read errors can be reported." -; (with-temp-buffer -; (insert-file-contents file) -; (while (not (eobp)) -; (condition-case type -; (let ((form (read (current-buffer)))) -; (eval form)) -; (error -; (unless (eq (car type) 'end-of-file) -; (let ((error (format "Error in %s line %d" file -; (count-lines (point-min) (point))))) -; (ding) -; (unless (gnus-yes-or-no-p (concat error "; continue? ")) -; (error "%s" error))))))))) - -(defvar figleaf-annotation-file ".figleaf.el") -(defvar figleaf-annotations nil) - -(defun find-figleaf-annotation-file () - (let ((dir (file-name-directory buffer-file-name)) - (olddir "/")) - (while (and (not (equal dir olddir)) - (not (file-regular-p (concat dir figleaf-annotation-file)))) - (setq olddir dir - dir (file-name-directory (directory-file-name dir)))) - (and (not (equal dir olddir)) (concat dir figleaf-annotation-file)) -)) - -(defun load-figleaf-annotations () - (let* ((annotation-file (find-figleaf-annotation-file)) - (coverage - (with-temp-buffer - (insert-file-contents annotation-file) - (let ((form (read (current-buffer)))) - (eval form))))) - (setq figleaf-annotations coverage) - coverage - )) - -(defun figleaf-unannotate () - (interactive) - (save-excursion - (dolist (ov (overlays-in (point-min) (point-max))) - (delete-overlay ov)) - (setq figleaf-this-buffer-is-annotated nil) - (message "Removed annotations") -)) - -;; in emacs22, it will be possible to put the annotations in the fringe. Set -;; a display property for one of the characters in the line, using -;; (right-fringe BITMAP FACE), where BITMAP should probably be right-triangle -;; or so, and FACE should probably be '(:foreground "red"). We can also -;; create new bitmaps, with faces. To do tartans will require a lot of -;; bitmaps, and you've only got about 8 pixels to work with. - -;; unfortunately emacs21 gives us less control over the fringe. We can use -;; overlays to put letters on the left or right margins (in the text area, -;; overriding actual program text), and to modify the text being displayed -;; (by changing its background color, or adding a box around each word). - -(defun figleaf-annotate (&optional show-code) - (interactive "P") - (let ((allcoverage (load-figleaf-annotations)) - (filename-key buffer-file-name) - thiscoverage code-lines covered-lines uncovered-code-lines - ) - (while (and (not (gethash filename-key allcoverage nil)) - (string-match "/" filename-key)) - ;; eat everything up to and including the first slash, then look again - (setq filename-key (substring filename-key - (+ 1 (string-match "/" filename-key))))) - (setq thiscoverage (gethash filename-key allcoverage nil)) - (if thiscoverage - (progn - (setq figleaf-this-buffer-is-annotated t) - (setq code-lines (nth 0 thiscoverage) - covered-lines (nth 1 thiscoverage) - uncovered-code-lines (nth 2 thiscoverage) - ) - - (save-excursion - (dolist (ov (overlays-in (point-min) (point-max))) - (delete-overlay ov)) - (if show-code - (dolist (line code-lines) - (goto-line line) - ;;(add-text-properties (point) (line-end-position) '(face bold) ) - (overlay-put (make-overlay (point) (line-end-position)) - ;'before-string "C" - ;'face '(background-color . "green") - 'face '(:background "dark green") - ) - )) - (dolist (line uncovered-code-lines) - (goto-line line) - (overlay-put (make-overlay (point) (line-end-position)) - ;'before-string "D" - ;'face '(:background "blue") - ;'face '(:underline "blue") - 'face '(:box "red") - ) - ) - (message "Added annotations") - ) - ) - (message "unable to find coverage for this file")) -)) - -(defun figleaf-toggle-annotations (show-code) - (interactive "P") - (if figleaf-this-buffer-is-annotated - (figleaf-unannotate) - (figleaf-annotate show-code)) -) - - -(setq figleaf-this-buffer-is-annotated nil) -(make-variable-buffer-local 'figleaf-this-buffer-is-annotated) - -(define-minor-mode figleaf-annotation-minor-mode - "Minor mode to annotate code-coverage information" - nil - " FA" - '( - ("\C-c\C-a" . figleaf-toggle-annotations) - ) - - () ; forms run on mode entry/exit -) - -(defun maybe-enable-figleaf-mode () - (if (string-match "/src/allmydata/" (buffer-file-name)) - (figleaf-annotation-minor-mode t) - )) - -(add-hook 'python-mode-hook 'maybe-enable-figleaf-mode) diff --git a/src/allmydata/test/trial_coverage.py b/src/allmydata/test/trial_coverage.py new file mode 100644 index 00000000..47569da1 --- /dev/null +++ b/src/allmydata/test/trial_coverage.py @@ -0,0 +1,110 @@ + +"""A Trial IReporter plugin that gathers coverage.py code-coverage information. + +Once this plugin is installed, trial can be invoked a new --reporter option: + + trial --reporter-bwverbose-coverage ARGS + +Once such a test run has finished, there will be a .coverage file in the +top-level directory. This file can be turned into a directory of .html files +(with index.html as the starting point) by running: + + coverage html -d OUTPUTDIR --omit=PREFIX1,PREFIX2,.. + +The 'coverage' tool thinks in terms of absolute filenames. 'coverage' doesn't +record data for files that come with Python, but it does record data for all +the various site-package directories. To show only information for Tahoe +source code files, you should provide --omit prefixes for everything else. +This probably means something like: + + --omit=/System/,/Library/,support/,src/allmydata/test/ + +Before using this, you need to install the 'coverage' package, which will +provide an executable tool named 'coverage' (as well as an importable +library). 'coverage report' will produce a basic text summary of the coverage +data. Our 'misc/coverage2text.py' tool produces a slightly more useful +summary, and 'misc/coverage2html.py' will produce a more useful HTML report. + +""" + +from twisted.trial.reporter import TreeReporter, VerboseTextReporter + +# These plugins are registered via twisted/plugins/allmydata_trial.py . See +# the notes there for an explanation of how that works. + +# Some notes about how trial Reporters are used: +# * Reporters don't really get told about the suite starting and stopping. +# * The Reporter class is imported before the test classes are. +# * The test classes are imported before the Reporter is created. To get +# control earlier than that requires modifying twisted/scripts/trial.py +# * Then Reporter.__init__ is called. +# * Then tests run, calling things like write() and addSuccess(). Each test is +# framed by a startTest/stopTest call. +# * Then the results are emitted, calling things like printErrors, +# printSummary, and wasSuccessful. +# So for code-coverage (not including import), start in __init__ and finish +# in printSummary. To include import, we have to start in our own import and +# finish in printSummary. + +import coverage +cov = coverage.coverage() +cov.start() + + +class CoverageTextReporter(VerboseTextReporter): + def __init__(self, *args, **kwargs): + VerboseTextReporter.__init__(self, *args, **kwargs) + + def stop_coverage(self): + cov.stop() + cov.save() + print "Coverage results written to .coverage" + def printSummary(self): + # for twisted-2.5.x + self.stop_coverage() + return VerboseTextReporter.printSummary(self) + def done(self): + # for twisted-8.x + self.stop_coverage() + return VerboseTextReporter.done(self) + +class sample_Reporter(object): + # this class, used as a reporter on a fully-passing test suite, doesn't + # trigger exceptions. So it is a guide to what methods are invoked on a + # Reporter. + def __init__(self, *args, **kwargs): + print "START HERE" + self.r = TreeReporter(*args, **kwargs) + self.shouldStop = self.r.shouldStop + self.separator = self.r.separator + self.testsRun = self.r.testsRun + self._starting2 = False + + def write(self, *args): + if not self._starting2: + self._starting2 = True + print "FIRST WRITE" + return self.r.write(*args) + + def startTest(self, *args, **kwargs): + return self.r.startTest(*args, **kwargs) + + def stopTest(self, *args, **kwargs): + return self.r.stopTest(*args, **kwargs) + + def addSuccess(self, *args, **kwargs): + return self.r.addSuccess(*args, **kwargs) + + def printErrors(self, *args, **kwargs): + return self.r.printErrors(*args, **kwargs) + + def writeln(self, *args, **kwargs): + return self.r.writeln(*args, **kwargs) + + def printSummary(self, *args, **kwargs): + print "PRINT SUMMARY" + return self.r.printSummary(*args, **kwargs) + + def wasSuccessful(self, *args, **kwargs): + return self.r.wasSuccessful(*args, **kwargs) + diff --git a/src/allmydata/test/trial_figleaf.py b/src/allmydata/test/trial_figleaf.py deleted file mode 100644 index 49a4e689..00000000 --- a/src/allmydata/test/trial_figleaf.py +++ /dev/null @@ -1,139 +0,0 @@ - -"""A Trial IReporter plugin that gathers figleaf code-coverage information. - -Once this plugin is installed, trial can be invoked with one of two new ---reporter options: - - trial --reporter=verbose-figleaf ARGS - trial --reporter-bwverbose-figleaf ARGS - -Once such a test run has finished, there will be a .figleaf file in the -top-level directory. This file can be turned into a directory of .html files -(with index.html as the starting point) by running: - - figleaf2html -d OUTPUTDIR [-x EXCLUDEFILE] - -Figleaf thinks of everyting in terms of absolute filenames rather than -modules. The EXCLUDEFILE may be necessary to keep it from providing reports -on non-Code-Under-Test files that live in unusual locations. In particular, -if you use extra PYTHONPATH arguments to point at some alternate version of -an upstream library (like Twisted), or if something like debian's -python-support puts symlinks to .py files in sys.path but not the .py files -themselves, figleaf will present coverage information on both of these. The -EXCLUDEFILE option might help to inhibit these. - -Other figleaf problems: - - the annotated code files are written to BASENAME(file).html, which results - in collisions between similarly-named source files. - - The line-wise coverage information isn't quite right. Blank lines are - counted as unreached code, lambdas aren't quite right, and some multiline - comments (docstrings?) aren't quite right. - -""" - -from twisted.trial.reporter import TreeReporter, VerboseTextReporter - -# These plugins are registered via twisted/plugins/allmydata_trial.py . See -# the notes there for an explanation of how that works. - - - -# Reporters don't really get told about the suite starting and stopping. - -# The Reporter class is imported before the test classes are. - -# The test classes are imported before the Reporter is created. To get -# control earlier than that requires modifying twisted/scripts/trial.py . - -# Then Reporter.__init__ is called. - -# Then tests run, calling things like write() and addSuccess(). Each test is -# framed by a startTest/stopTest call. - -# Then the results are emitted, calling things like printErrors, -# printSummary, and wasSuccessful. - -# So for code-coverage (not including import), start in __init__ and finish -# in printSummary. To include import, we have to start in our own import and -# finish in printSummary. - -import figleaf -figleaf.start() - - -class FigleafReporter(TreeReporter): - def __init__(self, *args, **kwargs): - TreeReporter.__init__(self, *args, **kwargs) - - def stop_figleaf(self): - figleaf.stop() - figleaf.write_coverage(".figleaf") - print "Figleaf results written to .figleaf" - def printSummary(self): - # for twisted-2.5.x - self.stop_figleaf() - return TreeReporter.printSummary(self) - def done(self): - # for twisted-8.x - self.stop_figleaf() - return TreeReporter.done(self) - -class FigleafTextReporter(VerboseTextReporter): - def __init__(self, *args, **kwargs): - VerboseTextReporter.__init__(self, *args, **kwargs) - - def stop_figleaf(self): - figleaf.stop() - figleaf.write_coverage(".figleaf") - print "Figleaf results written to .figleaf" - def printSummary(self): - # for twisted-2.5.x - self.stop_figleaf() - return VerboseTextReporter.printSummary(self) - def done(self): - # for twisted-8.x - self.stop_figleaf() - return VerboseTextReporter.done(self) - -class not_FigleafReporter(object): - # this class, used as a reporter on a fully-passing test suite, doesn't - # trigger exceptions. So it is a guide to what methods are invoked on a - # Reporter. - def __init__(self, *args, **kwargs): - print "FIGLEAF HERE" - self.r = TreeReporter(*args, **kwargs) - self.shouldStop = self.r.shouldStop - self.separator = self.r.separator - self.testsRun = self.r.testsRun - self._starting2 = False - - def write(self, *args): - if not self._starting2: - self._starting2 = True - print "FIRST WRITE" - return self.r.write(*args) - - def startTest(self, *args, **kwargs): - return self.r.startTest(*args, **kwargs) - - def stopTest(self, *args, **kwargs): - return self.r.stopTest(*args, **kwargs) - - def addSuccess(self, *args, **kwargs): - return self.r.addSuccess(*args, **kwargs) - - def printErrors(self, *args, **kwargs): - return self.r.printErrors(*args, **kwargs) - - def writeln(self, *args, **kwargs): - return self.r.writeln(*args, **kwargs) - - def printSummary(self, *args, **kwargs): - print "PRINT SUMMARY" - return self.r.printSummary(*args, **kwargs) - - def wasSuccessful(self, *args, **kwargs): - return self.r.wasSuccessful(*args, **kwargs) - diff --git a/twisted/plugins/allmydata_trial.py b/twisted/plugins/allmydata_trial.py index 275bbb24..11cbeade 100644 --- a/twisted/plugins/allmydata_trial.py +++ b/twisted/plugins/allmydata_trial.py @@ -4,18 +4,18 @@ from zope.interface import implements from twisted.trial.itrial import IReporter from twisted.plugin import IPlugin -# register a plugin that can create our FigleafReporter. The reporter itself -# lives in a separate place - -# note that this .py file is *not* in a package: there is no __init__.py in -# our parent directory. This is important, because otherwise ours would fight -# with Twisted's. When trial looks for plugins, it merely executes all the -# *.py files it finds in any twisted/plugins/ subdirectories of anything on -# sys.path . The namespace that results from executing these .py files is -# examined for instances which provide both IPlugin and the target interface -# (in this case, trial is looking for IReporter instances). Each such -# instance tells the application how to create a plugin by naming the module -# and class that should be instantiated. +# register a plugin that can create our CoverageReporter. The reporter itself +# lives separately, in src/allmydata/test/trial_figleaf.py + +# note that this allmydata_trial.py file is *not* in a package: there is no +# __init__.py in our parent directory. This is important, because otherwise +# ours would fight with Twisted's. When trial looks for plugins, it merely +# executes all the *.py files it finds in any twisted/plugins/ subdirectories +# of anything on sys.path . The namespace that results from executing these +# .py files is examined for instances which provide both IPlugin and the +# target interface (in this case, trial is looking for IReporter instances). +# Each such instance tells the application how to create a plugin by naming +# the module and class that should be instantiated. # When installing our package via setup.py, arrange for this file to be # installed to the system-wide twisted/plugins/ directory. @@ -32,17 +32,10 @@ class _Reporter(object): self.klass = klass -fig = _Reporter("Figleaf Code-Coverage Reporter", - "allmydata.test.trial_figleaf", - description="verbose color output (with figleaf coverage)", - longOpt="verbose-figleaf", - shortOpt="f", - klass="FigleafReporter") - -bwfig = _Reporter("Figleaf Code-Coverage Reporter (colorless)", - "allmydata.test.trial_figleaf", - description="Colorless verbose output (with figleaf coverage)", - longOpt="bwverbose-figleaf", +bwcov = _Reporter("Code-Coverage Reporter (colorless)", + "allmydata.test.trial_coverage", + description="Colorless verbose output (with 'coverage' coverage)", + longOpt="bwverbose-coverage", shortOpt=None, - klass="FigleafTextReporter") + klass="CoverageTextReporter")