[GRASS-SVN] r61405 - grass/trunk/lib/python/gunittest
svn_grass at osgeo.org
svn_grass at osgeo.org
Fri Jul 25 13:10:53 PDT 2014
Author: wenzeslaus
Date: 2014-07-25 13:10:53 -0700 (Fri, 25 Jul 2014)
New Revision: 61405
Modified:
grass/trunk/lib/python/gunittest/invoker.py
grass/trunk/lib/python/gunittest/reporters.py
Log:
gunittest: add reports for testsuites and their summary
Modified: grass/trunk/lib/python/gunittest/invoker.py
===================================================================
--- grass/trunk/lib/python/gunittest/invoker.py 2014-07-25 20:01:38 UTC (rev 61404)
+++ grass/trunk/lib/python/gunittest/invoker.py 2014-07-25 20:10:53 UTC (rev 61405)
@@ -24,8 +24,8 @@
from .loader import GrassTestLoader, discover_modules
from .reporters import (GrassTestFilesMultiReporter,
- GrassTestFilesTextReporter,
- GrassTestFilesHtmlReporter, get_svn_path_authors)
+ GrassTestFilesTextReporter, GrassTestFilesHtmlReporter,
+ TestsuiteDirReporter, get_svn_path_authors)
from .utils import silent_rmtree, ensure_dir
import grass.script.setup as gsetup
@@ -101,10 +101,12 @@
self.clean_mapsets = clean_mapsets
self.clean_outputs = clean_outputs
self.clean_before = clean_before
- self.testsuite_dir = testsuite_dir
+ self.testsuite_dir = testsuite_dir # TODO: solve distribution of this constant
# reporter is created for each call of run_in_location()
self.reporter = None
+ self.testsuite_dirs = None
+
def _create_mapset(self, gisdbase, location, module):
"""Create mapset according to informations in module.
@@ -132,6 +134,7 @@
def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
+ self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, 'data')
if os.path.exists(data_dir):
@@ -185,7 +188,7 @@
GrassTestFilesTextReporter(stream=sys.stderr),
GrassTestFilesHtmlReporter(),
])
-
+ self.testsuite_dirs = collections.defaultdict(list) # reset list of dirs each time
# TODO: move constants out of loader class or even module
modules = discover_modules(start_dir=self.start_dir,
grass_location=location_shortcut,
@@ -201,4 +204,8 @@
self._run_test_module(module=module, results_dir=results_dir,
gisdbase=gisdbase, location=location)
+ testsuite_dir_reporter = TestsuiteDirReporter(
+ main_page_name='testsuites.html')
+ testsuite_dir_reporter.report_for_dirs(root=results_dir,
+ directories=self.testsuite_dirs)
self.reporter.finish()
Modified: grass/trunk/lib/python/gunittest/reporters.py
===================================================================
--- grass/trunk/lib/python/gunittest/reporters.py 2014-07-25 20:01:38 UTC (rev 61404)
+++ grass/trunk/lib/python/gunittest/reporters.py 2014-07-25 20:10:53 UTC (rev 61405)
@@ -19,6 +19,7 @@
import subprocess
from .utils import ensure_dir
+from .checkers import text_to_keyvalue
def get_source_url(path, revision, line=None):
@@ -170,6 +171,36 @@
return None
+def get_html_test_authors_table(directory, tests_authors):
+ # SVN gives us authors of code together with authors of tests
+ # so test code authors list also contains authors of tests only
+ # TODO: don't do this for the top level directories?
+ tests_authors = set(tests_authors)
+ from_date = years_ago(datetime.date.today(), years=1)
+ tested_dir_authors = get_svn_path_authors(directory, from_date)
+ not_testing_authors = tested_dir_authors - tests_authors
+ if not not_testing_authors:
+ not_testing_authors = ['all recent authors contributed tests']
+
+ test_authors = (
+ '<h3>Code and test authors</h3>'
+ '<p style="font-size: 60%"><em>'
+ 'Note that determination of authors is approximate and only'
+ ' recent code authors are considered.'
+ '</em></p>'
+ '<table><tbody>'
+ '<tr><td>Test authors:</td><td>{file_authors}</td></tr>'
+ '<tr><td>Authors of tested code:</td><td>{code_authors}</td></tr>'
+ '<tr><td>Authors owing tests:</td><td>{not_testing}</td></tr>'
+ '</tbody></table>'
+ .format(
+ file_authors=', '.join(sorted(tests_authors)),
+ code_authors=', '.join(sorted(tested_dir_authors)),
+ not_testing=', '.join(sorted(not_testing_authors))
+ ))
+ return test_authors
+
+
class GrassTestFilesMultiReporter(object):
def __init__(self, reporters, forgiving=False):
@@ -284,9 +315,61 @@
percent=percent, color=color)
+def wrap_stdstream_to_html(infile, outfile, module, stream):
+ before = '<html><body><h1>%s</h1><pre>' % (module.name + ' ' + stream)
+ after = '</pre></body></html>'
+ html = open(outfile, 'w')
+ html.write(before)
+ with open(infile) as text:
+ for line in text:
+ html.write(color_error_line(html_escape(line)))
+ html.write(after)
+ html.close()
+
+
+def returncode_to_html_text(returncode):
+ if returncode:
+ return '<span style="color: red">FAILED</span>'
+ else:
+ # alternatives: SUCCEEDED, passed, OK
+ return '<span style="color: green">succeeded</span>'
+
+
+def returncode_to_html_sentence(returncode):
+ if returncode:
+ return ('<span style="color: red">❌</span>'
+ ' Test failed (return code %d)' % (returncode))
+ else:
+ return ('<span style="color: green">✓</span>'
+ ' Test succeeded (return code %d)' % (returncode))
+
+
+def success_to_html_text(total, successes):
+ if successes < total:
+ return '<span style="color: red">FAILED</span>'
+ elif successes == total:
+ # alternatives: SUCCEEDED, passed, OK
+ return '<span style="color: green">succeeded</span>'
+ else:
+ return ('<span style="color: red; font-size: 60%">'
+ '? more successes than total ?</span>')
+
+
+UNKNOWN_NUMBER_HTML = '<span style="font-size: 60%">unknown</span>'
+
+
+def success_to_html_percent(total, successes):
+ if total:
+ pass_per = 100 * (float(successes) / total)
+ pass_per = percent_to_html(pass_per)
+ else:
+ pass_per = UNKNOWN_NUMBER_HTML
+ return pass_per
+
+
class GrassTestFilesHtmlReporter(GrassTestFilesCountingReporter):
- unknown_number = '<span style="font-size: 60%">unknown</span>'
+ unknown_number = UNKNOWN_NUMBER_HTML
def __init__(self):
super(GrassTestFilesHtmlReporter, self).__init__()
@@ -300,12 +383,11 @@
# TODO: this can be moved to the counter class
self.failures = 0
self.errors = 0
- self.skiped = 0
+ self.skipped = 0
self.successes = 0
self.expected_failures = 0
self.unexpected_success = 0
self.total = 0
- # TODO: skiped and unexpected success
svn_info = get_svn_info()
if not svn_info:
@@ -380,32 +462,6 @@
super(GrassTestFilesHtmlReporter, self).start_file_test(module)
self.main_index.flush() # to get previous lines to the report
- def wrap_stdstream_to_html(self, infile, outfile, module, stream):
- before = '<html><body><h1>%s</h1><pre>' % (module.name + ' ' + stream)
- after = '</pre></body></html>'
- html = open(outfile, 'w')
- html.write(before)
- with open(infile) as text:
- for line in text:
- html.write(color_error_line(html_escape(line)))
- html.write(after)
- html.close()
-
- def returncode_to_html_text(self, returncode):
- if returncode:
- return '<span style="color: red">FAILED</span>'
- else:
- # alternatives: SUCCEEDED, passed, OK
- return '<span style="color: green">succeeded</span>'
-
- def returncode_to_html_sentence(self, returncode):
- if returncode:
- return ('<span style="color: red">❌</span>'
- ' Test failed (return code %d)' % (returncode))
- else:
- return ('<span style="color: green">✓</span>'
- ' Test succeeded (return code %d)' % (returncode))
-
def end_file_test(self, module, cwd, returncode, stdout, stderr,
test_summary):
super(GrassTestFilesHtmlReporter, self).end_file_test(
@@ -427,10 +483,11 @@
self.failures += failures
self.errors += errors
- self.skiped += skipped
+ self.skipped += skipped
self.expected_failures += expected_failures
self.unexpected_success += unexpected_successes
+ # TODO: should we test for zero?
if total is not None:
# success are only the clear ones
# percentage is influenced by all but putting only failures to table
@@ -448,14 +505,14 @@
'<td>{total}</td><td>{st}</td><td>{ft}</td><td>{pt}</td>'
'<tr>'.format(
d=module.tested_dir, m=module.name,
- sf=self.returncode_to_html_text(returncode),
+ sf=returncode_to_html_text(returncode),
st=successes, ft=bad_ones, total=total, pt=pass_per))
- self.wrap_stdstream_to_html(infile=stdout,
- outfile=os.path.join(cwd, 'stdout.html'),
- module=module, stream='stdout')
- self.wrap_stdstream_to_html(infile=stderr,
- outfile=os.path.join(cwd, 'stderr.html'),
- module=module, stream='stderr')
+ wrap_stdstream_to_html(infile=stdout,
+ outfile=os.path.join(cwd, 'stdout.html'),
+ module=module, stream='stdout')
+ wrap_stdstream_to_html(infile=stderr,
+ outfile=os.path.join(cwd, 'stderr.html'),
+ module=module, stream='stderr')
file_index_path = os.path.join(cwd, 'index.html')
file_index = open(file_index_path, 'w')
@@ -473,7 +530,7 @@
'</body></html>'
.format(
dur=self.file_time, m=module,
- status=self.returncode_to_html_sentence(returncode),
+ status=returncode_to_html_sentence(returncode),
))
file_index.close()
@@ -536,3 +593,213 @@
self._stream.write('\n')
# TODO: here we lost the possibility to include also file name
# of the appropriate report
+
+
+# TODO: there is a quite a lot duplication between this class and html reporter
+# TODO: document: do not use it for two reports, it accumulates the results
+# TODO: add also keyvalue summary generation?
+# wouldn't this conflict with collecting data from report afterwards?
+class TestsuiteDirReporter(object):
+ def __init__(self, main_page_name, testsuite_page_name='index.html'):
+ self.main_page_name = main_page_name
+ self.testsuite_page_name = testsuite_page_name
+
+ # TODO: this might be even a object which could add and validate
+ self.failures = 0
+ self.errors = 0
+ self.skipped = 0
+ self.successes = 0
+ self.expected_failures = 0
+ self.unexpected_successes = 0
+ self.total = 0
+
+ self.testsuites = 0
+ self.testsuites_successes = 0
+ self.files = 0
+ self.files_successes = 0
+
+ def report_for_dir(self, root, directory, test_files):
+ # TODO: create object from this, so that it can be passed from
+ # one function to another
+ # TODO: put the inside of for loop to another fucntion
+ dir_failures = 0
+ dir_errors = 0
+ dir_skipped = 0
+ dir_successes = 0
+ dir_expected_failures = 0
+ dir_unexpected_success = 0
+ dir_total = 0
+ test_files_authors = []
+
+ file_total = 0
+ file_successes = 0
+
+ page_name = os.path.join(root, directory, self.testsuite_page_name)
+ page = open(page_name, 'w')
+ head = (
+ '<html><body>'
+ '<h1>{name} testsuite results</h1>'
+ .format(name=directory))
+ tests_table_head = (
+ '<h3>Test files results</h3>'
+ '<table>'
+ '<thead><tr>'
+ '<th>Test file</th><th>Status</th>'
+ '<th>Tests</th><th>Successful</td>'
+ '<th>Failed</th><th>Percent successful</th>'
+ '</tr></thead><tbody>'
+ )
+ page.write(head)
+ page.write(tests_table_head)
+ for test_file_name in test_files:
+ # TODO: put keyvalue fine name to constant
+ summary_filename = os.path.join(root, directory, test_file_name,
+ 'test_keyvalue_result.txt')
+ #if os.path.exists(summary_filename):
+ with open(summary_filename, 'r') as keyval_file:
+ summary = text_to_keyvalue(keyval_file.read(), sep='=')
+ #else:
+ # TODO: write else here
+ # summary = None
+
+ if 'total' not in summary:
+ bad_ones = successes = UNKNOWN_NUMBER_HTML
+ total = None
+ else:
+ bad_ones = summary['failures'] + summary['errors']
+ successes = summary['successes']
+ total = summary['total']
+
+ self.failures += summary['failures']
+ self.errors += summary['errors']
+ self.skipped += summary['skipped']
+ self.successes += summary['successes']
+ self.expected_failures += summary['expected_failures']
+ self.unexpected_successes += summary['unexpected_successes']
+ self.total += summary['total']
+
+ dir_failures += summary['failures']
+ dir_errors += summary['failures']
+ dir_skipped += summary['skipped']
+ dir_successes += summary['successes']
+ dir_expected_failures += summary['expected_failures']
+ dir_unexpected_success += summary['unexpected_successes']
+ dir_total += summary['total']
+
+ # TODO: keyvalue method should have types for keys function
+ # perhaps just the current post processing function is enough
+ test_file_authors = summary['test_file_authors']
+ if type(test_file_authors) is not list:
+ test_file_authors = [test_file_authors]
+ test_files_authors += test_file_authors
+
+ file_total += 1
+ file_successes += 0 if summary['returncode'] else 1
+
+ pass_per = success_to_html_percent(total=total,
+ successes=successes)
+ row = (
+ '<tr>'
+ '<td><a href="{f}/index.html">{f}</a></td>'
+ '<td>{status}</td>'
+ '<td>{ntests}</td><td>{stests}</td>'
+ '<td>{ftests}</td><td>{ptests}</td>'
+ '<tr>'
+ .format(
+ f=test_file_name,
+ status=returncode_to_html_text(summary['returncode']),
+ stests=successes, ftests=bad_ones, ntests=total,
+ ptests=pass_per))
+ page.write(row)
+
+ self.testsuites += 1
+ self.testsuites_successes += 1 if file_successes == file_total else 0
+ self.files += file_total
+ self.files_successes += file_successes
+
+ dir_pass_per = success_to_html_percent(total=dir_total,
+ successes=dir_successes)
+ file_pass_per = success_to_html_percent(total=file_total,
+ successes=file_successes)
+ tests_table_foot = (
+ '</tbody><tfoot><tr>'
+ '<td>Summary</td>'
+ '<td>{status}</td>'
+ '<td>{ntests}</td><td>{stests}</td>'
+ '<td>{ftests}</td><td>{ptests}</td>'
+ '</tr></tfoot></table>'
+ .format(
+ status=file_pass_per,
+ stests=dir_successes, ftests=dir_failures + dir_errors,
+ ntests=dir_total, ptests=dir_pass_per))
+ page.write(tests_table_foot)
+ test_authors = get_html_test_authors_table(
+ directory=directory, tests_authors=test_files_authors)
+ page.write(test_authors)
+ page.write('</body></html>')
+
+ status = success_to_html_text(total=file_total, successes=file_successes)
+ row = (
+ '<tr>'
+ '<td><a href="{d}/{page}">{d}</a></td><td>{status}</td>'
+ '<td>{nfiles}</td><td>{sfiles}</td><td>{pfiles}</td>'
+ '<td>{ntests}</td><td>{stests}</td>'
+ '<td>{ftests}</td><td>{ptests}</td>'
+ '<tr>'
+ .format(
+ d=directory, page=self.testsuite_page_name, status=status,
+ nfiles=file_total, sfiles=file_successes, pfiles=file_pass_per,
+ stests=dir_successes, ftests=dir_failures + dir_errors,
+ ntests=dir_total, ptests=dir_pass_per))
+ return row
+
+ def report_for_dirs(self, root, directories):
+ # TODO: this will need chanages accoring to potential chnages in absolute/relative paths
+
+ page_name = os.path.join(root, self.main_page_name)
+ page = open(page_name, 'w')
+ head = (
+ '<html><body>'
+ '<h1>Testsuites results</h1>'
+ )
+ tests_table_head = (
+ '<table>'
+ '<thead><tr>'
+ '<th>Testsuite</th>'
+ '<th>Status</th>'
+ '<th>Test files</th><th>Successful</td>'
+ '<th>Percent successful</th>'
+ '<th>Tests</th><th>Successful</td>'
+ '<th>Failed</th><th>Percent successful</th>'
+ '</tr></thead><tbody>'
+ )
+ page.write(head)
+ page.write(tests_table_head)
+
+ for directory, test_files in directories.iteritems():
+ row = self.report_for_dir(root=root, directory=directory,
+ test_files=test_files)
+ page.write(row)
+
+ pass_per = success_to_html_percent(total=self.total,
+ successes=self.successes)
+ file_pass_per = success_to_html_percent(total=self.files,
+ successes=self.files_successes)
+ testsuites_pass_per = success_to_html_percent(
+ total=self.testsuites, successes=self.testsuites_successes)
+ tests_table_foot = (
+ '<tfoot>'
+ '<tr>'
+ '<td>Summary</td><td>{status}</td>'
+ '<td>{nfiles}</td><td>{sfiles}</td><td>{pfiles}</td>'
+ '<td>{ntests}</td><td>{stests}</td>'
+ '<td>{ftests}</td><td>{ptests}</td>'
+ '</tr>'
+ '</tfoot>'
+ .format(
+ status=testsuites_pass_per, nfiles=self.files,
+ sfiles=self.files_successes, pfiles=file_pass_per,
+ stests=self.successes, ftests=self.failures + self.errors,
+ ntests=self.total, ptests=pass_per))
+ page.write(tests_table_foot)
+ page.write('</body></html>')
More information about the grass-commit
mailing list