forked from Imagelibrary/littlefs
Added coverage.py, and optional coverage info to test.py
Now coverage information can be collected if you provide the --coverage to test.py. Internally this uses GCC's gcov instrumentation along with a new script, coverage.py, to parse *.gcov files. The main use for this is finding coverage info during CI runs. There's a risk that the instrumentation may make it more difficult to debug, so I decided to not make coverage collection enabled by default.
This commit is contained in:
Christopher Haster
111
scripts/test.py
111
scripts/test.py
@@ -21,19 +21,37 @@ import errno
|
||||
import signal
|
||||
|
||||
TESTDIR = 'tests'
|
||||
RESULTDIR = 'results' # only used for coverage
|
||||
RULES = """
|
||||
define FLATTEN
|
||||
tests/%$(subst /,.,$(target)): $(target)
|
||||
%(path)s%%$(subst /,.,$(target)): $(target)
|
||||
./scripts/explode_asserts.py $$< -o $$@
|
||||
endef
|
||||
$(foreach target,$(SRC),$(eval $(FLATTEN)))
|
||||
|
||||
-include tests/*.d
|
||||
|
||||
-include %(path)s*.d
|
||||
.SECONDARY:
|
||||
%.test: %.test.o $(foreach f,$(subst /,.,$(OBJ)),%.$f)
|
||||
|
||||
%(path)s.test: %(path)s.test.o $(foreach t,$(subst /,.,$(OBJ)),%(path)s.$t)
|
||||
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
|
||||
"""
|
||||
COVERAGE_TEST_RULES = """
|
||||
%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage
|
||||
|
||||
# delete lingering coverage info during build
|
||||
%(path)s.test: | %(path)s.test.clean
|
||||
.PHONY: %(path)s.test.clean
|
||||
%(path)s.test.clean:
|
||||
rm -f %(path)s*.gcda
|
||||
|
||||
override TEST_GCDAS += %(path)s*.gcda
|
||||
"""
|
||||
COVERAGE_RESULT_RULES = """
|
||||
# dependencies defined in test makefiles
|
||||
.PHONY: %(results)s/coverage.gcov
|
||||
%(results)s/coverage.gcov: $(patsubst %%,%%.gcov,$(wildcard $(TEST_GCDAS)))
|
||||
./scripts/coverage.py -s $^ --filter="$(SRC)" --merge=$@
|
||||
"""
|
||||
GLOBALS = """
|
||||
//////////////// AUTOGENERATED TEST ////////////////
|
||||
#include "lfs.h"
|
||||
@@ -516,13 +534,20 @@ class TestSuite:
|
||||
|
||||
# write makefiles
|
||||
with open(self.path + '.mk', 'w') as mk:
|
||||
mk.write(RULES.replace(4*' ', '\t'))
|
||||
mk.write(RULES.replace(4*' ', '\t') % dict(path=self.path))
|
||||
mk.write('\n')
|
||||
|
||||
# add coverage hooks?
|
||||
if args.get('coverage', False):
|
||||
mk.write(COVERAGE_TEST_RULES.replace(4*' ', '\t') % dict(
|
||||
results=args['results'],
|
||||
path=self.path))
|
||||
mk.write('\n')
|
||||
|
||||
# add truely global defines globally
|
||||
for k, v in sorted(self.defines.items()):
|
||||
mk.write('%s: override CFLAGS += -D%s=%r\n' % (
|
||||
self.path+'.test', k, v))
|
||||
mk.write('%s.test: override CFLAGS += -D%s=%r\n'
|
||||
% (self.path, k, v))
|
||||
|
||||
for path in tfs:
|
||||
if path is None:
|
||||
@@ -596,7 +621,7 @@ def main(**args):
|
||||
|
||||
# figure out the suite's toml file
|
||||
if os.path.isdir(testpath):
|
||||
testpath = testpath + '/test_*.toml'
|
||||
testpath = testpath + '/*.toml'
|
||||
elif os.path.isfile(testpath):
|
||||
testpath = testpath
|
||||
elif testpath.endswith('.toml'):
|
||||
@@ -674,12 +699,12 @@ def main(**args):
|
||||
sum(len(suite.cases) for suite in suites),
|
||||
sum(len(suite.perms) for suite in suites)))
|
||||
|
||||
filtered = 0
|
||||
total = 0
|
||||
for suite in suites:
|
||||
for perm in suite.perms:
|
||||
filtered += perm.shouldtest(**args)
|
||||
if filtered != sum(len(suite.perms) for suite in suites):
|
||||
print('filtered down to %d permutations' % filtered)
|
||||
total += perm.shouldtest(**args)
|
||||
if total != sum(len(suite.perms) for suite in suites):
|
||||
print('total down to %d permutations' % total)
|
||||
|
||||
# only requested to build?
|
||||
if args.get('build', False):
|
||||
@@ -723,6 +748,45 @@ def main(**args):
|
||||
sys.stdout.write('\n')
|
||||
failed += 1
|
||||
|
||||
if args.get('coverage', False):
|
||||
# mkdir -p resultdir
|
||||
os.makedirs(args['results'], exist_ok=True)
|
||||
|
||||
# collect coverage info
|
||||
hits, branches = 0, 0
|
||||
|
||||
with open(args['results'] + '/coverage.mk', 'w') as mk:
|
||||
mk.write(COVERAGE_RESULT_RULES.replace(4*' ', '\t') % dict(
|
||||
results=args['results']))
|
||||
|
||||
cmd = (['make', '-f', 'Makefile'] +
|
||||
list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
|
||||
['-f', args['results'] + '/coverage.mk',
|
||||
args['results'] + '/coverage.gcov'])
|
||||
mpty, spty = pty.openpty()
|
||||
if args.get('verbose', False):
|
||||
print(' '.join(shlex.quote(c) for c in cmd))
|
||||
proc = sp.Popen(cmd, stdout=spty)
|
||||
os.close(spty)
|
||||
mpty = os.fdopen(mpty, 'r', 1)
|
||||
while True:
|
||||
try:
|
||||
line = mpty.readline()
|
||||
except OSError as e:
|
||||
if e.errno == errno.EIO:
|
||||
break
|
||||
raise
|
||||
if args.get('verbose', False):
|
||||
sys.stdout.write(line)
|
||||
# get coverage status
|
||||
m = re.match('^TOTALS +([0-9]+)/([0-9]+)', line)
|
||||
if m:
|
||||
hits = int(m.group(1))
|
||||
branches = int(m.group(2))
|
||||
proc.wait()
|
||||
if proc.returncode != 0:
|
||||
sys.exit(-3)
|
||||
|
||||
if args.get('gdb', False):
|
||||
failure = None
|
||||
for suite in suites:
|
||||
@@ -735,8 +799,13 @@ def main(**args):
|
||||
failure.case.test(failure=failure, **args)
|
||||
sys.exit(0)
|
||||
|
||||
print('tests passed: %d' % passed)
|
||||
print('tests failed: %d' % failed)
|
||||
print('tests passed %d/%d (%.2f%%)' % (passed, total,
|
||||
100*(passed/total if total else 1.0)))
|
||||
print('tests failed %d/%d (%.2f%%)' % (failed, total,
|
||||
100*(failed/total if total else 1.0)))
|
||||
if args.get('coverage', False):
|
||||
print('coverage %d/%d (%.2f%%)' % (hits, branches,
|
||||
100*(hits/branches if branches else 1.0)))
|
||||
return 1 if failed > 0 else 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -749,6 +818,9 @@ if __name__ == "__main__":
|
||||
directory of tests, a specific file, a suite by name, and even a \
|
||||
specific test case by adding brackets. For example \
|
||||
\"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TESTDIR))
|
||||
parser.add_argument('--results', default=RESULTDIR,
|
||||
help="Directory to store results. Created implicitly. Only used in \
|
||||
this script for coverage information if --coverage is provided.")
|
||||
parser.add_argument('-D', action='append', default=[],
|
||||
help="Overriding parameter definitions.")
|
||||
parser.add_argument('-v', '--verbose', action='store_true',
|
||||
@@ -769,10 +841,15 @@ if __name__ == "__main__":
|
||||
help="Run tests normally.")
|
||||
parser.add_argument('-r', '--reentrant', action='store_true',
|
||||
help="Run reentrant tests with simulated power-loss.")
|
||||
parser.add_argument('-V', '--valgrind', action='store_true',
|
||||
parser.add_argument('--valgrind', action='store_true',
|
||||
help="Run non-leaky tests under valgrind to check for memory leaks.")
|
||||
parser.add_argument('-e', '--exec', default=[], type=lambda e: e.split(),
|
||||
parser.add_argument('--exec', default=[], type=lambda e: e.split(),
|
||||
help="Run tests with another executable prefixed on the command line.")
|
||||
parser.add_argument('-d', '--disk',
|
||||
parser.add_argument('--disk',
|
||||
help="Specify a file to use for persistent/reentrant tests.")
|
||||
parser.add_argument('--coverage', action='store_true',
|
||||
help="Collect coverage information across tests. This is stored in \
|
||||
the results directory. Coverage is not reset between runs \
|
||||
allowing multiple test runs to contribute to coverage \
|
||||
information.")
|
||||
sys.exit(main(**vars(parser.parse_args())))
|
||||
|
||||
Reference in New Issue
Block a user