Paul Fagerburg has uploaded this change for review. ( https://review.coreboot.org/c/coreboot/+/54072 )
Change subject: tests: code coverage improvements ......................................................................
tests: code coverage improvements
Fix the exclusion path for lcov; it should exclude the directory with source code, not object files.
Use the COV environment variable to * control whether we build for coverage or not * select the output directory
Add a separate target for generating the report, so we can get a report for all of the tests together or just a single test.
Add documentation.
Signed-off-by: Paul Fagerburg pfagerburg@google.com Change-Id: I2bd2bfdedfab291aabeaa968c10b17e9b61c9c0a --- A Documentation/technotes/2021-05-code-coverage.md M Documentation/tutorial/part3.md M tests/Makefile.inc 3 files changed, 85 insertions(+), 12 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/72/54072/1
diff --git a/Documentation/technotes/2021-05-code-coverage.md b/Documentation/technotes/2021-05-code-coverage.md new file mode 100644 index 0000000..bf90bab --- /dev/null +++ b/Documentation/technotes/2021-05-code-coverage.md @@ -0,0 +1,45 @@ +# Unit Test Code Coverage + +Code coverage for the coreboot unit tests allows us to see what lines of +code in the coreboot library are covered by unit tests, and allows a test +author to see where they need to add test cases for additional coverage. + +Enable code coverage in your unit test build by setting the environment +variable `COV` to 1; either `export COV=1` in your shell, or add it to your +`make` command, e.g. `COV=1 make unit-tests`. + +The build output directory is either `build/tests` or `build/coverage`, +depending on whether `COV=1` is set in the environment. + +There is a new `make` target, `coverage-report-tests`, which generates a code +coverage report from all of the GCOV data (`*.gcda` and `*.gcno` files) in the +build directory. + +All of the unit test targets are available with and without `COV=1` +* `clean-unit-tests` +* `build-unit-tests` +* `run-unit-tests` +* `unit-tests` (which is just `build-unit-tests` followed by `run-unit-tests`) + +## Examples + +`COV=1 make build-unit-tests` builds all of the unit tests for code coverage. + +`COV=1 make run-unit-tests` runs the unit tests, building them for code +coverage if they are out-of-date. + +`COV=1 make coverage-report-tests` creates the code coverage report. This +target does not explicitly depend on the tests being built and run; it gathers +the code coverage data from the output directory, which it assumes already +exists. + +`COV=1 make tests/lib/uuid-test coverage-report-tests` builds the uuid test +for code coverage, runs it, and generates a code coverage report just for +that test. + +As an example of using the different output directories: +1. `make build-unit-tests` builds unit tests without code coverage into +`build/tests`. +2. `COV=1 make clean-unit-tests` cleans `build/coverage` +3. `make build-unit-tests` doesn't need to build anything in `build/tests`, +because those files weren't affected by the previous `clean-unit-tests`. diff --git a/Documentation/tutorial/part3.md b/Documentation/tutorial/part3.md index 7ccee87..6d147ff 100644 --- a/Documentation/tutorial/part3.md +++ b/Documentation/tutorial/part3.md @@ -3,6 +3,8 @@ ## Introduction General thoughts about unit testing coreboot can be found in [Unit testing coreboot](../technotes/2020-03-unit-testing-coreboot.md). +Additionally, [code coverage](../technotes/2021-05-code-coverage.md) support +is available for unit tests.
This document aims to guide developers through the process of adding and writing unit tests for coreboot modules. diff --git a/tests/Makefile.inc b/tests/Makefile.inc index 44e3c69..71b3a4c 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -1,7 +1,15 @@ # SPDX-License-Identifier: GPL-2.0-only
testsrc = $(top)/tests + +# Place the build output in one of two places depending on COV, so that code +# built with code coverage never mixes with code built without code coverage. +ifeq ($(COV),1) +testobj = $(obj)/coverage +else testobj = $(obj)/tests +endif + cmockasrc = 3rdparty/cmocka cmockaobj = $(objutil)/cmocka
@@ -51,6 +59,12 @@ TEST_CFLAGS += -fno-pie -fno-pic TEST_LDFLAGS += -no-pie
+# Enable code coverage if COV=1 +ifeq ($(COV),1) +TEST_CFLAGS += --coverage +TEST_LDFLAGS += --coverage +endif + # Extra attributes for unit tests, declared per test attributes:= srcs cflags mocks stage
@@ -85,7 +99,7 @@ define TEST_CC_template $($(1)-objs): TEST_CFLAGS+= \ -D__$$(shell echo $$($(1)-stage) | tr '[:lower:]' '[:upper:]')__ -$($(1)-objs): $(obj)/$(1)/%.o: $$$$*.c $(TEST_KCONFIG_AUTOHEADER) +$($(1)-objs): $(testobj)/$(1)/%.o: $$$$*.c $(TEST_KCONFIG_AUTOHEADER) mkdir -p $$(dir $$@) $(HOSTCC) $(HOSTCFLAGS) $$(TEST_CFLAGS) $($(1)-cflags) -MMD \ -MT $$@ -c $$< -o $$@ @@ -97,10 +111,10 @@ endef
$(foreach test, $(alltests), \ - $(eval $(test)-objs:=$(addprefix $(obj)/$(test)/, \ + $(eval $(test)-objs:=$(addprefix $(testobj)/$(test)/, \ $(patsubst %.c,%.o,$($(test)-srcs))))) $(foreach test, $(alltests), \ - $(eval $(test)-bin:=$(obj)/$(test)/run)) + $(eval $(test)-bin:=$(testobj)/$(test)/run)) $(foreach test, $(alltests), \ $(eval $(call TEST_CC_template,$(test))))
@@ -154,14 +168,24 @@ rm -f $(testobj)/junit-$(subst /,_,$^).xml $(testobj)/$(subst /,_,$^).failed -./$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
-.PHONY: coverage-unit-tests +# Build a code coverage report by collecting all the gcov files into a single +# report. If COV is not set, this might be a user error, and they're trying +# to generate a coverage report without first having built and run the code +# with code coverage. So instead of silently correcting it by adding COV=1, +# let's flag it to the user so they can be sure they're doing the thing they +# want to do.
-coverage-unit-tests: TEST_CFLAGS += --coverage -coverage-unit-tests: TEST_LDFLAGS += --coverage -coverage-unit-tests: clean-unit-tests unit-tests - lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '*/$(testobj)/*' - genhtml -q -o build/tests/coverage_rpt -t "coreboot unit tests" \ +.PHONY: coverage-report-tests + +ifeq ($(COV),1) +coverage-report-tests: + lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '$(testsrc)/*' + genhtml -q -o $(testobj)/coverage_rpt -t "coreboot unit tests" \ -s $(testobj)/tests.info +else +coverage-report-tests: + @echo Please use COV=1 when creating code coverage reports. +endif
unit-tests: build-unit-tests run-unit-tests
@@ -181,7 +205,7 @@ fi
$(addprefix clean-,$(alltests)): clean-%: - rm -rf $(obj)/$* + rm -rf $(testobj)/$*
clean-unit-tests: rm -rf $(testobj) @@ -199,6 +223,8 @@ @echo ' list-unit-tests - List all unit-tests' @echo ' <unit-test> - Build and run single unit-test' @echo ' clean-<unit-test> - Remove single unit-test build artifacts' - @echo ' coverage-unit-tests - Build unit tests for code coverage and' - @echo ' generate a code coverage report' + @echo + @echo ' COV=1 - Enable code coverage for building a' + @echo ' single unit test or all tests' + @echo ' coverage-report-tests - Generate a code coverage report' @echo