Jakub Czapiga has uploaded this change for review. ( https://review.coreboot.org/c/coreboot/+/67372 )
Change subject: [WIP] tests: Add support for tests build failures detection ......................................................................
[WIP] tests: Add support for tests build failures detection
This patch introduces new target: junit.xml-unit-tests, which creates report containing tests biuld logs and allows Jenkins to show, which test failed to build. Failing tests should no longer require digging in the raw console output in order to find, why they failed.
Signed-off-by: Jakub Czapiga jacz@semihalf.com Change-Id: I94184379dcc2ac10f1a47f4a9d205cacbeb640fe --- M tests/Makefile.inc M tests/lib/uuid-test.c M util/testing/Makefile.inc 3 files changed, 62 insertions(+), 6 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/72/67372/1
diff --git a/tests/Makefile.inc b/tests/Makefile.inc index 10caf0a..2b7e3f9 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -60,22 +60,63 @@ DEPENDENCIES += $(addsuffix .d,$(basename $(all-test-objs))) -include $(DEPENDENCIES)
-.PHONY: $(alltests) $(addprefix clean-,$(alltests)) +.PHONY: $(alltests) $(addprefix build-,$(alltests)) +.PHONY: $(addprefix run-,$(alltests)) $(addprefix clean-,$(alltests)) .PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests
# %g in CMOCKA_XML_FILE will be replaced with "__TEST_NAME__(<test-group-name>)" # by macro cb_run_group_tests(), which should be used for running tests. # __TEST_NAME__ contains test name including path e.g. tests_lib_rtc-test ifeq ($(JUNIT_OUTPUT),y) -$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml -$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml +$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml +$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml endif
-$(alltests): $$($$(@)-bin) +$(addprefix run-,$(alltests)): run-%: $$(%-bin) rm -f $(testobj)/junit-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))(*).xml rm -f $(testobj)/$(subst /,_,$^).failed -$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
+$(addprefix build-,$(alltests)): build-%: $$(%-bin) + echo "Building $<" + +$(alltests): run-$$(@) + +TESTS_BUILD_XML_FILE := $(testobj)/junit-tests-build.xml + +$(TESTS_BUILD_XML_FILE): + mkdir -p $(dir $@) + echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@.tmp + for tst in $(alltests); do \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) "clean-$$tst"; \ + echo "<testcase classname='coreboot_build_unit_test' name='$$tst'>" >> $@.tmp; \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$$tst" >> $@.tmp.2 2>&1 \ + && type="system-out" || type="failure"; \ + if [ $$type = "failure" ]; then \ + echo "<failure type='buildFailed'>" >> $@.tmp; \ + else \ + echo "<$$type>" >> $@.tmp; \ + fi; \ + echo '<![CDATA[' >> $@.tmp; \ + cat $@.tmp.2 >> $@.tmp; \ + echo "]]></$$type>" >>$@.tmp; \ + rm -f $@.tmp.2; \ + echo "</testcase>" >> $@.tmp; \ + if [ $$type != 'failure' ]; then \ + $(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$$tst"; \ + fi; \ + done + echo "</testsuite>" >> $@.tmp + mv $@.tmp $@ + +.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests + +junit.xml-unit-tests: clean-junit.xml-unit-tests $(TESTS_BUILD_XML_FILE) + +clean-junit.xml-unit-tests: + rm -f $(TESTS_BUILD_XML_FILE) + + # Build a code coverage report by collecting all the gcov files into a single # report. If COV is not set, this might be a user error, and they're trying # to generate a coverage report without first having built and run the code diff --git a/tests/lib/uuid-test.c b/tests/lib/uuid-test.c index 6b56842..12f5719 100644 --- a/tests/lib/uuid-test.c +++ b/tests/lib/uuid-test.c @@ -63,7 +63,7 @@ const guid_t guid_bin = { { 0xBB, 0x66, 0x4D, 0x1F, 0xAD, 0xDE, - 0xEF, 0xBE, + 0xEF, 0xBE 0x08, 0xAB, 0x21, 0x37, 0xBB, 0x07, 0xDD, 0x4C } };
diff --git a/util/testing/Makefile.inc b/util/testing/Makefile.inc index df6a25e..afe4d11 100644 --- a/util/testing/Makefile.inc +++ b/util/testing/Makefile.inc @@ -104,7 +104,7 @@ $(MAKE) xcompile=$(COREBOOT_BUILD_DIR)/xcompile $(COREBOOT_BUILD_DIR)/xcompile $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra124/lp0 BLD=tegra124_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra210/lp0 BLD=tegra120_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml - $(MAKE) unit-tests JUNIT_OUTPUT=y COV=1 + $(MAKE) junit.xml-unit-tests COV=1 (cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) unit-tests coverage-report JUNIT_OUTPUT=y COV=1) $(MAKE) coverage-report JUNIT_OUTPUT=y COV=1 find . -name 'tests.info' -exec cat {} + >$(COREBOOT_BUILD_DIR)/coverage.info