Change in coreboot[master]: [WIP] Update testing routines

[Martin Roth (Code Review)]

Martin Roth has uploaded this change for review. ( https://review.coreboot.org/20874

Change subject: [WIP] Update testing routines
......................................................................

[WIP] Update testing routines

Currently the only testing we had was 'what-jenkins-does' and
'make lint'. While the lint testing is suitable for developers,
the 'what-jenkins-does' target really isn't, as it was designed
specifically for testing on jenkins.

This adds the infrastructure for tests that are more suitable for the
developer.

TODO:
- Add save/restore of .config file
- Add additional QEMU testing
- Add additional abuild tests for flags that are typically disabled
- Get extended payload test working - Need to create config files
for each.

Change-Id: I19e1256d79531112ff84e47a307f55791533806f
Signed-off-by: Martin Roth 
---
M Makefile
M Makefile.inc
M payloads/libpayload/Makefile.inc
M util/testing/Makefile.inc
4 files changed, 110 insertions(+), 14 deletions(-)

git pull ssh://review.coreboot.org:29418/coreboot refs/changes/74/20874/1

diff --git a/Makefile b/Makefile
index ff9a6b8..343ec65 100644
--- a/Makefile
+++ b/Makefile
@@ -92,9 +92,7 @@
    @echo  '  distclean             - Remove build artifacts and config files'
    @echo  '  doxygen               - Build doxygen documentation for coreboot'
    @echo  '  doxyplatform          - Build doxygen documentation for the current platform'
-	@echo  '  what-jenkins-does     - Run platform build tests (Use CPUS=# for more cores)'
    @echo  '  printall              - print makefile info for debugging'
-	@echo  '  lint / lint-stable    - run coreboot lint tools (all / minimal)'
    @echo  '  gitconfig             - set up git to submit patches to coreboot'
    @echo  '  ctags / ctags-project - make ctags file for all of coreboot or current board'
    @echo  '  cscope / cscope-project - make cscope.out file for coreboot or current board'
diff --git a/Makefile.inc b/Makefile.inc
index 9decd65..49344a6 100644
--- a/Makefile.inc
+++ b/Makefile.inc
@@ -75,7 +75,7 @@

#######################################################################
 # our phony targets
-PHONY+= clean-abuild coreboot lint lint-stable build-dirs build_complete
+PHONY+= clean-abuild coreboot build-dirs build_complete

#######################################################################
 # root source directories of coreboot
@@ -555,9 +555,6 @@

update:
    dongle.py -c /dev/term/1 $(obj)/coreboot.rom EOF
-
-lint lint-stable:
-	util/lint/lint $@

gitconfig:
    [ -d .git ]
diff --git a/payloads/libpayload/Makefile.inc b/payloads/libpayload/Makefile.inc
index b88cd20..7787762 100644
--- a/payloads/libpayload/Makefile.inc
+++ b/payloads/libpayload/Makefile.inc
@@ -147,3 +147,12 @@
    done
    echo "</testsuite>" >> $@.tmp
    mv $@.tmp $@
+
+test-configs:
+	for config in $(filter-out %.old,$(wildcard configs/*)); do \
+		$(MAKE) clean; \
+		cp "$$config" test_config; \
+		echo "*** Making libpayload config $$config ***"; \
+		$(MAKE) olddefconfig DOTCONFIG=test_config V=$(V) Q=$(Q) ; \
+		$(MAKE) V=$(V) Q=$(Q) CONFIG_LP_CCACHE=$(CONFIG_LP_CCACHE) DOTCONFIG=test_config; \
+	done
diff --git a/util/testing/Makefile.inc b/util/testing/Makefile.inc
index f5a4c5e..99f4d84 100644
--- a/util/testing/Makefile.inc
+++ b/util/testing/Makefile.inc
@@ -11,6 +11,24 @@
 ## GNU General Public License for more details.
 ##

+test-help help::
+	@echo  '*** coreboot test targets ***'
+	@echo  '  what-jenkins-does      - Run platform build tests with junit output'
+	@echo  '  lint / lint-stable     - run coreboot lint tools (all / minimal)'
+	@echo  '  test-basic             - Run stardard build tests. All expected to pass.'
+	@echo  '  test-extended          - Run extended tests.'
+	@echo  '  test-lint              - basic: Run stable and extended lint tests.'
+	@echo  '  test-tools             - basic: Tests a basic list of tools.'
+	@echo  '  test-abuild            - basic: Builds all platforms'
+	@echo  '  test-payloads          - basic: Builds internal payloads'
+	@echo  '  test-cleanup           - basic: Cleans coreboot directories'
+	@echo  '  test-external-payloads - extended: Builds external payloads'
+	@echo  '  test-qemu              - extended: Builds & boots QEMU targets'
+
+# junit.xml is a helper target to wrap builds that don't create junit.xml output
+# BLD = Subdirectory the build lives in - The name of the build
+# BLD_DIR = Top path from coreboot to the build subdirectory
+# MAKETARGET = target to build
 junit.xml:
    echo "Building $(BLD)"
    echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@.tmp
@@ -35,16 +53,32 @@
    echo

TOOLLIST= \
-	cbmem \
-	ectool \
-	futility \
-	inteltool \
-	intelvbttool \
-	nvramtool \
-	superiotool \
-	viatool
+cbmem \
+ectool \
+futility \
+inteltool \
+intelvbttool \
+nvramtool \
+superiotool \
+viatool
+
+TEST_PAYLOADLIST_INTERNAL= \
+coreinfo \
+nvramcui \
+
+TEST_PAYLOADLIST_EXTERNAL= \
+iPXE \
+Memtest86Plus \
+SeaBIOS \
+tint
+
 JENKINS_PAYLOAD?=none
+TEST_PAYLOAD?=$(JENKINS_PAYLOAD)
 CPUS?=4
+
+lint lint-stable:
+	util/lint/lint $@
+
 what-jenkins-does:
    util/lint/lint lint-stable --junit
    util/abuild/abuild -B -J $(if $(JENKINS_NOCCACHE),,-y) -c $(CPUS) -z -p $(JENKINS_PAYLOAD) -x -X $(top)/abuild-chromeos.xml
@@ -54,3 +88,61 @@
    unset COREBOOT_BUILD_DIR;$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=payloads/ BLD=nvramcui MFLAGS= MAKEFLAGS= MAKETARGET=all junit.xml
    unset COREBOOT_BUILD_DIR;$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=payloads/ BLD=coreinfo MFLAGS= MAKEFLAGS= MAKETARGET=defaultbuild junit.xml
    $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=util/ BLD=romcc MFLAGS= MAKEFLAGS= MAKETARGET=test junit.xml
+
+test-basic: test-lint test-tools test-abuild test-payloads test-cleanup
+
+test-extended: test-external_payloads test-qemu
+
+test-lint:
+	util/lint/lint lint-stable
+	util/lint/lint-015-final-newlines
+
+test-abuild:
+	util/abuild/abuild -B -e $(if $(TEST_NOCCACHE),,-y) -c $(CPUS) -p $(TEST_PAYLOAD) -x COREBOOT_BUILD_DIR=coreboot-builds-chromeos
+	util/abuild/abuild -B -e $(if $(TEST_NOCCACHE),,-y) -c $(CPUS) -p $(TEST_PAYLOAD)
+
+test-payloads:
+	$(MAKE) -C payloads/libpayload test-configs -j $(CPUS) V=$(V) Q=$(Q) MFLAGS= MAKEFLAGS= $(if $(TEST_NOCCACHE),,CONFIG_LP_CCACHE=y)
+	export MFLAGS=;export MAKEFLAGS=; \
+	$(foreach payload, $(TEST_PAYLOADLIST_INTERNAL), \
+		echo "*** Making payload $(payload) ***"; \
+		$(MAKE) -C payloads/$(payload) distclean ;\
+		$(MAKE) $(payload) -j $(CPUS) V=$(V) Q=$(Q)\
+		|| exit 1; )
+
+test-external-payloads:
+	export MFLAGS=;export MAKEFLAGS=;\
+	$(foreach payload, $(TEST_PAYLOADLIST_EXTERNAL), \
+		echo "*** Making payload $$payload ***"; \
+		$(MAKE) -C payloads/exeternal/$(payload) clean; \
+		$(MAKE) $(payload) -j $(CPUS) V=$(V) Q=$(Q) \
+		|| exit 1; )
+
+test-tools:
+	@echo "Build testing $(TOOLLIST)"
+	$(foreach tool, $(TOOLLIST), $(MAKE) -C util/$(tool) clean ; )
+	$(MAKE) -C util/romcc clean
+	$(foreach tool, $(TOOLLIST), echo "Building $(tool)";export MFLAGS= ;export MAKEFLAGS= ;$(MAKE) -C util/$(tool) all V=$(V) Q=$(Q) || exit 1; )
+	echo "Building romcc"
+	$(MAKE) -C util/romcc all test -j $(CPUS) V=$(V) Q=$(Q)
+
+test-qemu:
+	cp $(top)/configs/config.emulation_qemu_x86_i440fx $(top)/.config
+	echo "CONFIG_PAYLOAD_NONE=y" > $(top)/.config
+	make olddefconfig; make clean; make all
+	qemu-system-x86_64 -bios build/coreboot.rom -serial file:QEMU.LOG -daemonize -pidfile $(top)/QEMU.PID
+	sleep 5
+	kill $$(cat $(top)/QEMU.PID)
+	grep -q "Payload not loaded."  $(top)/QEMU.LOG
+	rm $(top)/QEMU.PID $(top)/QEMU.LOG
+
+test-cleanup:
+	rm -rf coreboot-builds coreboot-builds-chromeos
+	$(MAKE) clean
+	$(MAKE) distclean
+	$(foreach tool, $(TOOLLIST), $(MAKE) -C util/$(tool) clean ; )
+	$(MAKE) -C util/romcc clean
+
+.PHONY: test-basic test-extended test-lint test-abuild test-payloads
+.PHONY: test-external_payloads test-tools test-qemu test-cleanup test-help
+.PHONY: lint lint-stable what-jenkins-does