[coreboot-gerrit] Patch set updated for coreboot: romcc: Rewrite the test system

Sat Apr 9 02:12:09 CEST 2016

Jonathan Neuschäfer (j.neuschaefer at gmx.net) just uploaded a new patch set to gerrit, which you can find at https://review.coreboot.org/14291

-gerrit

commit 6302402fdc09e45fa659c478b7c66b013fc62499
Author: Jonathan Neuschäfer <j.neuschaefer at gmx.net>
Date:   Thu Apr 7 03:30:31 2016 +0200

    romcc: Rewrite the test system
    
    Differences:
     - The test logic is now only implemented in one place (pending the deletion of
       the old parts), whereas it previously was implemented both as make rules and
       as a pair of shell scripts.
     - Tests don't need to be registered anymore. Just adding a new file with the
       correct name is enough to have it tested.
     - The code is hopefully more readable and maintainable.
    
    Things to do in follow-up patches:
     - Remove the old test code
     - Test or remove fail_test*.c, hello_world*.c and raminit_test*.c
     - Fix regressions that have built up over the years, while making sure not to
       introduce new ones
     - Makefile integration
     - Jenkins integration
    
    There are tests in the makefile that specify -fno-always-inline, but this
    option doesn't exist anymore.
    
    Change-Id: Idd6b89368c1e36555cb880c37bbe07035c938cd7
    Signed-off-by: Jonathan Neuschäfer <j.neuschaefer at gmx.net>
---
 .gitignore         |   3 +
 util/romcc/test.sh | 206 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 209 insertions(+)

diff --git a/.gitignore b/.gitignore
index 912aa85..1d27201 100644
--- a/.gitignore
+++ b/.gitignore
@@ -96,8 +96,11 @@ util/nvramtool/.dependencies
 util/nvramtool/nvramtool
 util/optionlist/Options.wiki
 util/romcc/romcc
+util/romcc/logs
 util/romcc/tests/fail_test*.S
+util/romcc/tests/linux_test*.c.S
 util/romcc/tests/*.S-O2-mmmx
+util/romcc/tests/dummy.S
 util/runfw/googlesnow
 util/superiotool/superiotool
 util/vgabios/testbios
diff --git a/util/romcc/test.sh b/util/romcc/test.sh
new file mode 100755
index 0000000..748bdef
--- /dev/null
+++ b/util/romcc/test.sh
@@ -0,0 +1,206 @@
+#!/bin/sh
+#
+# This file is part of the coreboot project.
+#
+# Copyright 2016 Jonathan Neuschäfer <j.neuschaefer at gmx.net>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+
+# These tests are currently known to be broken and should not result in a build
+# failure:
+XFAIL_TESTS="
+	simple_test4.c
+	simple_test6.c
+	simple_test25.c
+	simple_test26.c
+	simple_test46.c
+	simple_test47.c
+	simple_test54.c
+	simple_test72.c
+	simple_test73.c
+	linux_test2.c
+	linux_test5.c
+	linux_test10.c
+	linux_test11.c
+	linux_test12.c
+"
+
+# ------------------------------------------------------------------- #
+
+BASEDIR="$(dirname "$0")"
+mkdir -p "$BASEDIR/logs"
+
+red()   { printf "\033[1;31m$@\033[0m"; }
+green() { printf "\033[1;32m$@\033[0m"; }
+blue()  { printf "\033[1;34m$@\033[0m"; }
+
+init_stats() {
+	NUM_TOTAL=0	# Number of tests that were run
+	NUM_FAIL=0	# Number of tests that failed unexpectedly
+	NUM_BROKEN=0	# Number of tests that failed expectedly
+	NUM_PASS=0	# Number of tests that passed expectedly
+	NUM_FIXED=0	# Number of tests that passed unexpectedly
+}
+
+get_romcc() {
+	ROMCC="$BASEDIR/romcc"
+	if [ ! -f "$ROMCC" ]; then
+		echo "romcc not found! Please run \"make romcc\"."
+		exit 1
+	fi
+}
+
+init_testing() {
+	init_stats
+	get_romcc
+}
+
+show_stats() {
+	printf "passed: $NUM_PASS\t($NUM_FIXED newly fixed)\n"
+	printf "failed: $NUM_FAIL\t($NUM_BROKEN known broken)\n"
+	printf "total:  $NUM_TOTAL\n"
+}
+
+is_xfail() {
+	local t
+	for t in $XFAIL_TESTS; do
+		if [ "$t" = "$1" ]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
+pass() {
+	NUM_TOTAL=$(($NUM_TOTAL + 1))
+	NUM_PASS=$(($NUM_PASS + 1))
+
+	green "passed"
+	if is_xfail "$(basename "$1")"; then
+		blue " (fixed)"
+		NUM_FIXED=$(($NUM_FIXED + 1))
+	fi
+	echo
+}
+
+fail() {
+	NUM_TOTAL=$(($NUM_TOTAL + 1))
+	NUM_FAIL=$(($NUM_FAIL + 1))
+
+	red "failed"
+	if is_xfail "$(basename "$1")"; then
+		blue " (known broken)"
+		NUM_BROKEN=$(($NUM_BROKEN + 1))
+	fi
+	echo
+}
+
+
+run_simple_test() {
+	timeout 60 "$ROMCC" $1 "$2" -o "$BASEDIR/tests/dummy.S"
+}
+
+run_simple_tests() {
+	echo "Running simple tests..."
+
+	local t
+	for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do
+		printf "$(basename "$t")"
+
+		local result=pass
+		local logfile="$BASEDIR/logs/$(basename "$t").log"
+		rm "$logfile" >/dev/null 2>&1
+		for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \
+		           "-O -mmmx" "-O -msse" "-O  -mmmx -msse" \
+		           "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do
+			if run_simple_test "$opt" "$t" >> "$logfile" 2>&1; then
+				printf .
+			else
+				result=fail
+			fi
+		done
+		printf " "
+		$result "$t"
+	done
+
+	echo
+}
+
+run_linux_test() {
+	timeout 60 "$ROMCC" "$1" -o "$1.S" || return 1
+	as --32 "$1.S" -o "$1.o" || return 1
+	ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" "$1.o" -o "$1.elf" || return 1
+	timeout 60 "./$1.elf" > "$1.out" || return 1
+
+	local base="$(basename "$1")"
+	diff -u "$BASEDIR/results/${base%.c}.out" "$1.out"
+}
+
+run_linux_tests() {
+	echo "Running linux tests..."
+
+	local t
+	for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do
+		printf "$(basename "$t")... "
+
+		if run_linux_test "$t" > "$BASEDIR/logs/$(basename "$t").log" 2>&1; then
+			pass "$t"
+		else
+			fail "$t"
+		fi
+	done
+
+	echo
+}
+
+usage () {
+	echo "Usage: test.sh CLASS"
+	echo ""
+	echo "CLASS selects a group of tests to run. It must be one of the following:"
+	echo "  all     - all tests"
+	echo "  simple  - simple tests"
+	echo "  linux   - linux programs whose output is checked against a reference"
+	exit 1
+}
+
+if [ $# -ne 1 ]; then
+	usage
+fi
+
+CLASS="$1"
+
+case "$CLASS" in
+	all)
+		init_testing
+		run_simple_tests
+		run_linux_tests
+		show_stats
+		;;
+	simple)
+		init_testing
+		run_simple_tests
+		show_stats
+		;;
+	linux)
+		init_testing
+		run_linux_tests
+		show_stats
+		;;
+	*)
+		echo "Invalid test class $CLASS"
+		echo
+		usage
+		;;
+esac
+
+if [ $NUM_FAIL -ne $NUM_BROKEN ]; then
+	exit 1
+fi

