New subject: Change in flashrom[master]: ubertest: Add blackbox test uber-script

9 Feb 2020

David Hendricks has uploaded this change for review. ( https://review.coreboot.org/c/flashrom/+/38788 )
Change subject: testing: Add blackbox test uber-script
......................................................................
testing: Add blackbox test uber-script
This modifies CB:23025 further to work with upstream as it is now,
without the syntax changes in the patch chain. I also gave it a new
name since this script is, well, uber.
Since flashrom currently only supports reading/writing ROM-sized files
we can't easily determine a targeted region offset and size except
when a layout file is used. Therefore, some extra arithmetic is needed
in the partial write test and the only modes allowed are clobber mode
and layout mode.
A few other changes:
- Update paths and script name
- Remove write-protect testing support
- Use ROM-sized files only, no region-sized files
- Return error if flashmap or ifd mode are used
Tested using a Raspberry Pi with a W25Q16
Change-Id: I1af55d5088c54ee33853009797adbd535a506b49
Signed-off-by: David Hendricks david.hendricks@gmail.com
---
A util/testing/cmd.sh
A util/testing/ubertest.sh
2 files changed, 1,074 insertions(+), 0 deletions(-)
git pull ssh://review.coreboot.org:29418/flashrom refs/changes/88/38788/1

diff --git a/util/testing/cmd.sh b/util/testing/cmd.sh
new file mode 100644
index 0000000..1d1c47a
--- /dev/null
+++ b/util/testing/cmd.sh
@@ -0,0 +1,137 @@
+#!/bin/sh
+#
+# This file is part of the flashrom project. It is derived from
+# board_status.sh in coreboot.
+#
+# Copyright (C) 2016 Google Inc.
+# Copyright (C) 2014 Sage Electronic Engineering, LLC.
+
+USE_CUSTOM_HOOKS=0
+if [ -n "$CUSTOM_HOOKS_FILENAME" ]; then
+	USE_CUSTOM_HOOKS=1
+	. "$CUSTOM_HOOKS_FILENAME"
+	if [ $? -ne 0 ]; then
+		echo "Failed to source custom hooks file."
+		exit $EXIT_FAILURE
+	fi
+
+	if ! custom_hook_sanity_check; then
+		echo "Failed to run sanity check for custom hooks."
+		exit $EXIT_FAILURE
+	fi
+fi
+
+# test a command
+#
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: command to test
+# $3: 0 ($FATAL) Exit with an error if the command fails
+#     1 ($NONFATAL) Don't exit on command test failure
+test_cmd()
+{
+	local rc
+	local cmd__="$(echo $2 | cut -d ' ' -f -1)"
+	local args="$(echo $2 | cut -d ' ' -f 2-)"
+
+	if [ -e "$cmd__" ]; then
+		return
+	fi
+
+	if [ "$1" -eq "$REMOTE" ] && [ -n "$REMOTE_HOST" ]; then
+		ssh $REMOTE_PORT_OPTION root@${REMOTE_HOST} command -v "$cmd__" $args > /dev/null 2>&1
+		rc=$?
+	else
+		command -v "$cmd__" $args >/dev/null 2>&1
+		rc=$?
+	fi
+
+	if [ $rc -eq 0 ]; then
+		return 0
+	fi
+
+	if [ "$3" = "1" ]; then
+		return 1
+	fi
+
+	echo "$2 not found"
+	exit $EXIT_FAILURE
+}
+
+# Same args as cmd() but with the addition of $4 which determines if the
+# command should be totally silenced or not.
+_cmd()
+{
+	local silent=$4
+	local rc=0
+
+	if [ -n "$3" ]; then
+		pipe_location="${3}"
+	else
+		pipe_location="/dev/null"
+	fi
+
+	if [ $1 -eq $REMOTE ] && [ -n "$REMOTE_HOST" ]; then
+		if [ $silent -eq 0 ]; then
+			ssh $REMOTE_PORT_OPTION "root@${REMOTE_HOST}" "$2" > "$pipe_location" 2>/dev/null
+			rc=$?
+		else
+			ssh $REMOTE_PORT_OPTION "root@${REMOTE_HOST}" "$2" >/dev/null 2>&1
+			rc=$?
+		fi
+	else
+		if [ $USE_CUSTOM_HOOKS -eq 1 ]; then
+			preflash_hook $1 "$2" "$3" $4
+		fi
+
+		if [ $silent -eq 0 ]; then
+			$SUDO_CMD $2 > "$pipe_location" 2>/dev/null
+			rc=$?
+		else
+			$SUDO_CMD $2 >/dev/null 2>&1
+			rc=$?
+		fi
+
+		if [ $USE_CUSTOM_HOOKS -eq 1 ]; then
+			postflash_hook $1 "$2" "$3" $4
+		fi
+	fi
+
+	return $rc
+}
+
+# run a command
+#
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: command
+# $3: filename to direct output of command into
+cmd()
+{
+	_cmd $1 "$2" "$3" 0
+
+	if [ $? -eq 0 ]; then
+		return
+	fi
+
+	echo "Failed to run "$2", aborting"
+	rm -f "$3"	# don't leave an empty file
+	return $EXIT_FAILURE
+}
+
+# run a command silently
+#
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: command
+scmd()
+{
+	_cmd $1 "$2" "" 1
+
+	if [ $? -eq 0 ]; then
+		return
+	fi
+
+	echo "Failed to run "$2", aborting"
+	return $EXIT_FAILURE
+}
diff --git a/util/testing/ubertest.sh b/util/testing/ubertest.sh
new file mode 100755
index 0000000..f51aa05
--- /dev/null
+++ b/util/testing/ubertest.sh
@@ -0,0 +1,937 @@
+#!/bin/sh
+#
+# Copyright (C) 2016 Google Inc.
+# Copyright (C) 2020 Facebook Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+#
+# Documentation for this script currently resides at: https://goo.gl/qNwdmm
+
+EXIT_SUCCESS=0
+EXIT_FAILURE=1
+RC=$EXIT_SUCCESS
+FATAL=0
+NONFATAL=1
+
+#
+# Stuff obtained from command-line
+#
+
+# Generic options
+BACKUP_IMAGE=""
+OLD_FLASHROM=""
+NEW_FLASHROM=""
+NO_CLEAN=0
+SKIP_CONSISTENCY_CHECK=0
+UPLOAD_RESULTS=0
+
+# LOCAL_FLASHROM is required if both a secondary programmer *and* a remote host
+# are used since OLD_FLASHROM and NEW_FLASHROM will be called on the remote
+# host and we need a local copy of flashrom to control the secondary programmer.
+# By default this will be set to the result of `which flashrom`.
+LOCAL_FLASHROM=""
+
+# Primary/Secondary programmer options
+PRIMARY_OPTS=""
+SECONDARY_OPTS=""
+
+# Calls preflash_hook() and postflash_hook() before and after doing a command.
+CUSTOM_HOOKS_FILENAME=""
+
+# logfile to store the script's output
+SCRIPT_LOGFILE="flashrom-test_script_output.txt"
+
+# Test type
+TEST_TYPE_UNKNOWN=0
+TEST_TYPE_SINGLE=1
+TEST_TYPE_ENDURANCE=2
+TEST_TYPE=$TEST_TYPE_UNKNOWN
+
+# Region modes
+REGION_MODE_UNKNOWN=0
+REGION_MODE_CLOBBER=1
+REGION_MODE_DESCRIPTOR=2
+REGION_MODE_FLASHMAP=3
+REGION_MODE_LAYOUT=4
+REGION_MODE=$REGION_MODE_UNKNOWN
+DESCRIPTOR_REGION="BIOS"
+FLASHMAP_REGION="RW_SECTION_A"
+LAYOUT_FILE=""
+LAYOUT_REGION="RW"
+SMALL_REGION=0
+
+# Remote testing options
+SSH_PORT=""
+REMOTE_HOST=""
+REMOTE_PORT_OPTION=""
+REMOTE_ONLY=0
+REMOTE_PROGRAMMER_PARAMS=""
+SSH_CMD="ssh $REMOTE_PORT_OPTION root@${REMOTE_HOST} command -v"
+
+LOCAL=0
+REMOTE=1
+DO_REMOTE=0	# boolean to use for cmd() and tests
+
+# relative path from flashrom root directory
+TEST_SCRIPT_PATH="util/testing/test_v2.1.sh"
+
+# In case we need to run flashrom locally and we're not already root.
+SUDO_CMD=""
+if [ "$(id -u)" -ne "0" ]; then
+	SUDO_CMD="sudo"
+fi
+
+# 1KB
+K=1024
+
+show_help() {
+	printf "Usage:
+	${0} <options>
+
+General options:
+    -b, --backup-image <path>
+        Backup image to write unconditionally at end of testing.
+    -h, --help
+        Show this message.
+    -l, --layout-file <path>
+        Layout file (required if mode is "layout", resides locally).
+    -m, --mode <arg>
+        Region access mode (clobber, descriptor, flashmap, layout).
+    -n, --new <path>
+        Path to new version of flashrom.
+    -o, --old <path>
+        Path to old (stable) version of flashrom.
+    -p, --primary-programmer <parameters>
+        Primary programmer options.
+    -r, --remote-host <host>
+        Remote host to test primary programmer on.
+    -s, --secondary-programmer <parameters>
+        Secondary programmer options.
+    -t, --type <arg>
+        Test type (single, endurance).
+    -u, --upload-results
+        Upload results to flashrom.org.
+    -v, --voltage
+        Chip voltage in millivolts (usually 1800 or 3300).
+
+Long options:
+    --custom-hooks <filename>
+        Supply a script with custom hooks to run before and after commands.
+    --descriptor-region <name>
+        Specify region to use in descriptor mode (default: $DESCRIPTOR_REGION)
+    --flashmap-region <name>
+        Specify region to use in flashmap mode (default: $FLASHMAP_REGION)
+    --layout-region <name>
+        Specify region to use in layout mode (default: $LAYOUT_REGION)
+    --local-flashrom <path>
+        Path to local version of flashrom when using both a secondary programmer
+        and remote host (default: $($SUDO_CMD which flashrom))
+    --no-clean
+        Do not remove temporary files.
+    --skip-consistency-check
+        Skip the consistency check (two consecutive reads) at beginning.
+    --small-region
+        Omit tests that require large amounts of space (>16KB).
+Remote connectivity options:
+    --ssh-port <port>
+        Use a specific SSH port.
+
+See documentation for usage examples (TODO: Migrate https://goo.gl/3jNoL7
+to flashrom wiki).\n
+"
+}
+
+getopt -T
+if [ $? -ne 4 ]; then
+	printf "GNU-compatible getopt(1) required.\n"
+	exit $EXIT_FAILURE
+fi
+
+LONGOPTS="backup-image:,help,,new:,old:,remote-host:,upload-results:"
+LONGOPTS="${LONGOPTS},primary-programmer:,secondary-programmer:,local-flashrom:"
+LONGOPTS="${LONGOPTS},custom-hooks:,mode:,skip-consistency-check,small-region"
+LONGOPTS="${LONGOPTS},type:,voltage:"
+LONGOPTS="${LONGOPTS},layout-file:,descriptor-region:,flashmap-region:,layout-region:"
+LONGOPTS="${LONGOPTS},no-clean"
+LONGOPTS="${LONGOPTS},ssh-port:"
+
+ARGS=$(getopt -o b:hl:m:n:o:p:r:s:t:u -l "$LONGOPTS" -n "$0" -- "$@");
+if [ $? != 0 ] ; then printf "Terminating...\n" >&2 ; exit 1 ; fi
+eval set -- "$ARGS"
+while true ; do
+	case "$1" in
+		# Generic options
+		-b|--backup-image)
+			shift
+			BACKUP_IMAGE="$1"
+			;;
+		-h|--help)
+			show_help
+			exit $EXIT_SUCCESS
+			;;
+		-l|--layout-file)
+			shift
+			LAYOUT_FILE="$1"
+			;;
+		-m|--mode)
+			shift
+			if [ "$1" = "clobber" ]; then
+				REGION_MODE=$REGION_MODE_CLOBBER
+			elif [ "$1" = "descriptor" ]; then
+				REGION_MODE=$REGION_MODE_DESCRIPTOR
+			elif [ "$1" = "flashmap" ]; then
+				REGION_MODE=$REGION_MODE_FLASHMAP
+			elif [ "$1" = "layout" ]; then
+				REGION_MODE=$REGION_MODE_LAYOUT
+			else
+				printf "Unknown mode: $1\n"
+				exit $EXIT_FAILURE
+			fi
+			;;
+		-n|--new)
+			shift
+			NEW_FLASHROM="$1"
+			;;
+		-o|--old)
+			shift
+			OLD_FLASHROM="$1"
+			;;
+		-p|--primary_programmer)
+			shift
+			PRIMARY_OPTS="-p $1"
+			;;
+		-s|--secondary_programmer)
+			shift
+			SECONDARY_OPTS="-p $1"
+			;;
+		-t|--type)
+			shift
+			if [ "$1" = "single" ]; then
+				TEST_TYPE=$TEST_TYPE_SINGLE
+			elif [ "$1" = "endurance" ]; then
+				TEST_TYPE=$TEST_TYPE_ENDURANCE
+			else
+				printf "Unknown type: $1\n"
+				exit $EXIT_FAILURE
+			fi
+			;;
+		-r|--remote-host)
+			DO_REMOTE=1
+			shift
+			REMOTE_HOST="$1"
+			;;
+		-u|--upload-results)
+			UPLOAD_RESULTS=1
+			;;
+		-v|--voltage)
+			shift
+			VOLTAGE="$1"
+			;;
+
+		# Longopts only
+		--custom-hooks)
+			shift
+			CUSTOM_HOOKS_FILENAME="$1"
+			;;
+		--descriptor-region)
+			shift
+			DESCRIPTOR_REGION="$1"
+			;;
+		--flashmap-region)
+			shift
+			FLASHMAP_REGION="$1"
+			;;
+		--layout-region)
+			shift
+			LAYOUT_REGION="$1"
+			;;
+		--local-flashrom)
+			shift
+			LOCAL_FLASHROM="$1"
+			;;
+		--no-clean)
+			NO_CLEAN=1
+			;;
+		--skip-consistency-check)
+			SKIP_CONSISTENCY_CHECK=1
+			;;
+		--small-region)
+			SMALL_REGION=1
+			;;
+
+		# Remote testing options
+		--ssh-port)
+			shift
+			REMOTE_PORT_OPTION="-p $1"
+			;;
+
+		# error handling
+		--)
+			shift
+			if [ -n "$*" ]; then
+				printf "Non-option parameters detected: '$*'\n"
+				exit $EXIT_FAILURE
+			fi
+			break
+			;;
+		*)
+			printf "error processing options at '$1'\n"
+			exit $EXIT_FAILURE
+	esac
+	shift
+done
+
+# TODO: Implement this.
+if [ $UPLOAD_RESULTS -eq 1 ]; then
+	printf "TODO: Implement ability to upload results.\n"
+	exit $EXIT_FAILURE
+fi
+
+#
+# Source helper scripts
+#
+export REMOTE_HOST REMOTE_PORT_OPTION
+export LOCAL REMOTE FATAL NONFATAL EXIT_SUCCESS EXIT_FAILURE
+export CUSTOM_HOOKS_FILENAME SUDO_CMD VOLTAGE
+. "$(pwd)/util/testing/cmd.sh"
+
+# We will set up a logs directory within the tmpdirs to store
+# all output logs.
+LOGS="logs"
+
+# Setup temporary working directories:
+# LOCAL_TMPDIR:  Working directory on local host.
+# REMOTE_TMPDIR: Working directory on remote host.
+# TMPDIR:        The temporary directy in which we do most of the work. This is
+#                convenient for commands that depend on $DO_REMOTE.
+LOCAL_TMPDIR=$(mktemp -d --tmpdir flashrom_test.XXXXXXXX)
+if [ $? -ne 0 ] ; then
+	printf "Could not create temporary directory\n"
+	exit $EXIT_FAILURE
+fi
+mkdir "${LOCAL_TMPDIR}/${LOGS}"
+
+if [ $DO_REMOTE -eq 1 ]; then
+	REMOTE_TMPDIR=$(ssh root@${REMOTE_HOST} mktemp -d --tmpdir flashrom_test.XXXXXXXX)
+	if [ $? -ne 0 ] ; then
+		printf "Could not create temporary directory\n"
+		exit $EXIT_FAILURE
+	fi
+	scmd $REMOTE "mkdir ${REMOTE_TMPDIR}/${LOGS}"
+fi
+
+if [ $DO_REMOTE -eq 0 ]; then
+	TMPDIR="$LOCAL_TMPDIR"
+else
+	TMPDIR="$REMOTE_TMPDIR"
+fi
+
+#
+# Test command-line validity.
+#
+if [ $TEST_TYPE -eq $TEST_TYPE_UNKNOWN ]; then
+	printf "Must specify a test type (-t/--type).\n"
+	exit $EXIT_FAILURE
+elif [ $TEST_TYPE -eq $TEST_TYPE_SINGLE ]; then
+	if [ $REGION_MODE -eq $REGION_MODE_UNKNOWN ]; then
+		printf "Must specify a region access mode (-m/--mode).\n"
+		exit $EXIT_FAILURE
+	elif [ $REGION_MODE -eq $REGION_MODE_LAYOUT ]; then
+		if [ -z "$LAYOUT_FILE" ]; then
+			printf "Must specify a layout file when using layout mode.\n"
+			exit $EXIT_FAILURE
+		fi
+
+		scmd $DO_REMOTE "stat $LAYOUT_FILE"
+		if [ $? -ne 0 ]; then
+			if [ $DO_REMOTE -eq 1 ]; then
+				tmp=" on remote host $REMOTE_HOST."
+			else
+				tmp=" on local host."
+			fi
+			printf "Layout file $LAYOUT_FILE not found${tmp}\n"
+			exit $EXIT_FAILURE
+		fi
+
+		if [ $DO_REMOTE -eq 1 ]; then
+			scp root@"${REMOTE_HOST}:$LAYOUT_FILE" "${LOCAL_TMPDIR}/" 2>&1 >/dev/null
+		else
+			cp "$LAYOUT_FILE" "${LOCAL_TMPDIR}/"
+		fi
+	fi
+fi
+
+if [ -n "$VOLTAGE" ]; then
+	echo "$VOLTAGE" | grep -q '[^0-9]'
+	if [ $? -ne 1 ]; then
+		printf "Voltage must be an integer with units of millivolts."
+		exit $EXIT_FAILURE
+	fi
+fi
+
+if [ $DO_REMOTE -eq 1 ]; then
+	# Test connection to remote host
+	test_cmd $DO_REMOTE "ls /" $NONFATAL
+	if [ $? -ne 0 ]; then
+		printf "Could not connect to remote host ${REMOTE_HOST}\n"
+		exit $EXIT_FAILURE
+	fi
+fi
+
+# Remote host and secondary programmer are in use, so either the user must
+# specify a local version of flashrom to control the secondary programmer
+# or it must be found in the default path.
+if [ $DO_REMOTE -eq 1 ] && [ -n "$SECONDARY_OPTS" ]; then
+	if [ -z "$LOCAL_FLASHROM" ]; then
+		LOCAL_FLASHROM="$($SUDO_CMD which flashrom)"
+	fi
+
+	if [ ! -e "$LOCAL_FLASHROM" ]; then
+		printf "$LOCAL_FLASHROM does not exist\n"
+		exit $EXIT_FAILURE
+	fi
+fi
+
+#
+# Dependencies
+#
+
+# cmp is used to compare files
+test_cmd $DO_REMOTE "cmp" $FATAL
+
+if [ ! -e "/dev/urandom" ]; then
+	printf "This script uses /dev/urandom\n"
+	exit $EXIT_FAILURE
+fi
+
+if [ ! -e "/dev/zero" ]; then
+	printf "This script uses /dev/zero\n"
+	exit $EXIT_FAILURE
+fi
+
+#
+# Setup.
+#
+
+# Naive path check
+if [ ! -e "$TEST_SCRIPT_PATH" ] ; then
+	printf "Script must be run from root of flashrom directory\n"
+	exit $EXIT_FAILURE
+fi
+
+if [ -z "$OLD_FLASHROM" ]; then
+	if [ $DO_REMOTE -eq 1 ]; then
+		OLD_FLASHROM="$(ssh root@${REMOTE_HOST} which flashrom)"
+	else
+		OLD_FLASHROM="$($SUDO_CMD which flashrom)"
+	fi
+fi
+test_cmd $DO_REMOTE "$OLD_FLASHROM --help" $NONFATAL
+if [ $? -ne 0 ]; then
+	printf "Old flashrom binary is not usable.\n"
+	exit $EXIT_FAILURE
+fi
+
+# print $1 and store it in the script log file
+print_and_log()
+{
+	printf "$1" | tee -a "${LOCAL_TMPDIR}/${LOGS}/${SCRIPT_LOGFILE}"
+}
+
+# Copy files from local tmpdir to remote host tmpdir
+copy_to_remote()
+{
+	for F in $@; do
+		scp "${LOCAL_TMPDIR}/${F}" root@"${REMOTE_HOST}:${REMOTE_TMPDIR}" 2>&1 >/dev/null
+	done
+}
+
+# Copy files from remote host tmpdir to local tmpdir
+copy_from_remote()
+{
+	for F in $@; do
+		scp root@"${REMOTE_HOST}:${REMOTE_TMPDIR}/${F}" "${LOCAL_TMPDIR}/" 2>&1 >/dev/null
+	done
+}
+
+# A wrapper for scmd calls to flashrom when we want to log the output
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: arguments to be passed into scmd
+# $3: context of the flashrom call (to be used in the logfile)
+flashrom_log_scmd()
+{
+	local logfile="flashrom-${3}.txt"
+	local rc
+
+	if [ $1 -eq $REMOTE ]; then
+		tmpdir=$REMOTE_TMPDIR
+	else
+		tmpdir=$LOCAL_TMPDIR
+	fi
+
+	scmd $1 "$2 -o ${tmpdir}/${LOGS}/${logfile}"; rc=$?
+	# if the call was successful, we don't want to save the log (only save failure logs)
+	if [ $rc -eq $EXIT_SUCCESS ]; then
+		scmd $1 "rm -f ${tmpdir}/${LOGS}/${logfile}"
+	else
+		# if the log was stored remotely, we want to copy it over to local tmpdir
+		if [ $1 -eq $REMOTE ]; then
+			scp root@"${REMOTE_HOST}:${REMOTE_TMPDIR}/${LOGS}/${logfile}" "${LOCAL_TMPDIR}/${LOGS}" 2>&1 >/dev/null
+		fi
+	fi
+
+	return $rc
+}
+
+# Read current image as backup in case one hasn't already been specified.
+if [ -z "$BACKUP_IMAGE" ]; then
+	backup_file="backup.bin"
+	if [ $DO_REMOTE -eq 1 ]; then
+		BACKUP_IMAGE="${REMOTE_TMPDIR}/${backup_file}"
+	else
+		BACKUP_IMAGE="${LOCAL_TMPDIR}/${backup_file}"
+	fi
+
+	print_and_log "Reading backup image..."
+	flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -r $BACKUP_IMAGE" "read_backup"
+
+	if [ $? -ne 0 ]; then
+		print_and_log "Failed to read backup image, aborting.\n"
+		exit $EXIT_FAILURE
+	fi
+
+	if [ $DO_REMOTE -eq 1 ]; then
+		copy_from_remote "$backup_file"
+	fi
+else
+	if [ $DO_REMOTE -eq 1 ]; then
+		scmd $DO_REMOTE "cp $BACKUP_IMAGE $REMOTE_TMPDIR"
+		copy_from_remote "$(basename $BACKUP_IMAGE)"
+	fi
+fi
+
+# The copy of flashrom to test. If unset, we'll assume the user wants to test
+# a newly built flashrom binary in the current directory.
+if [ -z "$NEW_FLASHROM" ] ; then
+	if [ -x "flashrom" ]; then
+		NEW_FLASHROM="flashrom"
+	else
+		print_and_log "Must supply new flashrom version to test\n"
+		exit $EXIT_FAILURE
+	fi
+fi
+
+print_and_log "Stable flashrom binary: ${OLD_FLASHROM}\n"
+print_and_log "New flashrom binary to test: ${NEW_FLASHROM}\n"
+print_and_log "Local temporary files will be stored in ${LOCAL_TMPDIR}\n"
+if [ $DO_REMOTE -eq 1 ]; then
+	print_and_log "Remote temporary files will be stored in ${REMOTE_HOST}:${REMOTE_TMPDIR}\n"
+	print_and_log "Backup image: ${REMOTE_HOST}:${BACKUP_IMAGE}\n"
+	print_and_log "Backup image also stored at: ${LOCAL_TMPDIR}/$(basename ${BACKUP_IMAGE})\n"
+else
+	print_and_log "Backup image: ${BACKUP_IMAGE}\n"
+fi
+
+#
+# Now the fun begins.
+#
+cmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS --flash-size" "${LOCAL_TMPDIR}/old_chip_size.txt.orig"
+cmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS --flash-size" "${LOCAL_TMPDIR}/new_chip_size.txt.orig"
+# get rid of banner and other superfluous output from flashrom
+tail -n 1 "${LOCAL_TMPDIR}/old_chip_size.txt.orig" > "${LOCAL_TMPDIR}/old_chip_size.txt"
+tail -n 1 "${LOCAL_TMPDIR}/new_chip_size.txt.orig" > "${LOCAL_TMPDIR}/new_chip_size.txt"
+tmp=$(cat ${LOCAL_TMPDIR}/old_chip_size.txt)
+CHIP_SIZE=$(cat ${LOCAL_TMPDIR}/new_chip_size.txt)
+CHIP_SIZE_KB=$(($CHIP_SIZE / $K))
+CHIP_SIZE_HALF=$(($CHIP_SIZE / 2))
+if [ $CHIP_SIZE -ne $tmp ]; then
+	print_and_log "New flashrom and old flashrom disagree on chip size. Aborting.\n"
+	exit $EXIT_FAILURE
+else
+	print_and_log "Chip size: $CHIP_SIZE_KB KiB\n"
+fi
+
+# Upload results
+#do_upload()
+#{
+#	# TODO: implement this
+#}
+
+# Remove temporary files
+do_cleanup()
+{
+	if [ $NO_CLEAN -eq 1 ]; then
+		print_and_log "Skipping cleanup.\n"
+		return $EXIT_SUCCESS
+	fi
+
+	rm -rf "$LOCAL_TMPDIR"
+	if [ -n "$REMOTE_HOST" ]; then
+		ssh root@${REMOTE_HOST} rm -rf "$REMOTE_TMPDIR"
+	fi
+
+	return $EXIT_SUCCESS
+}
+
+# $1: Message to display to user.
+test_fail()
+{
+	print_and_log "$1\n"
+	printf "Skipping cleanup (logs saved).\n"
+	exit $EXIT_FAILURE
+}
+
+write_backup_image()
+{
+	print_and_log "Writing backup image.\n"
+	flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -w $BACKUP_IMAGE" "write_backup"
+	if [ $? -ne 0 ]; then
+		test_fail "Failed to write backup image.\n"
+	fi
+}
+
+# Read a region twice and compare results
+# $1: address of region (in bytes)
+# $2: length of region (in bytes)
+double_read_test()
+{
+	local cmp1="${TMPDIR}/read_test1.bin"
+	local cmp2="${TMPDIR}/read_test2.bin"
+	local layout="double_read_test_layout.txt"
+	local len=$(($2 / $K))
+
+	print_and_log "Doing double read test, size: $len KiB\n"
+	# FIXME: Figure out how to do printf remotely...
+	printf "%06x:%06x region\n" $1 $(($1 + $2 - 1)) > "${LOCAL_TMPDIR}/${layout}"
+	if [ $DO_REMOTE -eq 1 ]; then copy_to_remote "$layout" ; fi
+
+	flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -l ${TMPDIR}/${layout} -i region -r ${cmp1}" "double_read_1"
+	# FIXME: second (or maybe third?) read should be done using secondary programmer, if applicable.
+	flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -l ${TMPDIR}/${layout} -i region -r ${cmp2}" "double_read_2"
+	scmd $DO_REMOTE "cmp $cmp1 $cmp2"
+	if [ $? -ne 0 ]; then
+		test_fail "Double-read test failed, aborting."
+	fi
+}
+
+PARTIAL_WRITE_TEST_REGION_SIZE=0
+PARTIAL_WRITE_TEST_ALIGN_SIZE_KB=0
+# FIXME: Hack due to lack of region-sized file handling.
+PARTIAL_WRITE_TEST_REGION_BASE_OFFSET=-1
+
+# helper function to reduce repetitiveness of partial_write_test
+partial_write_test_helper()
+{
+	local test_name="$1"
+	local pattern_offset_kb=$2
+	local pattern_size_kb=$3
+	local hex="$4"
+	local oct=""
+	local base_offset_kb=$(($PARTIAL_WRITE_TEST_REGION_BASE_OFFSET / $K))
+
+	oct="\$(printf "%03o" $hex)"
+	cp "${LOCAL_TMPDIR}/random_4k_test.bin" "${LOCAL_TMPDIR}/${test_name}.bin"
+	dd if=/dev/zero bs=1k count=${pattern_size_kb} 2>/dev/null | tr "\000" "$oct" > "${LOCAL_TMPDIR}/${hex}_${pattern_size_kb}k.bin"
+
+	while [ $(($(($pattern_offset_kb + $pattern_size_kb)) * $K)) -lt $PARTIAL_WRITE_TEST_REGION_SIZE ]; do
+		dd if="${LOCAL_TMPDIR}/${hex}_${pattern_size_kb}k.bin" of="${LOCAL_TMPDIR}/${test_name}.bin" bs=1k count=$pattern_size_kb seek=$(($base_offset_kb + $pattern_offset_kb)) conv=notrunc 2>/dev/null
+		pattern_offset_kb=$(($pattern_offset_kb + $PARTIAL_WRITE_TEST_ALIGN_SIZE_KB))
+	done
+}
+
+# Regional partial write test. Given a region name, this will write patterns
+# of bytes designed to test corner cases.
+#
+# We assume that eraseable block size can be either 4KB or 64KB and
+# must test for both. For simplicity, we'll assume the region size is
+# at least 256KB.
+#
+# $1: Region name
+partial_write_test()
+{
+	local opts="--noverify-all"
+	local secondary_opts=""		# for secondary programmer
+	local region_name="$1"
+	local filename=""
+	local test_num=0
+	local prev_test_num=0
+
+	# FIXME: Hack due to lack of region-sized file handling.
+	if [ $((PARTIAL_WRITE_TEST_REGION_SIZE)) -eq 0 ]; then
+		print_and_log "Size of $region_name unknown\n"
+		return $EXIT_FAILURE
+	fi
+	if [ $((PARTIAL_WRITE_TEST_REGION_BASE_OFFSET)) -lt 0 ]; then
+		print_and_log "Offset of $region_name is unknown\n"
+		return $EXIT_FAILURE
+	fi
+
+	if [ $TEST_TYPE -eq $TEST_TYPE_SINGLE ]; then
+		if [ $REGION_MODE -eq $REGION_MODE_LAYOUT ]; then
+			opts="$opts -l $LAYOUT_FILE"
+			secondary_opts="$opts"
+		elif [ $REGION_MODE -eq $REGION_MODE_CLOBBER ]; then
+			printf "000000:%06x RW\n" $(($CHIP_SIZE - 1)) > "${LOCAL_TMPDIR}/clobber_mode_layout.txt"
+			if [ $DO_REMOTE -eq 1 ]; then
+				copy_to_remote "clobber_mode_layout.txt"
+			fi
+			secondary_opts="$opts -l ${LOCAL_TMPDIR}/clobber_mode_layout.txt"
+			opts="$opts -l ${TMPDIR}/clobber_mode_layout.txt"
+		fi
+	fi
+
+	if [ $SMALL_REGION -eq 1 ]; then
+		PARTIAL_WRITE_TEST_ALIGN_SIZE_KB=16
+	else
+		PARTIAL_WRITE_TEST_ALIGN_SIZE_KB=256
+	fi
+
+	# FIXME: Add sanity checks.
+
+	print_and_log "Doing region-based partial write test on region "$region_name"\n"
+	flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -r ${TMPDIR}/${region_name}.bin" "read_region_${region_name}"
+	if [ $DO_REMOTE -eq 1 ]; then
+		copy_from_remote "${region_name}.bin"
+	fi
+
+	if [ $(($PARTIAL_WRITE_TEST_REGION_SIZE % $(($PARTIAL_WRITE_TEST_ALIGN_SIZE_KB)))) -ne 0 ]; then
+		print_and_log "Region $region_name is not aligned to $PARTIAL_WRITE_TEST_ALIGN_SIZE_KB\n"
+		return $EXIT_FAILURE
+	fi
+
+	# Test procedure:
+	# Clobber region with random content first. Then do writes using the
+	# following sequences for each 128KB:
+	# 0-2K		: 0x00 (\000)	Partial 4KB sector, lower half
+	# 2K-6K		: 0x11 (\021)	Crossover 4KB sector boundary
+	# 6K-8K		: 0x22 (\042)	Partial 4KB sector, upper half
+	# 8K-16K	: 0x33 (\063)	Full 4KB sectors
+	#
+	# Repeat the above sequence for 64KB-aligned sizes
+	# 0-32K		: 0x44 (\104)	Partial 64KB block, lower half
+	# 32K-96K	: 0x55 (\125)	Crossover 64KB block boundary
+	# 96K-128K	: 0x66 (\146)	Partial 64KB block, upper half
+	# 128K-256K	: 0x77 (\167)	Full 64KB blocks
+
+	test_num=0
+	dd if=/dev/zero of="${LOCAL_TMPDIR}/random_4k_test.bin" bs=1k count=$CHIP_SIZE_KB 2>/dev/null
+	dd if=/dev/urandom of="${LOCAL_TMPDIR}/random_4k_test.bin" bs=4k count=$(($PARTIAL_WRITE_TEST_REGION_SIZE / $((4 * $K)))) seek=$(($((PARTIAL_WRITE_TEST_REGION_BASE_OFFSET)) / $((4 * $K)))) conv=notrunc 2>/dev/null
+
+	# 0-2K		: 0x00 (\000)	Partial 4KB sector, lower half
+	partial_write_test_helper "4k_test_${test_num}" 0 2 "0x00"
+	prev_test_num=$test_num
+	test_num=$(($test_num + 1))
+
+	# 2K-6K		: 0x11 (\021)	Crossover 4KB sector boundary
+	partial_write_test_helper "4k_test_${test_num}" 2 4 "0x11"
+	test_num=$(($test_num + 1))
+
+	# 6K-8K		: 0x22 (\042)	Partial 4KB sector, upper half
+	partial_write_test_helper "4k_test_${test_num}" 6 2 "0x22"
+	test_num=$(($test_num + 1))
+
+	# 8K-16K	: 0x33 (\063)	Full 4KB sectors
+	partial_write_test_helper "4k_test_${test_num}" 8 8 "0x33"
+
+	for F in ${LOCAL_TMPDIR}/random_4k_test.bin ${LOCAL_TMPDIR}/4k_test_*.bin ; do
+		filename=$(basename $F)
+		if [ $DO_REMOTE -eq 1 ]; then
+			copy_to_remote $filename
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -w ${TMPDIR}/${filename}" "write_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to write $filename to $region_name"
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -v ${TMPDIR}/${filename}" "verify_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to verify write of $filename to $region_name"
+		fi
+
+		if [ -n "$SECONDARY_OPTS" ]; then
+			flashrom_log_scmd $LOCAL "$LOCAL_FLASHROM $SECONDARY_OPTS $secondary_opts -i ${region_name} -v ${LOCAL_TMPDIR}/${filename}" "verify_secondary_${filename}"
+			if [ $? -ne 0 ]; then
+				test_fail "Failed to verify write of $filename to $region_name using secondary programmer"
+			fi
+		fi
+
+		print_and_log "\tWrote $filename to $region_name region successfully.\n"
+	done
+
+	if [ $SMALL_REGION -eq 1 ]; then
+		return $EXIT_SUCCESS
+	fi
+
+	#
+	# Second half: Tests for 64KB chunks
+	#
+	test_num=0
+#	dd if=/dev/urandom of="${LOCAL_TMPDIR}/random_64k_test.bin" bs=128k count=$(($PARTIAL_WRITE_TEST_REGION_SIZE / $((128*$K)))) 2>/dev/null
+	dd if=/dev/zero of="${LOCAL_TMPDIR}/random_64k_test.bin" bs=1k count=$CHIP_SIZE_KB 2>/dev/null
+	dd if=/dev/urandom of="${LOCAL_TMPDIR}/random_64k_test.bin" bs=128k count=$(($PARTIAL_WRITE_TEST_REGION_SIZE / $((128 * $K)))) seek=$(($((PARTIAL_WRITE_TEST_REGION_BASE_OFFSET)) / $((128 * $K)))) conv=notrunc 2>/dev/null
+
+	# 0-32K		: 0x44 (\104)	Partial 64KB block, lower half
+	partial_write_test_helper "64k_test_${test_num}"  0 32 "0x44"
+	prev_test_num=$test_num
+	test_num=$(($test_num + 1))
+
+	# 32K-96K	: 0x55 (\125)	Crossover 64KB block boundary
+	partial_write_test_helper "64k_test_${test_num}" 32 64 "0x55"
+	test_num=$(($test_num + 1))
+
+	# 96K-128K	: 0x66 (\146)	Partial 64KB block, upper half
+	partial_write_test_helper "64k_test_${test_num}"  96 32 "0x66"
+	test_num=$(($test_num + 1))
+
+	# 128K-256K	: 0x77 (\167)	Full 64KB blocks
+	partial_write_test_helper "64k_test_${test_num}"  128 128 "0x77"
+
+	for F in ${LOCAL_TMPDIR}/random_64k_test.bin ${LOCAL_TMPDIR}/64k_test_*.bin ; do
+		filename=$(basename $F)
+		if [ $DO_REMOTE -eq 1 ]; then
+			copy_to_remote $filename
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -w ${TMPDIR}/${filename}" "write_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to write $filename to $region_name"
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -v ${TMPDIR}/${filename}" "verify_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to verify write of $filename to $region_name"
+		fi
+
+		if [ -n "$SECONDARY_OPTS" ]; then
+			flashrom_log_scmd $LOCAL "$LOCAL_FLASHROM $SECONDARY_OPTS $secondary_opts -i ${region_name} -v ${LOCAL_TMPDIR}/${filename}" "verify_secondary_${filename}"
+			if [ $? -ne 0 ]; then
+				test_fail "Failed to verify write of $filename to $region_name using secondary programmer"
+			fi
+		fi
+
+		print_and_log "\tWrote $filename to $region_name region successfully.\n"
+	done
+
+	return $EXIT_SUCCESS
+}
+
+# Before anything else, check to see if Flashrom can succesfully probe
+# for and find the flash chips. If not, we will abort.
+flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS" "verify_probe"
+if [ $? -ne 0 ]; then
+	test_fail "Failed to find flash chips while probing, aborting."
+fi
+
+# Read ROM twice to test for consistency.
+if [ $SKIP_CONSISTENCY_CHECK -eq 0 ]; then
+	double_read_test 0 $CHIP_SIZE
+fi
+
+if [ $TEST_TYPE -eq $TEST_TYPE_SINGLE ]; then
+	if [ $REGION_MODE -eq $REGION_MODE_CLOBBER ]; then
+		random_file="${TMPDIR}/random_${CHIP_SIZE_KB}K.bin"
+		cmp_file="${TMPDIR}/cmp.bin"
+
+		scmd $DO_REMOTE "dd if=/dev/urandom of=${random_file} bs=1k count=${CHIP_SIZE_KB}"
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -w $random_file" "clobber_write"
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -r $cmp_file" "clobber_verify"
+		scmd $DO_REMOTE "cmp $random_file $cmp_file"
+		if [ $? -ne 0 ]; then
+			write_backup_image
+			test_fail "Failed to clobber entire ROM."
+		fi
+	elif [ $REGION_MODE -eq $REGION_MODE_DESCRIPTOR ]; then
+		# FIXME: Need region-sized file handling or some way to get region info
+		print_and_log "Currently broken due to lack of region-sized file handling."
+		exit $EXIT_FAILURE
+	elif [ $REGION_MODE -eq $REGION_MODE_FLASHMAP ]; then
+		# FIXME: Need region-sized file handling or some way to get region info
+		print_and_log "Currently broken due to lack of region-sized file handling."
+		exit $EXIT_FAILURE
+	elif [ $REGION_MODE -eq $REGION_MODE_LAYOUT ]; then
+		rw_layout=""
+		addr=""
+		end=""
+		size=""
+		size_kb=""
+
+		# Look for a region name with any amount of leading whitespace
+		# and no trailing whitespace or characters.
+		rw_layout=$(grep "\s${LAYOUT_REGION}$" "${LOCAL_TMPDIR}/$(basename $LAYOUT_FILE)" | head -n 1)
+		if [ -z "$rw_layout" ]; then
+			print_and_log "No region matching "${LAYOUT_REGION}" found layout file "${LAYOUT_FILE}"\n"
+			test_fail ""
+		fi
+
+		addr="0x$(echo "$rw_layout" | cut -d ' ' -f -1 | awk -F ':' '{ print $1 }')"
+		end="0x$(echo "$rw_layout" | cut -d ' ' -f -1 | awk -F ':' '{ print $2 }')"
+		size="$(($end - $addr + 1))"
+		size_kb="$(($size / $K))"
+
+		# FIXME: Hack to make this work without region-sized file handling.
+		PARTIAL_WRITE_TEST_REGION_BASE_OFFSET=$addr
+		PARTIAL_WRITE_TEST_REGION_SIZE=$size
+
+		print_and_log ""$LAYOUT_REGION" region address: ${addr}, size: $size_kb KiB\n"
+		partial_write_test "$LAYOUT_REGION"
+		if [ $? -ne 0 ]; then
+			print_and_log "Layout mode test failed\n"
+			RC=$EXIT_FAILURE
+		fi
+	fi
+elif [ $TEST_TYPE -eq $TEST_TYPE_ENDURANCE ]; then
+	iteration=1
+	terminate=0
+	random_file="${TMPDIR}/random_${CHIP_SIZE_KB}K.bin"
+	cmp_file="${TMPDIR}/cmp.bin"
+	# TODO: We can measure how long the tests take on average, throughput, etc.
+	# i.e. { time $NEW_FLASHROM $PRIMARY_OPTS -w $random_file ; } 2>&1 | grep user | cut -f2
+
+	# For this test we want to run clobber mode until failure
+	while [ $terminate -eq 0 ]
+	do
+		print_and_log "Running iteration #${iteration}\n"
+
+		scmd $DO_REMOTE "dd if=/dev/urandom of=${random_file} bs=1k count=${CHIP_SIZE_KB}"
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -w $random_file" "endurance_write_${iteration}"
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -r $cmp_file" "endurance_verify_${iteration}"
+		scmd $DO_REMOTE "cmp $random_file $cmp_file"
+		if [ $? -ne 0 ]; then
+			terminate=1
+		fi
+		scmd $DO_REMOTE "rm -f $cmp_file $random_file"
+
+		iteration=$(($iteration + 1))
+	done
+
+	# TODO: Determine what to return for the endurance test exit status
+	# i.e. what constitutes a test pass and what constitutes a test fail?
+	print_and_log "Failed on iteration $iteration\n"
+	# TODO - Print performance metrics?
+fi
+
+# restore and cleanup
+write_backup_image
+
+if [ $RC -eq 0 ]; then
+	print_and_log "Test status: PASS\n"
+else
+	print_and_log "Test status: FAIL\n"
+fi
+do_cleanup
+
+exit $RC
-- 
To view, visit https://review.coreboot.org/c/flashrom/+/38788
To unsubscribe, or for help writing mail filters, visit https://review.coreboot.org/settings

Gerrit-Project: flashrom
Gerrit-Branch: master
Gerrit-Change-Id: I1af55d5088c54ee33853009797adbd535a506b49
Gerrit-Change-Number: 38788
Gerrit-PatchSet: 1
Gerrit-Owner: David Hendricks david.hendricks@gmail.com
Gerrit-MessageType: newchange



    

Change in flashrom[master]: testing: Add blackbox test uber-script