New subject: Change in flashrom[master]: ubertest: Add blackbox test uber-script

Feb. 9, 2020

David Hendricks has uploaded this change for review. ( https://review.coreboot.org/c/flashrom/+/38788 )


Change subject: testing: Add blackbox test uber-script
......................................................................

testing: Add blackbox test uber-script

This modifies CB:23025 further to work with upstream as it is now,
without the syntax changes in the patch chain. I also gave it a new
name since this script is, well, uber.

Since flashrom currently only supports reading/writing ROM-sized files
we can't easily determine a targeted region offset and size except
when a layout file is used. Therefore, some extra arithmetic is needed
in the partial write test and the only modes allowed are clobber mode
and layout mode.

A few other changes:
- Update paths and script name
- Remove write-protect testing support
- Use ROM-sized files only, no region-sized files
- Return error if flashmap or ifd mode are used

Tested using a Raspberry Pi with a W25Q16

Change-Id: I1af55d5088c54ee33853009797adbd535a506b49
Signed-off-by: David Hendricks <david.hendricks@gmail.com>
---
A util/testing/cmd.sh
A util/testing/ubertest.sh
2 files changed, 1,074 insertions(+), 0 deletions(-)



  git pull ssh://review.coreboot.org:29418/flashrom refs/changes/88/38788/1

diff --git a/util/testing/cmd.sh b/util/testing/cmd.sh
new file mode 100644
index 0000000..1d1c47a
--- /dev/null
+++ b/util/testing/cmd.sh
@@ -0,0 +1,137 @@
+#!/bin/sh
+#
+# This file is part of the flashrom project. It is derived from
+# board_status.sh in coreboot.
+#
+# Copyright (C) 2016 Google Inc.
+# Copyright (C) 2014 Sage Electronic Engineering, LLC.
+
+USE_CUSTOM_HOOKS=0
+if [ -n "$CUSTOM_HOOKS_FILENAME" ]; then
+	USE_CUSTOM_HOOKS=1
+	. "$CUSTOM_HOOKS_FILENAME"
+	if [ $? -ne 0 ]; then
+		echo "Failed to source custom hooks file."
+		exit $EXIT_FAILURE
+	fi
+
+	if ! custom_hook_sanity_check; then
+		echo "Failed to run sanity check for custom hooks."
+		exit $EXIT_FAILURE
+	fi
+fi
+
+# test a command
+#
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: command to test
+# $3: 0 ($FATAL) Exit with an error if the command fails
+#     1 ($NONFATAL) Don't exit on command test failure
+test_cmd()
+{
+	local rc
+	local cmd__="$(echo $2 | cut -d ' ' -f -1)"
+	local args="$(echo $2 | cut -d ' ' -f 2-)"
+
+	if [ -e "$cmd__" ]; then
+		return
+	fi
+
+	if [ "$1" -eq "$REMOTE" ] && [ -n "$REMOTE_HOST" ]; then
+		ssh $REMOTE_PORT_OPTION root@${REMOTE_HOST} command -v "$cmd__" $args > /dev/null 2>&1
+		rc=$?
+	else
+		command -v "$cmd__" $args >/dev/null 2>&1
+		rc=$?
+	fi
+
+	if [ $rc -eq 0 ]; then
+		return 0
+	fi
+
+	if [ "$3" = "1" ]; then
+		return 1
+	fi
+
+	echo "$2 not found"
+	exit $EXIT_FAILURE
+}
+
+# Same args as cmd() but with the addition of $4 which determines if the
+# command should be totally silenced or not.
+_cmd()
+{
+	local silent=$4
+	local rc=0
+
+	if [ -n "$3" ]; then
+		pipe_location="${3}"
+	else
+		pipe_location="/dev/null"
+	fi
+
+	if [ $1 -eq $REMOTE ] && [ -n "$REMOTE_HOST" ]; then
+		if [ $silent -eq 0 ]; then
+			ssh $REMOTE_PORT_OPTION "root@${REMOTE_HOST}" "$2" > "$pipe_location" 2>/dev/null
+			rc=$?
+		else
+			ssh $REMOTE_PORT_OPTION "root@${REMOTE_HOST}" "$2" >/dev/null 2>&1
+			rc=$?
+		fi
+	else
+		if [ $USE_CUSTOM_HOOKS -eq 1 ]; then
+			preflash_hook $1 "$2" "$3" $4
+		fi
+
+		if [ $silent -eq 0 ]; then
+			$SUDO_CMD $2 > "$pipe_location" 2>/dev/null
+			rc=$?
+		else
+			$SUDO_CMD $2 >/dev/null 2>&1
+			rc=$?
+		fi
+
+		if [ $USE_CUSTOM_HOOKS -eq 1 ]; then
+			postflash_hook $1 "$2" "$3" $4
+		fi
+	fi
+
+	return $rc
+}
+
+# run a command
+#
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: command
+# $3: filename to direct output of command into
+cmd()
+{
+	_cmd $1 "$2" "$3" 0
+
+	if [ $? -eq 0 ]; then
+		return
+	fi
+
+	echo "Failed to run \"$2\", aborting"
+	rm -f "$3"	# don't leave an empty file
+	return $EXIT_FAILURE
+}
+
+# run a command silently
+#
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: command
+scmd()
+{
+	_cmd $1 "$2" "" 1
+
+	if [ $? -eq 0 ]; then
+		return
+	fi
+
+	echo "Failed to run \"$2\", aborting"
+	return $EXIT_FAILURE
+}
diff --git a/util/testing/ubertest.sh b/util/testing/ubertest.sh
new file mode 100755
index 0000000..f51aa05
--- /dev/null
+++ b/util/testing/ubertest.sh
@@ -0,0 +1,937 @@
+#!/bin/sh
+#
+# Copyright (C) 2016 Google Inc.
+# Copyright (C) 2020 Facebook Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+#
+# Documentation for this script currently resides at: https://goo.gl/qNwdmm
+
+EXIT_SUCCESS=0
+EXIT_FAILURE=1
+RC=$EXIT_SUCCESS
+FATAL=0
+NONFATAL=1
+
+#
+# Stuff obtained from command-line
+#
+
+# Generic options
+BACKUP_IMAGE=""
+OLD_FLASHROM=""
+NEW_FLASHROM=""
+NO_CLEAN=0
+SKIP_CONSISTENCY_CHECK=0
+UPLOAD_RESULTS=0
+
+# LOCAL_FLASHROM is required if both a secondary programmer *and* a remote host
+# are used since OLD_FLASHROM and NEW_FLASHROM will be called on the remote
+# host and we need a local copy of flashrom to control the secondary programmer.
+# By default this will be set to the result of `which flashrom`.
+LOCAL_FLASHROM=""
+
+# Primary/Secondary programmer options
+PRIMARY_OPTS=""
+SECONDARY_OPTS=""
+
+# Calls preflash_hook() and postflash_hook() before and after doing a command.
+CUSTOM_HOOKS_FILENAME=""
+
+# logfile to store the script's output
+SCRIPT_LOGFILE="flashrom-test_script_output.txt"
+
+# Test type
+TEST_TYPE_UNKNOWN=0
+TEST_TYPE_SINGLE=1
+TEST_TYPE_ENDURANCE=2
+TEST_TYPE=$TEST_TYPE_UNKNOWN
+
+# Region modes
+REGION_MODE_UNKNOWN=0
+REGION_MODE_CLOBBER=1
+REGION_MODE_DESCRIPTOR=2
+REGION_MODE_FLASHMAP=3
+REGION_MODE_LAYOUT=4
+REGION_MODE=$REGION_MODE_UNKNOWN
+DESCRIPTOR_REGION="BIOS"
+FLASHMAP_REGION="RW_SECTION_A"
+LAYOUT_FILE=""
+LAYOUT_REGION="RW"
+SMALL_REGION=0
+
+# Remote testing options
+SSH_PORT=""
+REMOTE_HOST=""
+REMOTE_PORT_OPTION=""
+REMOTE_ONLY=0
+REMOTE_PROGRAMMER_PARAMS=""
+SSH_CMD="ssh $REMOTE_PORT_OPTION root@${REMOTE_HOST} command -v"
+
+LOCAL=0
+REMOTE=1
+DO_REMOTE=0	# boolean to use for cmd() and tests
+
+# relative path from flashrom root directory
+TEST_SCRIPT_PATH="util/testing/test_v2.1.sh"
+
+# In case we need to run flashrom locally and we're not already root.
+SUDO_CMD=""
+if [ "$(id -u)" -ne "0" ]; then
+	SUDO_CMD="sudo"
+fi
+
+# 1KB
+K=1024
+
+show_help() {
+	printf "Usage:
+	${0} <options>
+
+General options:
+    -b, --backup-image <path>
+        Backup image to write unconditionally at end of testing.
+    -h, --help
+        Show this message.
+    -l, --layout-file <path>
+        Layout file (required if mode is \"layout\", resides locally).
+    -m, --mode <arg>
+        Region access mode (clobber, descriptor, flashmap, layout).
+    -n, --new <path>
+        Path to new version of flashrom.
+    -o, --old <path>
+        Path to old (stable) version of flashrom.
+    -p, --primary-programmer <parameters>
+        Primary programmer options.
+    -r, --remote-host <host>
+        Remote host to test primary programmer on.
+    -s, --secondary-programmer <parameters>
+        Secondary programmer options.
+    -t, --type <arg>
+        Test type (single, endurance).
+    -u, --upload-results
+        Upload results to flashrom.org.
+    -v, --voltage
+        Chip voltage in millivolts (usually 1800 or 3300).
+
+Long options:
+    --custom-hooks <filename>
+        Supply a script with custom hooks to run before and after commands.
+    --descriptor-region <name>
+        Specify region to use in descriptor mode (default: $DESCRIPTOR_REGION)
+    --flashmap-region <name>
+        Specify region to use in flashmap mode (default: $FLASHMAP_REGION)
+    --layout-region <name>
+        Specify region to use in layout mode (default: $LAYOUT_REGION)
+    --local-flashrom <path>
+        Path to local version of flashrom when using both a secondary programmer
+        and remote host (default: $($SUDO_CMD which flashrom))
+    --no-clean
+        Do not remove temporary files.
+    --skip-consistency-check
+        Skip the consistency check (two consecutive reads) at beginning.
+    --small-region
+        Omit tests that require large amounts of space (>16KB).
+Remote connectivity options:
+    --ssh-port <port>
+        Use a specific SSH port.
+
+See documentation for usage examples (TODO: Migrate https://goo.gl/3jNoL7
+to flashrom wiki).\n
+"
+}
+
+getopt -T
+if [ $? -ne 4 ]; then
+	printf "GNU-compatible getopt(1) required.\n"
+	exit $EXIT_FAILURE
+fi
+
+LONGOPTS="backup-image:,help,,new:,old:,remote-host:,upload-results:"
+LONGOPTS="${LONGOPTS},primary-programmer:,secondary-programmer:,local-flashrom:"
+LONGOPTS="${LONGOPTS},custom-hooks:,mode:,skip-consistency-check,small-region"
+LONGOPTS="${LONGOPTS},type:,voltage:"
+LONGOPTS="${LONGOPTS},layout-file:,descriptor-region:,flashmap-region:,layout-region:"
+LONGOPTS="${LONGOPTS},no-clean"
+LONGOPTS="${LONGOPTS},ssh-port:"
+
+ARGS=$(getopt -o b:hl:m:n:o:p:r:s:t:u -l "$LONGOPTS" -n "$0" -- "$@");
+if [ $? != 0 ] ; then printf "Terminating...\n" >&2 ; exit 1 ; fi
+eval set -- "$ARGS"
+while true ; do
+	case "$1" in
+		# Generic options
+		-b|--backup-image)
+			shift
+			BACKUP_IMAGE="$1"
+			;;
+		-h|--help)
+			show_help
+			exit $EXIT_SUCCESS
+			;;
+		-l|--layout-file)
+			shift
+			LAYOUT_FILE="$1"
+			;;
+		-m|--mode)
+			shift
+			if [ "$1" = "clobber" ]; then
+				REGION_MODE=$REGION_MODE_CLOBBER
+			elif [ "$1" = "descriptor" ]; then
+				REGION_MODE=$REGION_MODE_DESCRIPTOR
+			elif [ "$1" = "flashmap" ]; then
+				REGION_MODE=$REGION_MODE_FLASHMAP
+			elif [ "$1" = "layout" ]; then
+				REGION_MODE=$REGION_MODE_LAYOUT
+			else
+				printf "Unknown mode: $1\n"
+				exit $EXIT_FAILURE
+			fi
+			;;
+		-n|--new)
+			shift
+			NEW_FLASHROM="$1"
+			;;
+		-o|--old)
+			shift
+			OLD_FLASHROM="$1"
+			;;
+		-p|--primary_programmer)
+			shift
+			PRIMARY_OPTS="-p $1"
+			;;
+		-s|--secondary_programmer)
+			shift
+			SECONDARY_OPTS="-p $1"
+			;;
+		-t|--type)
+			shift
+			if [ "$1" = "single" ]; then
+				TEST_TYPE=$TEST_TYPE_SINGLE
+			elif [ "$1" = "endurance" ]; then
+				TEST_TYPE=$TEST_TYPE_ENDURANCE
+			else
+				printf "Unknown type: $1\n"
+				exit $EXIT_FAILURE
+			fi
+			;;
+		-r|--remote-host)
+			DO_REMOTE=1
+			shift
+			REMOTE_HOST="$1"
+			;;
+		-u|--upload-results)
+			UPLOAD_RESULTS=1
+			;;
+		-v|--voltage)
+			shift
+			VOLTAGE="$1"
+			;;
+
+		# Longopts only
+		--custom-hooks)
+			shift
+			CUSTOM_HOOKS_FILENAME="$1"
+			;;
+		--descriptor-region)
+			shift
+			DESCRIPTOR_REGION="$1"
+			;;
+		--flashmap-region)
+			shift
+			FLASHMAP_REGION="$1"
+			;;
+		--layout-region)
+			shift
+			LAYOUT_REGION="$1"
+			;;
+		--local-flashrom)
+			shift
+			LOCAL_FLASHROM="$1"
+			;;
+		--no-clean)
+			NO_CLEAN=1
+			;;
+		--skip-consistency-check)
+			SKIP_CONSISTENCY_CHECK=1
+			;;
+		--small-region)
+			SMALL_REGION=1
+			;;
+
+		# Remote testing options
+		--ssh-port)
+			shift
+			REMOTE_PORT_OPTION="-p $1"
+			;;
+
+		# error handling
+		--)
+			shift
+			if [ -n "$*" ]; then
+				printf "Non-option parameters detected: '$*'\n"
+				exit $EXIT_FAILURE
+			fi
+			break
+			;;
+		*)
+			printf "error processing options at '$1'\n"
+			exit $EXIT_FAILURE
+	esac
+	shift
+done
+
+# TODO: Implement this.
+if [ $UPLOAD_RESULTS -eq 1 ]; then
+	printf "TODO: Implement ability to upload results.\n"
+	exit $EXIT_FAILURE
+fi
+
+#
+# Source helper scripts
+#
+export REMOTE_HOST REMOTE_PORT_OPTION
+export LOCAL REMOTE FATAL NONFATAL EXIT_SUCCESS EXIT_FAILURE
+export CUSTOM_HOOKS_FILENAME SUDO_CMD VOLTAGE
+. "$(pwd)/util/testing/cmd.sh"
+
+# We will set up a logs directory within the tmpdirs to store
+# all output logs.
+LOGS="logs"
+
+# Setup temporary working directories:
+# LOCAL_TMPDIR:  Working directory on local host.
+# REMOTE_TMPDIR: Working directory on remote host.
+# TMPDIR:        The temporary directy in which we do most of the work. This is
+#                convenient for commands that depend on $DO_REMOTE.
+LOCAL_TMPDIR=$(mktemp -d --tmpdir flashrom_test.XXXXXXXX)
+if [ $? -ne 0 ] ; then
+	printf "Could not create temporary directory\n"
+	exit $EXIT_FAILURE
+fi
+mkdir "${LOCAL_TMPDIR}/${LOGS}"
+
+if [ $DO_REMOTE -eq 1 ]; then
+	REMOTE_TMPDIR=$(ssh root@${REMOTE_HOST} mktemp -d --tmpdir flashrom_test.XXXXXXXX)
+	if [ $? -ne 0 ] ; then
+		printf "Could not create temporary directory\n"
+		exit $EXIT_FAILURE
+	fi
+	scmd $REMOTE "mkdir ${REMOTE_TMPDIR}/${LOGS}"
+fi
+
+if [ $DO_REMOTE -eq 0 ]; then
+	TMPDIR="$LOCAL_TMPDIR"
+else
+	TMPDIR="$REMOTE_TMPDIR"
+fi
+
+#
+# Test command-line validity.
+#
+if [ $TEST_TYPE -eq $TEST_TYPE_UNKNOWN ]; then
+	printf "Must specify a test type (-t/--type).\n"
+	exit $EXIT_FAILURE
+elif [ $TEST_TYPE -eq $TEST_TYPE_SINGLE ]; then
+	if [ $REGION_MODE -eq $REGION_MODE_UNKNOWN ]; then
+		printf "Must specify a region access mode (-m/--mode).\n"
+		exit $EXIT_FAILURE
+	elif [ $REGION_MODE -eq $REGION_MODE_LAYOUT ]; then
+		if [ -z "$LAYOUT_FILE" ]; then
+			printf "Must specify a layout file when using layout mode.\n"
+			exit $EXIT_FAILURE
+		fi
+
+		scmd $DO_REMOTE "stat $LAYOUT_FILE"
+		if [ $? -ne 0 ]; then
+			if [ $DO_REMOTE -eq 1 ]; then
+				tmp=" on remote host $REMOTE_HOST."
+			else
+				tmp=" on local host."
+			fi
+			printf "Layout file $LAYOUT_FILE not found${tmp}\n"
+			exit $EXIT_FAILURE
+		fi
+
+		if [ $DO_REMOTE -eq 1 ]; then
+			scp root@"${REMOTE_HOST}:$LAYOUT_FILE" "${LOCAL_TMPDIR}/" 2>&1 >/dev/null
+		else
+			cp "$LAYOUT_FILE" "${LOCAL_TMPDIR}/"
+		fi
+	fi
+fi
+
+if [ -n "$VOLTAGE" ]; then
+	echo "$VOLTAGE" | grep -q '[^0-9]'
+	if [ $? -ne 1 ]; then
+		printf "Voltage must be an integer with units of millivolts."
+		exit $EXIT_FAILURE
+	fi
+fi
+
+if [ $DO_REMOTE -eq 1 ]; then
+	# Test connection to remote host
+	test_cmd $DO_REMOTE "ls /" $NONFATAL
+	if [ $? -ne 0 ]; then
+		printf "Could not connect to remote host ${REMOTE_HOST}\n"
+		exit $EXIT_FAILURE
+	fi
+fi
+
+# Remote host and secondary programmer are in use, so either the user must
+# specify a local version of flashrom to control the secondary programmer
+# or it must be found in the default path.
+if [ $DO_REMOTE -eq 1 ] && [ -n "$SECONDARY_OPTS" ]; then
+	if [ -z "$LOCAL_FLASHROM" ]; then
+		LOCAL_FLASHROM="$($SUDO_CMD which flashrom)"
+	fi
+
+	if [ ! -e "$LOCAL_FLASHROM" ]; then
+		printf "$LOCAL_FLASHROM does not exist\n"
+		exit $EXIT_FAILURE
+	fi
+fi
+
+#
+# Dependencies
+#
+
+# cmp is used to compare files
+test_cmd $DO_REMOTE "cmp" $FATAL
+
+if [ ! -e "/dev/urandom" ]; then
+	printf "This script uses /dev/urandom\n"
+	exit $EXIT_FAILURE
+fi
+
+if [ ! -e "/dev/zero" ]; then
+	printf "This script uses /dev/zero\n"
+	exit $EXIT_FAILURE
+fi
+
+#
+# Setup.
+#
+
+# Naive path check
+if [ ! -e "$TEST_SCRIPT_PATH" ] ; then
+	printf "Script must be run from root of flashrom directory\n"
+	exit $EXIT_FAILURE
+fi
+
+if [ -z "$OLD_FLASHROM" ]; then
+	if [ $DO_REMOTE -eq 1 ]; then
+		OLD_FLASHROM="$(ssh root@${REMOTE_HOST} which flashrom)"
+	else
+		OLD_FLASHROM="$($SUDO_CMD which flashrom)"
+	fi
+fi
+test_cmd $DO_REMOTE "$OLD_FLASHROM --help" $NONFATAL
+if [ $? -ne 0 ]; then
+	printf "Old flashrom binary is not usable.\n"
+	exit $EXIT_FAILURE
+fi
+
+# print $1 and store it in the script log file
+print_and_log()
+{
+	printf "$1" | tee -a "${LOCAL_TMPDIR}/${LOGS}/${SCRIPT_LOGFILE}"
+}
+
+# Copy files from local tmpdir to remote host tmpdir
+copy_to_remote()
+{
+	for F in $@; do
+		scp "${LOCAL_TMPDIR}/${F}" root@"${REMOTE_HOST}:${REMOTE_TMPDIR}" 2>&1 >/dev/null
+	done
+}
+
+# Copy files from remote host tmpdir to local tmpdir
+copy_from_remote()
+{
+	for F in $@; do
+		scp root@"${REMOTE_HOST}:${REMOTE_TMPDIR}/${F}" "${LOCAL_TMPDIR}/" 2>&1 >/dev/null
+	done
+}
+
+# A wrapper for scmd calls to flashrom when we want to log the output
+# $1: 0 ($LOCAL) to run command locally,
+#     1 ($REMOTE) to run remotely if remote host defined
+# $2: arguments to be passed into scmd
+# $3: context of the flashrom call (to be used in the logfile)
+flashrom_log_scmd()
+{
+	local logfile="flashrom-${3}.txt"
+	local rc
+
+	if [ $1 -eq $REMOTE ]; then
+		tmpdir=$REMOTE_TMPDIR
+	else
+		tmpdir=$LOCAL_TMPDIR
+	fi
+
+	scmd $1 "$2 -o ${tmpdir}/${LOGS}/${logfile}"; rc=$?
+	# if the call was successful, we don't want to save the log (only save failure logs)
+	if [ $rc -eq $EXIT_SUCCESS ]; then
+		scmd $1 "rm -f ${tmpdir}/${LOGS}/${logfile}"
+	else
+		# if the log was stored remotely, we want to copy it over to local tmpdir
+		if [ $1 -eq $REMOTE ]; then
+			scp root@"${REMOTE_HOST}:${REMOTE_TMPDIR}/${LOGS}/${logfile}" "${LOCAL_TMPDIR}/${LOGS}" 2>&1 >/dev/null
+		fi
+	fi
+
+	return $rc
+}
+
+# Read current image as backup in case one hasn't already been specified.
+if [ -z "$BACKUP_IMAGE" ]; then
+	backup_file="backup.bin"
+	if [ $DO_REMOTE -eq 1 ]; then
+		BACKUP_IMAGE="${REMOTE_TMPDIR}/${backup_file}"
+	else
+		BACKUP_IMAGE="${LOCAL_TMPDIR}/${backup_file}"
+	fi
+
+	print_and_log "Reading backup image..."
+	flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -r $BACKUP_IMAGE" "read_backup"
+
+	if [ $? -ne 0 ]; then
+		print_and_log "Failed to read backup image, aborting.\n"
+		exit $EXIT_FAILURE
+	fi
+
+	if [ $DO_REMOTE -eq 1 ]; then
+		copy_from_remote "$backup_file"
+	fi
+else
+	if [ $DO_REMOTE -eq 1 ]; then
+		scmd $DO_REMOTE "cp $BACKUP_IMAGE $REMOTE_TMPDIR"
+		copy_from_remote "$(basename $BACKUP_IMAGE)"
+	fi
+fi
+
+# The copy of flashrom to test. If unset, we'll assume the user wants to test
+# a newly built flashrom binary in the current directory.
+if [ -z "$NEW_FLASHROM" ] ; then
+	if [ -x "flashrom" ]; then
+		NEW_FLASHROM="flashrom"
+	else
+		print_and_log "Must supply new flashrom version to test\n"
+		exit $EXIT_FAILURE
+	fi
+fi
+
+print_and_log "Stable flashrom binary: ${OLD_FLASHROM}\n"
+print_and_log "New flashrom binary to test: ${NEW_FLASHROM}\n"
+print_and_log "Local temporary files will be stored in ${LOCAL_TMPDIR}\n"
+if [ $DO_REMOTE -eq 1 ]; then
+	print_and_log "Remote temporary files will be stored in ${REMOTE_HOST}:${REMOTE_TMPDIR}\n"
+	print_and_log "Backup image: ${REMOTE_HOST}:${BACKUP_IMAGE}\n"
+	print_and_log "Backup image also stored at: ${LOCAL_TMPDIR}/$(basename ${BACKUP_IMAGE})\n"
+else
+	print_and_log "Backup image: ${BACKUP_IMAGE}\n"
+fi
+
+#
+# Now the fun begins.
+#
+cmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS --flash-size" "${LOCAL_TMPDIR}/old_chip_size.txt.orig"
+cmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS --flash-size" "${LOCAL_TMPDIR}/new_chip_size.txt.orig"
+# get rid of banner and other superfluous output from flashrom
+tail -n 1 "${LOCAL_TMPDIR}/old_chip_size.txt.orig" > "${LOCAL_TMPDIR}/old_chip_size.txt"
+tail -n 1 "${LOCAL_TMPDIR}/new_chip_size.txt.orig" > "${LOCAL_TMPDIR}/new_chip_size.txt"
+tmp=$(cat ${LOCAL_TMPDIR}/old_chip_size.txt)
+CHIP_SIZE=$(cat ${LOCAL_TMPDIR}/new_chip_size.txt)
+CHIP_SIZE_KB=$(($CHIP_SIZE / $K))
+CHIP_SIZE_HALF=$(($CHIP_SIZE / 2))
+if [ $CHIP_SIZE -ne $tmp ]; then
+	print_and_log "New flashrom and old flashrom disagree on chip size. Aborting.\n"
+	exit $EXIT_FAILURE
+else
+	print_and_log "Chip size: $CHIP_SIZE_KB KiB\n"
+fi
+
+# Upload results
+#do_upload()
+#{
+#	# TODO: implement this
+#}
+
+# Remove temporary files
+do_cleanup()
+{
+	if [ $NO_CLEAN -eq 1 ]; then
+		print_and_log "Skipping cleanup.\n"
+		return $EXIT_SUCCESS
+	fi
+
+	rm -rf "$LOCAL_TMPDIR"
+	if [ -n "$REMOTE_HOST" ]; then
+		ssh root@${REMOTE_HOST} rm -rf "$REMOTE_TMPDIR"
+	fi
+
+	return $EXIT_SUCCESS
+}
+
+# $1: Message to display to user.
+test_fail()
+{
+	print_and_log "$1\n"
+	printf "Skipping cleanup (logs saved).\n"
+	exit $EXIT_FAILURE
+}
+
+write_backup_image()
+{
+	print_and_log "Writing backup image.\n"
+	flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -w $BACKUP_IMAGE" "write_backup"
+	if [ $? -ne 0 ]; then
+		test_fail "Failed to write backup image.\n"
+	fi
+}
+
+# Read a region twice and compare results
+# $1: address of region (in bytes)
+# $2: length of region (in bytes)
+double_read_test()
+{
+	local cmp1="${TMPDIR}/read_test1.bin"
+	local cmp2="${TMPDIR}/read_test2.bin"
+	local layout="double_read_test_layout.txt"
+	local len=$(($2 / $K))
+
+	print_and_log "Doing double read test, size: $len KiB\n"
+	# FIXME: Figure out how to do printf remotely...
+	printf "%06x:%06x region\n" $1 $(($1 + $2 - 1)) > "${LOCAL_TMPDIR}/${layout}"
+	if [ $DO_REMOTE -eq 1 ]; then copy_to_remote "$layout" ; fi
+
+	flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -l ${TMPDIR}/${layout} -i region -r ${cmp1}" "double_read_1"
+	# FIXME: second (or maybe third?) read should be done using secondary programmer, if applicable.
+	flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -l ${TMPDIR}/${layout} -i region -r ${cmp2}" "double_read_2"
+	scmd $DO_REMOTE "cmp $cmp1 $cmp2"
+	if [ $? -ne 0 ]; then
+		test_fail "Double-read test failed, aborting."
+	fi
+}
+
+PARTIAL_WRITE_TEST_REGION_SIZE=0
+PARTIAL_WRITE_TEST_ALIGN_SIZE_KB=0
+# FIXME: Hack due to lack of region-sized file handling.
+PARTIAL_WRITE_TEST_REGION_BASE_OFFSET=-1
+
+# helper function to reduce repetitiveness of partial_write_test
+partial_write_test_helper()
+{
+	local test_name="$1"
+	local pattern_offset_kb=$2
+	local pattern_size_kb=$3
+	local hex="$4"
+	local oct=""
+	local base_offset_kb=$(($PARTIAL_WRITE_TEST_REGION_BASE_OFFSET / $K))
+
+	oct="\\$(printf "%03o" $hex)"
+	cp "${LOCAL_TMPDIR}/random_4k_test.bin" "${LOCAL_TMPDIR}/${test_name}.bin"
+	dd if=/dev/zero bs=1k count=${pattern_size_kb} 2>/dev/null | tr "\000" "$oct" > "${LOCAL_TMPDIR}/${hex}_${pattern_size_kb}k.bin"
+
+	while [ $(($(($pattern_offset_kb + $pattern_size_kb)) * $K)) -lt $PARTIAL_WRITE_TEST_REGION_SIZE ]; do
+		dd if="${LOCAL_TMPDIR}/${hex}_${pattern_size_kb}k.bin" of="${LOCAL_TMPDIR}/${test_name}.bin" bs=1k count=$pattern_size_kb seek=$(($base_offset_kb + $pattern_offset_kb)) conv=notrunc 2>/dev/null
+		pattern_offset_kb=$(($pattern_offset_kb + $PARTIAL_WRITE_TEST_ALIGN_SIZE_KB))
+	done
+}
+
+# Regional partial write test. Given a region name, this will write patterns
+# of bytes designed to test corner cases.
+#
+# We assume that eraseable block size can be either 4KB or 64KB and
+# must test for both. For simplicity, we'll assume the region size is
+# at least 256KB.
+#
+# $1: Region name
+partial_write_test()
+{
+	local opts="--noverify-all"
+	local secondary_opts=""		# for secondary programmer
+	local region_name="$1"
+	local filename=""
+	local test_num=0
+	local prev_test_num=0
+
+	# FIXME: Hack due to lack of region-sized file handling.
+	if [ $((PARTIAL_WRITE_TEST_REGION_SIZE)) -eq 0 ]; then
+		print_and_log "Size of $region_name unknown\n"
+		return $EXIT_FAILURE
+	fi
+	if [ $((PARTIAL_WRITE_TEST_REGION_BASE_OFFSET)) -lt 0 ]; then
+		print_and_log "Offset of $region_name is unknown\n"
+		return $EXIT_FAILURE
+	fi
+
+	if [ $TEST_TYPE -eq $TEST_TYPE_SINGLE ]; then
+		if [ $REGION_MODE -eq $REGION_MODE_LAYOUT ]; then
+			opts="$opts -l $LAYOUT_FILE"
+			secondary_opts="$opts"
+		elif [ $REGION_MODE -eq $REGION_MODE_CLOBBER ]; then
+			printf "000000:%06x RW\n" $(($CHIP_SIZE - 1)) > "${LOCAL_TMPDIR}/clobber_mode_layout.txt"
+			if [ $DO_REMOTE -eq 1 ]; then
+				copy_to_remote "clobber_mode_layout.txt"
+			fi
+			secondary_opts="$opts -l ${LOCAL_TMPDIR}/clobber_mode_layout.txt"
+			opts="$opts -l ${TMPDIR}/clobber_mode_layout.txt"
+		fi
+	fi
+
+	if [ $SMALL_REGION -eq 1 ]; then
+		PARTIAL_WRITE_TEST_ALIGN_SIZE_KB=16
+	else
+		PARTIAL_WRITE_TEST_ALIGN_SIZE_KB=256
+	fi
+
+	# FIXME: Add sanity checks.
+
+	print_and_log "Doing region-based partial write test on region \"$region_name\"\n"
+	flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -r ${TMPDIR}/${region_name}.bin" "read_region_${region_name}"
+	if [ $DO_REMOTE -eq 1 ]; then
+		copy_from_remote "${region_name}.bin"
+	fi
+
+	if [ $(($PARTIAL_WRITE_TEST_REGION_SIZE % $(($PARTIAL_WRITE_TEST_ALIGN_SIZE_KB)))) -ne 0 ]; then
+		print_and_log "Region $region_name is not aligned to $PARTIAL_WRITE_TEST_ALIGN_SIZE_KB\n"
+		return $EXIT_FAILURE
+	fi
+
+	# Test procedure:
+	# Clobber region with random content first. Then do writes using the
+	# following sequences for each 128KB:
+	# 0-2K		: 0x00 (\000)	Partial 4KB sector, lower half
+	# 2K-6K		: 0x11 (\021)	Crossover 4KB sector boundary
+	# 6K-8K		: 0x22 (\042)	Partial 4KB sector, upper half
+	# 8K-16K	: 0x33 (\063)	Full 4KB sectors
+	#
+	# Repeat the above sequence for 64KB-aligned sizes
+	# 0-32K		: 0x44 (\104)	Partial 64KB block, lower half
+	# 32K-96K	: 0x55 (\125)	Crossover 64KB block boundary
+	# 96K-128K	: 0x66 (\146)	Partial 64KB block, upper half
+	# 128K-256K	: 0x77 (\167)	Full 64KB blocks
+
+	test_num=0
+	dd if=/dev/zero of="${LOCAL_TMPDIR}/random_4k_test.bin" bs=1k count=$CHIP_SIZE_KB 2>/dev/null
+	dd if=/dev/urandom of="${LOCAL_TMPDIR}/random_4k_test.bin" bs=4k count=$(($PARTIAL_WRITE_TEST_REGION_SIZE / $((4 * $K)))) seek=$(($((PARTIAL_WRITE_TEST_REGION_BASE_OFFSET)) / $((4 * $K)))) conv=notrunc 2>/dev/null
+
+	# 0-2K		: 0x00 (\000)	Partial 4KB sector, lower half
+	partial_write_test_helper "4k_test_${test_num}" 0 2 "0x00"
+	prev_test_num=$test_num
+	test_num=$(($test_num + 1))
+
+	# 2K-6K		: 0x11 (\021)	Crossover 4KB sector boundary
+	partial_write_test_helper "4k_test_${test_num}" 2 4 "0x11"
+	test_num=$(($test_num + 1))
+
+	# 6K-8K		: 0x22 (\042)	Partial 4KB sector, upper half
+	partial_write_test_helper "4k_test_${test_num}" 6 2 "0x22"
+	test_num=$(($test_num + 1))
+
+	# 8K-16K	: 0x33 (\063)	Full 4KB sectors
+	partial_write_test_helper "4k_test_${test_num}" 8 8 "0x33"
+
+	for F in ${LOCAL_TMPDIR}/random_4k_test.bin ${LOCAL_TMPDIR}/4k_test_*.bin ; do
+		filename=$(basename $F)
+		if [ $DO_REMOTE -eq 1 ]; then
+			copy_to_remote $filename
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -w ${TMPDIR}/${filename}" "write_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to write $filename to $region_name"
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -v ${TMPDIR}/${filename}" "verify_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to verify write of $filename to $region_name"
+		fi
+
+		if [ -n "$SECONDARY_OPTS" ]; then
+			flashrom_log_scmd $LOCAL "$LOCAL_FLASHROM $SECONDARY_OPTS $secondary_opts -i ${region_name} -v ${LOCAL_TMPDIR}/${filename}" "verify_secondary_${filename}"
+			if [ $? -ne 0 ]; then
+				test_fail "Failed to verify write of $filename to $region_name using secondary programmer"
+			fi
+		fi
+
+		print_and_log "\tWrote $filename to $region_name region successfully.\n"
+	done
+
+	if [ $SMALL_REGION -eq 1 ]; then
+		return $EXIT_SUCCESS
+	fi
+
+	#
+	# Second half: Tests for 64KB chunks
+	#
+	test_num=0
+#	dd if=/dev/urandom of="${LOCAL_TMPDIR}/random_64k_test.bin" bs=128k count=$(($PARTIAL_WRITE_TEST_REGION_SIZE / $((128*$K)))) 2>/dev/null
+	dd if=/dev/zero of="${LOCAL_TMPDIR}/random_64k_test.bin" bs=1k count=$CHIP_SIZE_KB 2>/dev/null
+	dd if=/dev/urandom of="${LOCAL_TMPDIR}/random_64k_test.bin" bs=128k count=$(($PARTIAL_WRITE_TEST_REGION_SIZE / $((128 * $K)))) seek=$(($((PARTIAL_WRITE_TEST_REGION_BASE_OFFSET)) / $((128 * $K)))) conv=notrunc 2>/dev/null
+
+	# 0-32K		: 0x44 (\104)	Partial 64KB block, lower half
+	partial_write_test_helper "64k_test_${test_num}"  0 32 "0x44"
+	prev_test_num=$test_num
+	test_num=$(($test_num + 1))
+
+	# 32K-96K	: 0x55 (\125)	Crossover 64KB block boundary
+	partial_write_test_helper "64k_test_${test_num}" 32 64 "0x55"
+	test_num=$(($test_num + 1))
+
+	# 96K-128K	: 0x66 (\146)	Partial 64KB block, upper half
+	partial_write_test_helper "64k_test_${test_num}"  96 32 "0x66"
+	test_num=$(($test_num + 1))
+
+	# 128K-256K	: 0x77 (\167)	Full 64KB blocks
+	partial_write_test_helper "64k_test_${test_num}"  128 128 "0x77"
+
+	for F in ${LOCAL_TMPDIR}/random_64k_test.bin ${LOCAL_TMPDIR}/64k_test_*.bin ; do
+		filename=$(basename $F)
+		if [ $DO_REMOTE -eq 1 ]; then
+			copy_to_remote $filename
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -w ${TMPDIR}/${filename}" "write_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to write $filename to $region_name"
+		fi
+
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS $opts -i ${region_name} -v ${TMPDIR}/${filename}" "verify_${filename}"
+		if [ $? -ne 0 ]; then
+			test_fail "Failed to verify write of $filename to $region_name"
+		fi
+
+		if [ -n "$SECONDARY_OPTS" ]; then
+			flashrom_log_scmd $LOCAL "$LOCAL_FLASHROM $SECONDARY_OPTS $secondary_opts -i ${region_name} -v ${LOCAL_TMPDIR}/${filename}" "verify_secondary_${filename}"
+			if [ $? -ne 0 ]; then
+				test_fail "Failed to verify write of $filename to $region_name using secondary programmer"
+			fi
+		fi
+
+		print_and_log "\tWrote $filename to $region_name region successfully.\n"
+	done
+
+	return $EXIT_SUCCESS
+}
+
+# Before anything else, check to see if Flashrom can succesfully probe
+# for and find the flash chips. If not, we will abort.
+flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS" "verify_probe"
+if [ $? -ne 0 ]; then
+	test_fail "Failed to find flash chips while probing, aborting."
+fi
+
+# Read ROM twice to test for consistency.
+if [ $SKIP_CONSISTENCY_CHECK -eq 0 ]; then
+	double_read_test 0 $CHIP_SIZE
+fi
+
+if [ $TEST_TYPE -eq $TEST_TYPE_SINGLE ]; then
+	if [ $REGION_MODE -eq $REGION_MODE_CLOBBER ]; then
+		random_file="${TMPDIR}/random_${CHIP_SIZE_KB}K.bin"
+		cmp_file="${TMPDIR}/cmp.bin"
+
+		scmd $DO_REMOTE "dd if=/dev/urandom of=${random_file} bs=1k count=${CHIP_SIZE_KB}"
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -w $random_file" "clobber_write"
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -r $cmp_file" "clobber_verify"
+		scmd $DO_REMOTE "cmp $random_file $cmp_file"
+		if [ $? -ne 0 ]; then
+			write_backup_image
+			test_fail "Failed to clobber entire ROM."
+		fi
+	elif [ $REGION_MODE -eq $REGION_MODE_DESCRIPTOR ]; then
+		# FIXME: Need region-sized file handling or some way to get region info
+		print_and_log "Currently broken due to lack of region-sized file handling."
+		exit $EXIT_FAILURE
+	elif [ $REGION_MODE -eq $REGION_MODE_FLASHMAP ]; then
+		# FIXME: Need region-sized file handling or some way to get region info
+		print_and_log "Currently broken due to lack of region-sized file handling."
+		exit $EXIT_FAILURE
+	elif [ $REGION_MODE -eq $REGION_MODE_LAYOUT ]; then
+		rw_layout=""
+		addr=""
+		end=""
+		size=""
+		size_kb=""
+
+		# Look for a region name with any amount of leading whitespace
+		# and no trailing whitespace or characters.
+		rw_layout=$(grep "\s${LAYOUT_REGION}$" "${LOCAL_TMPDIR}/$(basename $LAYOUT_FILE)" | head -n 1)
+		if [ -z "$rw_layout" ]; then
+			print_and_log "No region matching \"${LAYOUT_REGION}\" found layout file \"${LAYOUT_FILE}\"\n"
+			test_fail ""
+		fi
+
+		addr="0x$(echo "$rw_layout" | cut -d ' ' -f -1 | awk -F ':' '{ print $1 }')"
+		end="0x$(echo "$rw_layout" | cut -d ' ' -f -1 | awk -F ':' '{ print $2 }')"
+		size="$(($end - $addr + 1))"
+		size_kb="$(($size / $K))"
+
+		# FIXME: Hack to make this work without region-sized file handling.
+		PARTIAL_WRITE_TEST_REGION_BASE_OFFSET=$addr
+		PARTIAL_WRITE_TEST_REGION_SIZE=$size
+
+		print_and_log "\"$LAYOUT_REGION\" region address: ${addr}, size: $size_kb KiB\n"
+		partial_write_test "$LAYOUT_REGION"
+		if [ $? -ne 0 ]; then
+			print_and_log "Layout mode test failed\n"
+			RC=$EXIT_FAILURE
+		fi
+	fi
+elif [ $TEST_TYPE -eq $TEST_TYPE_ENDURANCE ]; then
+	iteration=1
+	terminate=0
+	random_file="${TMPDIR}/random_${CHIP_SIZE_KB}K.bin"
+	cmp_file="${TMPDIR}/cmp.bin"
+	# TODO: We can measure how long the tests take on average, throughput, etc.
+	# i.e. { time $NEW_FLASHROM $PRIMARY_OPTS -w $random_file ; } 2>&1 | grep user | cut -f2
+
+	# For this test we want to run clobber mode until failure
+	while [ $terminate -eq 0 ]
+	do
+		print_and_log "Running iteration #${iteration}\n"
+
+		scmd $DO_REMOTE "dd if=/dev/urandom of=${random_file} bs=1k count=${CHIP_SIZE_KB}"
+		flashrom_log_scmd $DO_REMOTE "$NEW_FLASHROM $PRIMARY_OPTS -w $random_file" "endurance_write_${iteration}"
+		flashrom_log_scmd $DO_REMOTE "$OLD_FLASHROM $PRIMARY_OPTS -r $cmp_file" "endurance_verify_${iteration}"
+		scmd $DO_REMOTE "cmp $random_file $cmp_file"
+		if [ $? -ne 0 ]; then
+			terminate=1
+		fi
+		scmd $DO_REMOTE "rm -f $cmp_file $random_file"
+
+		iteration=$(($iteration + 1))
+	done
+
+	# TODO: Determine what to return for the endurance test exit status
+	# i.e. what constitutes a test pass and what constitutes a test fail?
+	print_and_log "Failed on iteration $iteration\n"
+	# TODO - Print performance metrics?
+fi
+
+# restore and cleanup
+write_backup_image
+
+if [ $RC -eq 0 ]; then
+	print_and_log "Test status: PASS\n"
+else
+	print_and_log "Test status: FAIL\n"
+fi
+do_cleanup
+
+exit $RC

-- 
To view, visit https://review.coreboot.org/c/flashrom/+/38788
To unsubscribe, or for help writing mail filters, visit https://review.coreboot.org/settings

Gerrit-Project: flashrom
Gerrit-Branch: master
Gerrit-Change-Id: I1af55d5088c54ee33853009797adbd535a506b49
Gerrit-Change-Number: 38788
Gerrit-PatchSet: 1
Gerrit-Owner: David Hendricks <david.hendricks@gmail.com>
Gerrit-MessageType: newchange

    

Change in flashrom[master]: testing: Add blackbox test uber-script

David Hendricks (Code Review)

Edward O'Callaghan (Code Review)

David Hendricks (Code Review)

David Hendricks (Code Review)

David Hendricks (Code Review)

Edward O'Callaghan (Code Review)

David Hendricks (Code Review)

tags

participants (2)