Author: stepan Date: 2006-10-14 22:34:46 +0200 (Sat, 14 Oct 2006) New Revision: 92
Added: fcode-utils/testsuite/AutoCompare fcode-utils/testsuite/AutoExec Log: add testsuite scripts
Added: fcode-utils/testsuite/AutoCompare =================================================================== --- fcode-utils/testsuite/AutoCompare (rev 0) +++ fcode-utils/testsuite/AutoCompare 2006-10-14 20:34:46 UTC (rev 92) @@ -0,0 +1,183 @@ +#! /bin/csh -f +# +# Part of automation of the testing process for the Tokenizer/De-Tokenizer +# Automate Comparison of results of Auto-Execution of the various test-cases + +# Updated Fri, 16 Jun 2006 at 12:12 PDT by David L. Paktor + +# To Do: +# Conditional comparison of .DeTok files: +# Detect difference in .fc file, but do not display +# Display differences in .Log file +# If difference detected in .fc file, display mssg +# Otherwise, detect difference in .DeTok file and display mssg +# If either diff detected in .fc or in .DeTok file, show .DeTok diffs +# Display differences in .RomHdr +# +# Compare the source (.fth) files (for regression testing...) + +# Usage: +# Run this from the parent-directory of the various Test categories. +# This script uses the same TestArgs files that were used by +# (and described in) the AutoExec script. +# +# This script needs a shell environment variable called CygTestLogs +# whose value is the path to the parent-directory that contains +# the results of Auto-Execution of the various test-cases under +# Cygwin, which is the "Gold Standard" to which the other +# platforms' results will be compared. +# +# This script accepts an optional environment variable called LogFilesDiff +# which may be set to change the number of lines by which two .Log +# files are expected to differ (due to different compilation-signatures) +# If this is not supplied in the environment, a default will be provided. +# +# This script compares only one platform's results at a time; +# the parent-directory that contains the Auto-Execution results +# for the platform to be compared should be the current directory. +# +# This script displays both operational errors and detected discrepancies +# in Standard Output and also collects the same in a log file whose +# name is AutoComp.Log.<time-and-date-stamp> +# +# A secondary output file, called AutoComp.TKdiffs.<time-and-date-stamp> +# is produced; it can be "source"d to cause a sequential tkdiff of +# the detected discrepancies + + +# Set the number of lines by which two .Log files are expected to differ. +# (this is due to different compilation-signature lines) +set LogFilesDiffLines = 6 +# Over-ride with optional shell env't variable called LogFilesDiff +if ( $?LogFilesDiff ) then + set notallnumbers = `echo $LogFilesDiff | tr '0-9' ' '` + if ( "$notallnumbers" == "" ) then + set LogFilesDiffLines = $LogFilesDiff + endif +endif + +# Preliminary error checking: +if ( ! $?CygTestLogs ) then + echo 'Please define a shell environment variable called CygTestLogs' + echo ' whose value is the path to the Cygwin test-results directory.' + exit 1 +endif + +set TArgFiles = `find . -name TestArgs -exec expr {} : './(.*)' ;` +if ( $#TArgFiles == 0 ) then + echo 'No TestArgs files found in subdirectories.' + echo 'Starting in wrong directory?' + exit 2 +endif + +set TArgReslts = `find $CygTestLogs -name TestArgs -exec expr {} : $CygTestLogs/'(.*)' ;` +if ( $#TArgReslts != $#TArgFiles ) then + echo 'Number of TestArgs files in CygTestLogs ('$CygTestLogs') directory' + echo ' does not match those found under current directory.' + echo 'Please correct discrepancy and try again.' + exit 4 +endif + +if ( "$TArgReslts" != "$TArgFiles" ) then + echo 'List of TestArgs files in CygTestLogs ('$CygTestLogs') directory' + echo ' does not match those found under current directory.' + echo 'Please correct discrepancy and try again.' + exit 8 +endif + +# Let's save mismatch identifiers: +set datemark = `date '+%y%m%d.%H%M%S'` +set ErrResltFil = AutoComp.Log.$datemark +set ScriptResltFil = AutoComp.TKdiffs.$datemark + +# Let's show a progress-downcount +# Find the count of actual tests; leave the max number on display: +set cnt = `egrep '^[^#].*' $TArgFiles | wc -l` +echo '' +echo $cnt + +foreach TArgFil ( $TArgFiles ) + set dir = $TArgFil:h + + set lindxmax = `cat $TArgFil | wc -l` + set lindx = 0 + while ( $lindx < $lindxmax ) + @ lindx++ + set targline = `sed -n ${lindx}p $TArgFil` + if ( $#targline == 0 ) continue + if ( "$targline[1]" == "#" ) continue + set ttarg = $dir/$targline[1] +# One more error-check: + if ( ! -f ${ttarg}.fth ) then + echo File Not found: ${ttarg}.fth + echo Please update ${TArgFil} file, line $lindx + continue + endif + + set label = `echo $targline | awk -F , '{print $2}'` + if ( "$label" != "" ) set label = .$label + +# Now begins the real fun... +# Show a running down-count + echo -n X${cnt}' ' | tr X \015 + @ cnt-- + +# Check the extensions that are text-type files + foreach txext ( Log RomHdr fl fl.missing ) +# Is the file in both? + if ( -f $CygTestLogs/${ttarg}${label}.${txext} && ! -f ${ttarg}${label}.${txext} ) then + echo Missing ${ttarg}${label}.${txext} | tee -a $ErrResltFil + else + if ( -f ${ttarg}${label}.${txext} ) then + set lim = 0 + set fxext = `echo $txext | tr '.' '_'` + if ( `eval echo '$?'${fxext}FilesDiffLines` ) then + set lim = `eval echo '$'${fxext}FilesDiffLines` + endif + set logdiff = `diff {,$CygTestLogs/}${ttarg}${label}.${txext} | wc -l` + if ( $logdiff > $lim ) then + echo ${ttarg}${label}.${txext} files differ. | tee -a $ErrResltFil + diff {$CygTestLogs/,}${ttarg}${label}.${txext} | tee -a $ErrResltFil + echo 'tkdiff {$CygTestLogs/,}'${ttarg}${label}.${txext} >> $ScriptResltFil + endif + endif + endif + end + +# Now, check the FCode file. +# If there's an FCode file in the reference directory ($CygTestLogs) +# there should be one in the directory under test. + if ( -f $CygTestLogs/${ttarg}${label}.fc && ! -f ${ttarg}${label}.fc ) then + echo Missing ${ttarg}${label}.fc | tee -a $ErrResltFil + else +# Is there an FCode file in the directory under test? + unset ShowDetokDiffs + if ( -f ${ttarg}${label}.fc ) then + set bindiff = `cmp {,$CygTestLogs/}${ttarg}${label}.fc` + if ( "$bindiff" != "" ) then + echo "$bindiff" | tee -a $ErrResltFil +# If the binaries are different, show the difference in the DeTok form +# If the DeTok form is missing, it's an even stranger error! + if ( ! -f $CygTestLogs/${ttarg}${label}.DeTok || ! -f ${ttarg}${label}.DeTok ) then + echo Missing DeTok file for ${ttarg}${label} | tee -a $ErrResltFil + else + set ShowDetokDiffs + endif + else +# Even if the binaries are not different, +# check for changes in in the DeTok form + set detokdiff = `diff {,$CygTestLogs/}${ttarg}${label}.DeTok | wc -l` + if ( $detokdiff > 0 ) set ShowDetokDiffs + endif + endif + if ( $?ShowDetokDiffs ) then + echo ${ttarg}${label}.DeTok files differ. | tee -a $ErrResltFil + diff {$CygTestLogs/,}${ttarg}${label}.DeTok | tee -a $ErrResltFil + echo 'tkdiff {$CygTestLogs/,}'${ttarg}${label}.DeTok >> $ScriptResltFil + endif + endif + + end +end + +
Added: fcode-utils/testsuite/AutoExec =================================================================== --- fcode-utils/testsuite/AutoExec (rev 0) +++ fcode-utils/testsuite/AutoExec 2006-10-14 20:34:46 UTC (rev 92) @@ -0,0 +1,164 @@ +#! /bin/csh -f +# +# Part of automation of the testing process for the Tokenizer/De-Tokenizer +# Automate Execution of Tokenizing and De-Tokenizing the various test-cases + +# Updated Thu, 09 Feb 2006 at 13:10 PST by David L. Paktor + +# Run this from the parent-directory of the various Test categories. +# Sym-links to the binaries should be here. + +# Each Test category directory must have a TestArgs file, +# formatted as follows: +# Lines starting with # (Pound-sign Space) are comments, +# and are ignored +# Blank lines are also allowed, and are also ignored +# Valid lines have four comma-separated fields: +# Test-file base-name +# Result-file label +# Extra command-line switches +# A script-command, to be run after, for specific +# verification of this Test. Arguments can also +# be included in this field, as long as they have +# no commas. +# An unspecified intermediate field must be represented as +# an empty field separated by commas from the fields +# preceding and following. +# Lines that have no label, no switches, and no script +# do not need any commas. +# The -v (verbose) switch will always be used and does not need +# to be explicitly specified. However, if the "script-command" +# field starts with +V (case-sensitive), then the -v (verbose) +# switch will not be used. +# +# The Test-file base-name field may have multiple base-names, +# to run a multiple-file batch command. +# If a multiple-file batch is run: +# The Result-file label will apply only to the Log file. + +# To Do: +# Contrive a way to run the second through last input files as +# separate individual jobs (with same command-line switches) +# This needs to be co-ordinated with the Auto-Compare script: +# It will need to compare the .FC files from the batch with +# those from the individual runs. + + +# Initial error-checking: +if ( ( ! -x ./toke ) || ( ! -x ./detok ) ) then + echo 'Starting in wrong directory. Executable toke and detok are not here.' + exit 1 +endif + +set TArgFiles = `find . -name TestArgs -exec expr {} : './(.*)' ;` +if ( $#TArgFiles == 0 ) then + echo 'No TestArgs files found in subdirectories.' + echo 'Starting in wrong directory?' + exit 1 +endif + +set parent = `pwd` +set tokex = `pwd`/toke +set detokex = `pwd`/detok +set romhdrex = `pwd`/romheaders + +set chirren = `echo $TArgFiles | tr ' ' \012 | sed -e 'sX/[^/]*$XXg'` + +# The "verbose" flag is now a variable. +set vflg = '-v' + +# Let's show a progress-downcount and time, stamps and elapsed. +# Find the count; leave the max number on display: +set cnt = `egrep '^[^#].*' $TArgFiles | wc -l` +echo '' +set starttime = `date +'%T'` +echo $starttime +echo $cnt + +foreach dir ( $chirren ) + cd $dir + set lindxmax = `cat TestArgs | wc -l` + set lindx = 0 + while ( $lindx < $lindxmax ) + @ lindx++ + set targline = `sed -n ${lindx}p TestArgs` + if ( $#targline == 0 ) continue + if ( "$targline[1]" == "#" ) continue + set ttargs = `echo $targline | awk -F , '{print $1}'` +# One more error-check: + set tfths = '' + set fcfils = '' + unset FMissing + foreach tfth ( $ttargs ) + if ( ! -f ${tfth}.fth ) then + echo File Not found: ${dir}/${tfth}.fth + echo Please update ${dir}/TestArgs file, line $lindx + set FMissing + endif + set tfths = ( $tfths ${tfth}.fth ) + set fcfils = ( $fcfils ${tfth}.fc ) + end + if ( $?FMissing ) continue + + set ttarg = $ttargs[1] + + set outfile = '' + set label = `echo $targline | awk -F , '{print $2}'` + if ( "$label" != "" ) then + set label = .$label + if ( $#tfths == 1 ) then + set fcfils = ${ttarg}${label}.fc + set outfile = ( -o $fcfils ) + endif + endif +# We don't set the switches as a shell-variable because we might +# need to preserve quoted groupings that have embedded spaces. + +# Collect the script-command. +# Awkward place to expect a "+V" option, but it'll have to serve.... + set scriptcmd = `echo $targline | awk -F , '{print $4}'` + if ( $#scriptcmd > 0 ) then + if ( "$scriptcmd[1]" == "+V" ) then + set vflg = "" + if ( $#scriptcmd == 1 ) then + set scriptcmd = "" + else + set scriptcmd = ( $scriptcmd[2-] ) + endif + endif + endif + + echo -n X${cnt}' ' | tr X \015 + @ cnt-- + +# Now do it. + set doromhdr = 0 + echo toke $vflg `sed -n ${lindx}p TestArgs | awk -F , '{print $3}'` $outfile ${tfths} >& ${ttarg}${label}.Log + echo '' >>& ${ttarg}${label}.Log + eval $tokex $vflg `sed -n ${lindx}p TestArgs | awk -F , '{print $3}'` $outfile ${tfths} >>& ${ttarg}${label}.Log + set vflg = '-v' + + foreach fcfil ( $fcfils ) + if ( -f ${fcfil} ) then + $detokex -v -o ${fcfil} > ${fcfil:r}.DeTok + set doromhdr = `grep 'PCI Header identified' ${fcfil:r}.DeTok | wc -l` + if ( $doromhdr ) then + $romhdrex ${fcfil} > ${fcfil:r}.RomHdr + endif + endif + end + + + if ( "$scriptcmd" != "" ) then + eval $scriptcmd + endif + + end + cd $parent +end +# Clear the display-line +echo -n X' 'X | tr X \015 +# Show time, stamps and elapsed. +date +'%T' +echo $starttime ' (Started)' +