[OpenBIOS] r76 - in fcode-utils: detok romheaders toke

Author: stepan
Date: 2006-08-18 14:47:12 +0200 (Fri, 18 Aug 2006)
New Revision: 76

Added:
   fcode-utils/detok/addfcodes.c
   fcode-utils/detok/addfcodes.h
   fcode-utils/detok/pcihdr.c
   fcode-utils/detok/printformats.c
   fcode-utils/toke/clflags.c
   fcode-utils/toke/clflags.h
   fcode-utils/toke/conditl.c
   fcode-utils/toke/conditl.h
   fcode-utils/toke/devnode.c
   fcode-utils/toke/devnode.h
   fcode-utils/toke/errhandler.c
   fcode-utils/toke/errhandler.h
   fcode-utils/toke/flowcontrol.c
   fcode-utils/toke/flowcontrol.h
   fcode-utils/toke/macros.h
   fcode-utils/toke/nextfcode.c
   fcode-utils/toke/nextfcode.h
   fcode-utils/toke/parselocals.c
   fcode-utils/toke/parselocals.h
   fcode-utils/toke/scanner.h
   fcode-utils/toke/strsubvocab.c
   fcode-utils/toke/strsubvocab.h
   fcode-utils/toke/ticvocab.c
   fcode-utils/toke/ticvocab.h
   fcode-utils/toke/tokzesc.c
   fcode-utils/toke/tokzesc.h
   fcode-utils/toke/tracesyms.c
   fcode-utils/toke/tracesyms.h
   fcode-utils/toke/usersymbols.c
   fcode-utils/toke/usersymbols.h
   fcode-utils/toke/vocabfuncts.h
Removed:
   fcode-utils/detok/Rules.make
   fcode-utils/toke/Rules.make
Modified:
   fcode-utils/detok/Makefile
   fcode-utils/detok/decode.c
   fcode-utils/detok/detok.c
   fcode-utils/detok/detok.h
   fcode-utils/detok/dictionary.c
   fcode-utils/detok/stream.c
   fcode-utils/detok/stream.h
   fcode-utils/romheaders/Makefile
   fcode-utils/romheaders/romheaders.c
   fcode-utils/toke/Makefile
   fcode-utils/toke/dictionary.c
   fcode-utils/toke/dictionary.h
   fcode-utils/toke/emit.c
   fcode-utils/toke/emit.h
   fcode-utils/toke/macros.c
   fcode-utils/toke/scanner.c
   fcode-utils/toke/stack.c
   fcode-utils/toke/stack.h
   fcode-utils/toke/stream.c
   fcode-utils/toke/stream.h
   fcode-utils/toke/toke.c
   fcode-utils/toke/toke.h
Log:
initial merge of David Paktor's fine work.

I redid all of the Makefiles and fixed some minor 
issues with the code.

Modified: fcode-utils/detok/Makefile
===================================================================
--- fcode-utils/detok/Makefile	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/Makefile	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,11 +1,12 @@
 #
-#                     OpenBIOS - free your system! 
-#                            ( detokenizer )
-#                          
-#  This program is part of a free implementation of the IEEE 1275-1994 
+#                     OpenBIOS - free your system!
+#                             ( Utilities )
+#
+#  This program is part of a free implementation of the IEEE 1275-1994
 #  Standard for Boot (Initialization Configuration) Firmware.
 #
-#  Copyright (C) 2001-2005 Stefan Reinauer, 
+#  Copyright (C) 2001-2006  Stefan Reinauer 
+#  Copyright (C) 2006 coresystems GmbH 
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -21,48 +22,37 @@
 #  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
 #

-ARCH     := $(shell uname -m | sed -e s/i.86/x86/ -e s/sun4u/sparc64/ -e s/arm.*/arm/ -e s/sa110/arm/ -e s/x86_64/amd64/ -e "s/Power Macintosh/ppc/")
-TOPDIR   := $(shell /bin/pwd)
-BUILDDIR ?= $(TOPDIR)/obj-$(ARCH)
+PROGRAM = detok

-VPATH    := $(BUILDDIR)
+CC      = gcc
+STRIP	= strip
+INCLUDES = -I../shared
+#CFLAGS  = -O2 -g -Wall 
+CFLAGS  = -Os -Wall -Wno-pointer-sign 
+LDFLAGS =

+OBJS  = addfcodes.o decode.o detok.o dictionary.o pcihdr.o printformats.o \
+        stream.o ../shared/classcodes.o

-include $(TOPDIR)/Rules.make
+all: .dependencies $(PROGRAM)

+$(PROGRAM): $(OBJS)
+	$(CC) -o $(PROGRAM) $(OBJS) $(LDFLAGS)
+	$(STRIP) -s $(PROGRAM)

-CC	  =  gcc
-CFLAGS	  =  -O2 -Wall -ansi
+clean:
+	rm -f $(OBJS) *~

-# For debugging the indentation code of detok, define DEBUG_INDENT
-#CFLAGS	 :=  $(CFLAGS) -DDEBUG_INDENT
+distclean: clean
+	rm -f $(PROGRAM) .dependencies
+	
+.dependencies: *.c 
+	@$(CC) $(CFLAGS) $(INCLUDES) -MM *.c > .dependencies

-all: main detok
-	@echo -e "\nOpenBIOS detokenizer detok build finished\n"
+.PHONY: all clean distclean

-main:
-	@echo -e "\nWelcome to the OpenBIOS detokenizer.."
-	@test -r $(BUILDDIR) || ( mkdir -p $(BUILDDIR); \
-	  echo -e "\nCreating build directory $(BUILDDIR)" )
-		
-detok: detok.o dictionary.o decode.o stream.o 
-	@echo -en "\nLinking fcode detokenizer detok..."
-	@cd $(BUILDDIR) && ( $(CC) $(CFLAGS) $^ -o $@; strip detok )
-	@echo -e "\tok"
-	
-clean:
-	@test ! -d $(BUILDDIR) && \
-		echo "Architecture $(ARCH) is already clean." || \
-		( 						 \
-			echo "Cleaning up architecture $(ARCH)"; \
-			rm -rf $(BUILDDIR)			 \
-			rm forth.dict.core			 \
-		)
+-include .dependencies

-distclean: clean
-	rm -f detok
+.c.o:
+	$(CC) -c $(CFLAGS) $(INCLUDES) $< -o $@

-detok.o:      detok.h stream.h detok.c
-stream.o:     detok.h stream.c
-decode.o:     detok.h stream.h decode.c Makefile
-dictionary.o: detok.h dictionary.c

Deleted: fcode-utils/detok/Rules.make
===================================================================
--- fcode-utils/detok/Rules.make	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/Rules.make	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,17 +0,0 @@
-# tag: Makefile rules
-
-VPATH    := $(VPATH):.
-
-.S.o:
-	echo -n "  assembling $<... "
-	$(CC) -c -nostdlib $(INCLUDES) $(CFLAGS) $< -o $(BUILDDIR)/$@ && \
-		echo -e "    \t\tok" || \
-		echo -e "    \t\tfailed"
-
-.c.o:
-	@echo -n "  compiling $<... "
-	@$(CC) -c $(CFLAGS) $(INCLUDES) $< -o $(BUILDDIR)/$@ && \
-		echo -e "   \t\tok" || \
-		echo -e "   \t\failed"
-		
-

Added: fcode-utils/detok/addfcodes.c
===================================================================
--- fcode-utils/detok/addfcodes.c	                        (rev 0)
+++ fcode-utils/detok/addfcodes.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,343 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Function(s) for entering Vendor-Specific FCodes to detokenizer.
+ *
+ *      (C) Copyright 2006 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          add_fcodes_from_list           Add Vendor-Specific FCodes from
+ *                                             the file whose name is supplied.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Revision History:
+ *          Tue, 25 Apr 2006 by David L. Paktor
+ *              Identified this need when working with in-house code,
+ *                  which uses some custom functions.  This solution
+ *                  is (hoped to be) general enough to cover all cases.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              indata                Buffer into which the file will be read
+ *              stream_max            Size of the file buffer.
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "detok.h"
+#include "stream.h"
+#include "addfcodes.h"
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *     current_vfc_line       Line to be scanned in Vendor-FCodes buffer
+ *     vfc_remainder          Remainder of Vendor-FCodes buffer to be scanned
+ *     vfc_line_no            Number of current line in Vendor-FCodes buffer
+ *     vfc_buf_end            Pointer to end of Vendor-FCodes buffer
+ *
+ **************************************************************************** */
+
+static char *current_vfc_line;
+static char *vfc_remainder;
+static int vfc_line_no = 0;
+static char *vfc_buf_end;
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_whitespace
+ *      Synopsis:       Advance the given string-pointer past blanks and tabs.
+ *                          (String is presumed to end before new-line)
+ *
+ *      Inputs:
+ *         Parameters:
+ *             string_line_ptr        Address of pointer to string
+ *
+ *      Outputs:
+ *         Returned Value:            None
+ *             *string_line_ptr       Advanced past blanks and tabs
+ *
+ **************************************************************************** */
+
+static void skip_whitespace( char **string_line_ptr)
+{
+    char *cur_char_ptr = *string_line_ptr;
+    for ( ; *cur_char_ptr != 0 ; cur_char_ptr++ )
+    {
+	if ( (*cur_char_ptr != '\t') && (*cur_char_ptr != ' ') )
+	{
+	    *string_line_ptr = cur_char_ptr;
+	    break;
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  get_next_vfc_line
+ *      Synopsis:       Advance to the next  vfc_line  to be processed.
+ *                      Skip blanks and comments.  Indicate when end reached.
+ *
+ *      Inputs:
+ *         Parameters:                    None
+ *         Local Static Variables:
+ *             vfc_remainder
+ *             vfc_buf_end
+ *
+ *      Outputs:
+ *         Returned Value:               FALSE if reached end of buffer
+ *         Local Static Variables:
+ *             current_vfc_line          Advanced to next line to be scanned
+ *             vfc_line_no               Kept in sync with line number in file
+ *
+ *      Process Explanation:
+ *          Comments begin with a pound-sign  ('#') or a backslash  ('')
+ *          Comment-lines or blank or empty lines will be skipped. 
+ *
+ **************************************************************************** */
+
+static bool get_next_vfc_line( void )
+{
+    bool retval = FALSE;   /*  TRUE = not at end yet  */
+    while ( vfc_remainder < vfc_buf_end )
+    {
+	current_vfc_line = vfc_remainder;
+	vfc_remainder = strchr( current_vfc_line, '\n');
+	*vfc_remainder = 0;
+	vfc_remainder++;
+	vfc_line_no++;
+	skip_whitespace( &current_vfc_line);
+	if ( *current_vfc_line == 0 )  continue;  /*  Blank line */
+	if ( *current_vfc_line == '#' )  continue;  /*  Comment  */
+	if ( *current_vfc_line == '\' )  continue;  /*  Comment  */
+	retval = TRUE;
+	break;  /*  Found something  */
+    }
+    return( retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  vfc_splash
+ *      Synopsis:       Print a "splash" message to show that we
+ *                          are processing Vendor-Specific FCodes,
+ *                          but only once.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             vf_file_name            Vendor-Specific FCodes file name
+ *         Local Static Variables:
+ *             did_not_splash          Control printing; once only.
+ *
+ *      Outputs:
+ *         Returned Value:             None
+ *         Local Static Variables:
+ *             did_not_splash          FALSE after first call.
+ *         Printout:
+ *             "Splash" message...
+ *
+ **************************************************************************** */
+static bool did_not_splash = TRUE;
+static void vfc_splash( char *vf_file_name)
+{
+    if ( did_not_splash )
+    {
+	/*  Temporary substring buffer                */
+	/*  Guarantee that the malloc will be big enough.  */
+	char *strbfr = malloc( strlen( vf_file_name) +65 ) ;
+	sprintf( strbfr,
+	    "Reading additional FCodes from file:  %s\n",
+		vf_file_name);
+	printremark( strbfr);
+	free( strbfr);
+	did_not_splash = FALSE;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  add_fcodes_from_list
+ *      Synopsis:       Add Vendor-Specific FCodes from the named file
+ *                          to the permanent resident dictionary.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             vf_file_name            Vendor-Specific FCodes file name
+ *         Global Variables:
+ *             verbose                 "Verbose" flag.
+ *             indata                  Start of file buffer
+ *             stream_max              Size of the file buffer.
+ *
+ *      Outputs:
+ *         Returned Value:             TRUE if FCodes have actually been added
+ *         Global Variables:
+ *             check_tok_seq           Cleared to FALSE, then restored to TRUE
+ *         Local Static Variables:
+ *             vfc_remainder           Initted to start of file buffer
+ *             vfc_buf_end             Initted to end of file buffer
+ *         Memory Allocated
+ *             Permanent copy of FCode Name
+ *         When Freed?
+ *             Never.  Rmeains until program termination.
+ *         Printout:
+ *             If verbose, "Splash" line and count of added entries.
+ *
+ *      Error Detection:
+ *          Fail to open or read Vendor-FCodes file -- Exit program
+ *          Improperly formatted input line -- print message and ignore
+ *          FCode value out of valid range -- print message and ignore
+ *          FCode value already in use -- print message and ignore
+ *
+ *      Process Explanation:
+ *          Valid lines are formatted with the FCode number first
+ *              and the name after, one entry per line.  Extra text
+ *              after the name will be ignored, so an extra "comment"
+ *              is permitted.  The FCode number must be in hex, with
+ *              an optional leading  0x  or  0X   For example:  0X407
+ *          The valid range is 0x010 to 0x7ff.  Numbers above 0x800
+ *              infringe upon the are reserved for FCodes generated
+ *              by the tokenization process.
+ *          Numbers already in use will be ignored.  A Message will be
+ *              printed even if the name matches the one on the line.
+ *
+ **************************************************************************** */
+
+bool add_fcodes_from_list( char *vf_file_name)
+{
+    bool retval = FALSE;
+    int added_fc_count = 0;
+    check_tok_seq = FALSE;
+
+    if ( verbose )  vfc_splash( vf_file_name);
+
+    if ( init_stream( vf_file_name) != 0 )
+    {
+	char *strbfr = malloc( strlen( vf_file_name) +65 );
+	sprintf( strbfr,
+	    "Could not open Additional FCodes file:  %s\n",
+		vf_file_name);
+	printremark( strbfr);
+	free( strbfr);
+	exit(1);
+    }
+    vfc_remainder = indata;
+    vfc_buf_end = indata + stream_max -1;
+
+    while (  get_next_vfc_line() )
+    {
+	char vs_fc_name[36];
+	int  vs_fc_number;
+	int  scan_result;
+	char *lookup_result;
+	char *fc_name_cpy;
+
+	scan_result = sscanf( current_vfc_line, "0x%x %32s",
+	     &vs_fc_number, vs_fc_name);
+
+	if ( scan_result != 2 )   /*  Allow a capital  0X   */
+	{
+	    scan_result = sscanf( current_vfc_line, "0X%x %32s",
+		&vs_fc_number, vs_fc_name);
+	}
+	if ( scan_result != 2 )   /*  Try it without the  0x   */
+	{
+	    scan_result = sscanf( current_vfc_line, "%x %32s",
+		&vs_fc_number, vs_fc_name);
+	}
+
+	if ( scan_result != 2 )   /*  That's it... */
+	{
+	    char *strbfr = malloc( strlen( current_vfc_line) +65 );
+	    vfc_splash( vf_file_name);
+	    sprintf( strbfr,
+		"Line #%d, invalid format.  Ignoring:  %s\n",
+		    vfc_line_no, current_vfc_line);
+	    printremark( strbfr);
+	    free( strbfr);
+	    continue;
+	}
+
+	if ( ( vs_fc_number < 0x10 ) || ( vs_fc_number > 0x7ff ) )
+	{
+	    char *strbfr = malloc( 85 );
+	    vfc_splash( vf_file_name);
+	    sprintf( strbfr,
+		"Line #%d, FCode number out of range:  0x%x  Ignoring.\n",
+		    vfc_line_no, vs_fc_number);
+	    printremark( strbfr);
+	    free( strbfr);
+	    continue;
+	}
+
+	lookup_result = lookup_token( (u16)vs_fc_number);
+	if ( strcmp( lookup_result, "ferror") != 0 )
+	{
+	    char *strbfr = malloc( strlen( lookup_result) + 85 );
+	    vfc_splash( vf_file_name);
+	    sprintf( strbfr,
+		"Line #%d.  FCode number 0x%x is already "
+		    "defined as %s  Ignoring.\n",
+			vfc_line_no, vs_fc_number, lookup_result);
+	    printremark( strbfr);
+	    free( strbfr);
+	    continue;
+	}
+
+	/*  We've passed all the tests!  */
+	fc_name_cpy = strdup( vs_fc_name);
+	add_token( (u16)vs_fc_number, fc_name_cpy);
+	added_fc_count++;
+	retval = TRUE;
+    }
+
+    if ( verbose )
+    {
+	char *strbfr = malloc( 85 );
+        sprintf( strbfr,
+	    "Added %d FCode numbers\n", added_fc_count);
+	printremark( strbfr);
+	free( strbfr);
+    }
+
+    close_stream();
+    check_tok_seq = TRUE;
+    return( retval);
+}

Added: fcode-utils/detok/addfcodes.h
===================================================================
--- fcode-utils/detok/addfcodes.h	                        (rev 0)
+++ fcode-utils/detok/addfcodes.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,48 @@
+#ifndef _DETOK_VSFCODES_H
+#define _DETOK_VSFCODES_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Header for function for entering Vendor-Specific FCodes
+ *          to detokenizer.
+ *
+ *      (C) Copyright 2006 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "types.h"
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+bool add_fcodes_from_list( char *vf_file_name);
+
+#endif   /*  _DETOK_VSFCODES_H    */

Modified: fcode-utils/detok/decode.c
===================================================================
--- fcode-utils/detok/decode.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/decode.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,96 +24,322 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <ctype.h>
+#include <setjmp.h>

#include "detok.h"
 #include "stream.h"

-/* Dictionary function prototypes */
-char *lookup_token(u16 number);
-int  add_token(u16 number, char *name);
+static int indent;    /*  Current level of indentation   */

-extern u16 fcode;
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Handling of indent-level is not correct.  Branches should
+ *              balance with their resolvers; constructs like do..loop
+ *              case/of/endof/endcase are a few major examples.
+ *          This will be tricky; the rules need to be carefully thought
+ *              out, and the implementation might be more complex than
+ *              at first meets the eye...
+ *
+ **************************************************************************** */

+static bool ended_okay = TRUE;    /*  FALSE if finished prematurely  */
+
 bool offs16=TRUE;
-int indent, verbose=0, decode_all=0, linenumbers=0, linenum;
-u32 fclen;
+unsigned int linenum;
+bool end_found=FALSE;
+unsigned int token_streampos;   /*  Streampos() of currently-gotten token  */
+u16 last_defined_token = 0;

-static u8 *unnamed=(u8 *)"(unnamed-fcode)";
+jmp_buf eof_exception;

-void pretty_string(u8 *string)
+static int fclen;
+static const char *unnamed = "(unnamed-fcode)" ;
+
+static void decode_indent(void) 
 {
-	u8 c;
-	unsigned int i;
-	bool qopen=TRUE;
+	int i;
+	if (indent<0) {
+#ifdef DEBUG_INDENT
+		printf("detok: error in indentation code.\n");
+#endif
+		indent=0;
+	}
+	for (i=0; i<indent; i++)
+		printf ("    ");
+}
    
+/*  Print forth string ( [len] char[0] ... char[len] ) */
+static void pretty_print_string(void)
+{
+    u8 len;
+    u8 *strptr;
+    int indx;
+    bool in_parens = FALSE ;  /*  Are we already inside parentheses?  */
+
+    strptr = get_string( &len );
+
+    printf("( len=%s%x", len >= 10 ? "0x":"", len);
+    if ( len >= 10 )   printf(" [%d bytes]", len);
+    printf(" )\n");
+    if (show_linenumbers)  printf("        ");
+    decode_indent();
    printf("" ");

-	for (i=1; i<1+(unsigned int)string[0]; i++) {
-		c=string[i];
+    for ( indx = 0; indx < len; indx++ )
+    {
+	u8 c = *strptr++;
    	if (isprint(c)) {
-			if (!qopen) {
+	    if ( in_parens )
+	    {
    			printf(" )");
-				qopen=TRUE;
+		in_parens = FALSE;
    		}
    		printf("%c",c);
+	    /*  Quote-mark must escape itself  */
+	    if ( c == '"' ) printf("%c",c);
    	} else {
-			if (qopen) {
+	    if ( !in_parens )
+	    {
    			printf(""(");
-				qopen=FALSE;
+		in_parens = TRUE;
    		}
    		printf(" %02x",c);
    	}
    }
-	if (!qopen)
+    if ( in_parens )
    	printf(" )");
    printf(""");
 }

-static void decode_indent(void) 
+static void decode_lines(void)
 {
-	int i;
-#ifdef DEBUG_INDENT
-	if (indent<0) {
-		printf("detok: error in indentation code.\n");
-		indent=0;
+	if (show_linenumbers) {
+		printf("%6d: ",show_offsets ? token_streampos : linenum++);
    }
-#endif
-	for (i=0; i<indent; i++)
-		printf ("    ");
 }

-static void decode_lines(void)
+/* **************************************************************************
+ *
+ *      Function name:  output_token_name
+ *      Synopsis:       Print the name of the token just retrieved
+ *                          along with any interesting related information...
+ *
+ *      Inputs:
+ *         Global/Static Variables:
+ *             fcode                  The token # just retrieved
+ *             last_defined_token     Used to screen invalid tokens.
+ *             token_streampos        Location of token just retrieved
+ *         Global/Static Constants:
+ *             unnamed                Namefield of headerless token 
+ *
+ *      Outputs:
+ *         Printout:
+ *             Print the function name (if known) and the FCode number,
+ *                  if interesting.
+ *             The fcode number is interesting if either
+ *                 a) the token has no name
+ *                           or
+ *                 b) verbose mode is in effect.
+ *             If the token is named, show its FCode number in
+ *                 the syntax of a FORTH Comment, otherwise, its
+ *                 FCode number -- in [brackets] -- acts as its name.
+ *
+ *      Error Detection:
+ *          If the token # is larger than the last defined token, it is
+ *              probably an artifact of an error that was allowed in the
+ *              tokenization; if it were treated normally, it would lead
+ *              to a cascade of failures.  Print a message, skip the first
+ *              byte, and return.
+ *
+ *      Revision History:
+ *          Refactored.  The line number or offset (if selected) and the
+ *              indent are printed in the  output_token()  routine, which
+ *              calls this one.
+ *          Add Error Detection
+ *          
+ *
+ **************************************************************************** */
+
+static void output_token_name(void)
 {
-	if (linenumbers) {
-		printf("%6d: ",linenumbers==1?linenum:get_streampos());
-		linenum++;
+	char *tname;
+
+	/*  Run error detection only if last_defined_token was assigned  */
+	if ( (fcode > last_defined_token) && (last_defined_token > 0) )
+	{
+	    char temp_buf[80];
+	    int buf_pos;
+	    u8 top_byte = fcode >> 8;
+	    printf ("Invalid token:  [0x%03x]\n", fcode);
+	    sprintf(temp_buf, "Backing up over first byte, which is ");
+	    buf_pos = strlen(temp_buf);
+	    if ( top_byte < 10 )
+	    {
+	        sprintf(&temp_buf[buf_pos], " %02x", top_byte);
+	    }else{
+	        sprintf( &temp_buf[buf_pos], "0x%02x ( =dec %d)",
+		    top_byte, top_byte);
    }
+	    printremark(temp_buf);
+	    set_streampos(token_streampos+1);
+	    return;
 }

-static void output_token(void)
-{
-	u8 *tname=lookup_token(fcode);
    
+	tname = lookup_token(fcode);
    printf ("%s ", tname);

-	/* The fcode number is interesting if
-	 *  a) detok is in verbose mode
-	 *  b) the token has no name.
+	/* The fcode number is interesting
+	 *  if either
+	 *  a) the token has no name
+	 *            or
+	 *  b) detok is in verbose mode.
     */
-	if (verbose || tname == unnamed)
+       if ( tname == unnamed )
+       {
    	printf("[0x%03x] ", fcode);
+       } else {
+	    if ( verbose )
+	    {
+		/*  If the token is named,
+		 *  show its fcode number in
+		 *  the syntax of a FORTH Comment
+		 */
+		printf("( 0x%03x ) ", fcode);
+	    }
+	}
 }

+/* **************************************************************************
+ *
+ *      Function name:  output_token
+ *      Synopsis:       Print the full line for the token just retrieved:
+ *                          line number, indentation and name.
+ *
+ *      Revision History:
+ *          Updated Mon, 11 Jul 2005 by David L. Paktor
+ *          Separate out  output_token_name()  for specific uses.
+ *          Incorporate calls to  decode_lines()  and   decode_indent()
+ *              to allow better control of indentation.
+ *
+ **************************************************************************** */
+
+static void output_token(void)
+{
+	decode_lines();
+	decode_indent();
+	output_token_name();
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  decode_offset
+ *      Synopsis:       Gather and display an FCode-offset associated with
+ *                          a branch or suchlike function.
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Global/Static Variables:
+ *             show_offsets            Whether to show offsets in full detail
+ *             offs16                  Whether 16- (or 8-) -bit offsets
+ *             stream_max              Maximum valid destimation      
+ *         Local Static Variables:
+ *
+ *      Outputs:
+ *         Returned Value:             The offset, converted to signed 16-bit #
+ *         Global/Static Variables:
+ *             stream-position         Reset if invalid destination; otherwise,
+ *                                         advanced in the normal manner.
+ *         Printout:
+ *             The offset and destination, if display of offsets was specified
+ *                 or if the destination is invalid
+ *
+ *      Error Detection:
+ *          Crude and rudimentary:
+ *          If the target-destination is outside the theoretical limits,
+ *              it's obviously wrong.
+ *          Notification remark is printed and the stream-position reset
+ *              to the location of the offset, to allow it to be processed
+ *              in the manner of normal tokens.
+ *          If the offset is zero, that's obviously wrong, but don't reset
+ *              the stream-position:  zero gets processed as  end0  and that
+ *              is also wrong...
+ *
+ *      Still to be done:
+ *          More sophisticated error-checking for invalid offsets.  Look
+ *              at the type of branch, and what should be expected in the
+ *              vicinity of the destination.  (This might be best served
+ *              by a separate routine).
+ *         This might also help with handling the indent-level correctly...
+ *              Also, if indentation were to be handled in this routine,
+ *              there would be no need to return the value of the offset.
+ *
+ **************************************************************************** */
+
 static s16 decode_offset(void)
 {
         s16 offs;
+	int dest;
+	bool invalid_dest;
+	int streampos = get_streampos();

+	output_token();
         offs=get_offset();
-	output_token();
-	printf("0x%x\n",offs);
+
+	/*  The target-destination is the source-byte offset
+	 *      at which the FCode-offset is found, plus
+	 *      the FCode-offset.
+	 */
+	dest = streampos + offs;
+
+	/*  A destination of zero is invalid because there must be a
+	 *      token -- such as  b(<mark)  or   b(do)  -- preceding
+	 *      the target of a backward branch.
+	 *  A destination at the end of the stream is unlikely but
+	 *      theoretically possible, so we'll treat it as valid.
+	 *  An offset of zero is also, of course, invalid.
+	 */
+	invalid_dest = BOOLVAL ( (dest <= 0)
+	                      || (dest > stream_max)
+	                      || (offs == 0) );
+
+	/*  Show the offset in hex and again as a signed decimal number.   */
+	if ( offs16 )
+	{
+	    printf("0x%04x (", (u16)(offs & 0xffff) );
+	}else{
+	    printf("0x%02x (",  (u8)(offs & 0x00ff) );
+	}
+	if ( (offs < 0) || (offs > 9) )  printf(" =dec %d", offs);
+	/*  If we're showing source-byte offsets, show targets of offsets  */
+	if ( show_offsets || invalid_dest )
+	{
+	    printf("  dest = %d ",dest);
+	}
+	printf(")\n");
+
+	if ( invalid_dest )
+	{
+	    if (offs == 0) 
+	    {
+	      printremark("Error:  Unresolved offset.");
+	    }else{
+	      printremark("Error:  Invalid offset.  Ignoring...");
+	      set_streampos( streampos);
+	    }
+	}
    return offs;
 }

@@ -127,59 +353,39 @@
 {
    u16 token;
    output_token();
-	token=get_token();
+	token = next_token();
    printf("0x%03x\n",token);
-	add_token(token, unnamed);
+	add_token(token, (char *)unnamed);
 }

static void named_token(void)
 {
    u16 token;
+	u8 len;
    u8* string;

output_token();
    /* get forth string ( [len] [char0] ... [charn] ) */
-	string=get_string();
-	token=get_token();
-
-	printf("%s 0x%03x\n", string+1, token);
-	
-	add_token(token,string+1);
+	string=get_name(&len);
+	token = next_token();
+	printf("%s 0x%03x\n", string, token);
+	add_token(token,string);
 }

-static void external_token(void)
+static void bquote(void)
 {
-	u16 token;
-	u8* string;
-
    output_token();
    /* get forth string ( [len] [char0] ... [charn] ) */
-	string=get_string();
-	token=get_token();
-
-	printf("%s 0x%03x\n", string+1, token);
-
-	add_token(token,string+1);
-}
-
-static void bquote(void)
-{
-	u8 *string;
-
-	/* get forth string ( [len] [char0] ... [charn] ) */
-	string=get_string();
-	output_token();
-	pretty_string(string);
+	pretty_print_string();
    printf("\n");
-	free(string);
 }

static void blit(void)
 {
    u32 lit;

+	output_token();
    lit=get_num32();
-	output_token();
    printf("0x%x\n",lit);
 }

@@ -204,56 +410,81 @@
    u16 token;

output_token();
-	token=get_token();
-	decode_default();
+	token = next_token();
+	output_token_name();
+	printf ("\n");
 }

+/* **************************************************************************
+ *
+ *      Function name:  decode_start
+ *      Synopsis:       Display the (known valid) FCode block Header
+ *
+ *      Outputs:
+ *         Global/Static Variables:    
+ *             fclen         Length of the FCode block as shown in its Header
+ *
+ **************************************************************************** */
+
 static void decode_start(void)
 {
    u8  fcformat;
    u16 fcchecksum, checksum=0;
-	long pos;
-	u32 i;

-	decode_default();
+	output_token();
+	printf("  ( %d-bit offsets)\n", offs16 ? 16 : 8 );
    
+        token_streampos = get_streampos();
    decode_lines();
         fcformat=get_num8();
    printf("  format:    0x%02x\n", fcformat);

-	decode_lines();
-        fcchecksum=get_num16();
-        /* missing: check for checksum correctness. */
- 
-        fclen=get_num32(); /* skip len */
-	pos=get_streampos();
    
-	for (i=0; i<fclen-pos; i++)
-		checksum+=get_num8();
+        /* Check for checksum correctness. */
    
-	set_streampos(pos-4);
+        token_streampos = get_streampos();
+	decode_lines();
+        fcchecksum=get_num16();        /*  Read the stored checksum  */
+	checksum = calc_checksum();    /*  Calculate the actual checksum  */
    
-	printf("  checksum:  0x%04x (%sOk)\n", fcchecksum,
-			fcchecksum==checksum?"":"not ");
+	if ( fcchecksum==checksum )
+	{
+		printf("  checksum:  0x%04x (Ok)\n", fcchecksum);
+	} else {
+		printf("  checksum should be:  0x%04x, but is 0x%04x\n",
+			checksum,fcchecksum);
+	}

+        token_streampos = get_streampos();
    decode_lines();
    fclen=get_num32();
-        printf("  len:       0x%x (%d bytes)\n", fclen, fclen);
+        printf("  len:       0x%04x ( %d bytes)\n", fclen, fclen);
 }

-void decode_token(u16 token)
+/* **************************************************************************
+ *
+ *      Function name: decode_token
+ *      Synopsis:      Display detokenization for one token.
+ *                     Handle complicated cases and dispatch simple ones.
+ *
+ *      Revision History:
+ *          Detect FCode-Starters in the middle of an FCode block.
+ *          Some tuning of adjustment of indent, particularly wrt branches...
+ *
+ **************************************************************************** */
+
+static void decode_token(u16 token)
 {
+    bool handy_flag = TRUE;
    switch (token) {
    case 0x0b5:
    	new_token();
    	break;
-	case 0x0b6:
+	case 0x0b6:    /*   Named Token  */
+	case 0x0ca:    /*   External Token  */
    	named_token();
    	break;
-	case 0x0ca:
-		external_token();
-		break;
    case 0x012:
    	bquote();
    	break;
@@ -276,14 +507,14 @@
    case 0x0c2: /* b(;) */
    case 0x0b2: /* b(>resolve) */
    case 0x0c5: /* b(endcase) */
+		indent--;
    	decode_default();
-		indent--;
    	break;
    case 0x015: /* b(loop) */
    case 0x016: /* b(+loop) */
    case 0x0c6: /* b(endof) */
+		indent--;
    	decode_offset();
-		indent--;
    	break;
    case 0x017: /* b(do) */
    case 0x018: /* b/?do) */
@@ -295,38 +526,274 @@
    case 0x0c3: /* b(to) */
    	decode_two();
    	break;
+	case 0x0fd: /* version1 */
+		handy_flag = FALSE;
    case 0x0f0: /* start0 */
    case 0x0f1: /* start1 */
    case 0x0f2: /* start2 */
    case 0x0f3: /* start4 */
+		offs16 = handy_flag;
+		printremark("Unexpected FCode-Block Starter.");
    	decode_start();
+		printremark("  Ignoring length field.");
    	break;
+	case    0:  /* end0  */
+	case 0xff:  /* end1  */
+		end_found=TRUE;
+		decode_default();
+		break;
+	default:
+		decode_default();
+	}
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  decode_fcode_header
+ *      Synopsis:       Detokenize the FCode Header.
+ *                          Check for file corruption
+ *
+ *      Inputs:
+ *         Parameters:          NONE
+ *         Global/Static Variables:        
+ *             stream_max       Length of input stream
+ *
+ *      Outputs:
+ *         Returned Value:      NONE
+ *         Global/Static Variables:
+ *             offs16           FALSE if Starter was version1, else TRUE
+ *             fclen            On error, gets set to reach end of input stream
+ *                                  Otherwise, gets set by  decode_start() 
+ *         Printout:
+ *
+ *      Error Detection:
+ *          First byte not a valid FCode Start:  Print message, restore
+ *              input pointer to initial value, set fclen to [(end of
+ *              input stream) - (input pointer)], return FALSE.
+ *
+ *      Process Explanation:
+ *          This routine error-checks and dispatches to the routine that
+ *              does the actual printing.
+ *          Refrain from showing offset until correctness of starter-byte
+ *              has been confirmed.
+ *
+ **************************************************************************** */
+
+static void decode_fcode_header(void)
+{
+    long err_pos;
+    u16 token;
+
+    err_pos = get_streampos();
+    indent = 0;
+    token = next_token();
+    offs16=TRUE;
+    switch (token)
+    {
    case 0x0fd: /* version1 */
-		decode_start();
    	offs16=FALSE;
+	case 0x0f0: /* start0 */
+	case 0x0f1: /* start1 */
+	case 0x0f2: /* start2 */
+	case 0x0f3: /* start4 */
+	    decode_start();
    	break;
    default:
-		decode_default();
+	    {
+		char temp_bufr[128] = 
+			"Invalid FCode Start Byte.  Ignoring FCode header." ;
+		set_streampos( err_pos );
+		fclen = max - pc;
+		printf("\n");
+		if (show_linenumbers)
+		{
+		    sprintf( &(temp_bufr[strlen(temp_bufr)]),
+		    "  Remaining len = 0x%04x ( %d bytes)", fclen, fclen); 
+		}
+		printremark( temp_bufr );
    }
 }
+}

-int detokenize(void)
+/* **************************************************************************
+ *
+ *      Function name:  decode_fcode_block
+ *      Synopsis:       Detokenize one FCode block.
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *         Global/Static Variables:
+ *             fclen         Length of the FCode block as shown in its Header
+ *             end_found     Whether the END0 code was seen
+ *             decode_all    TRUE = continue even after END0
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Global/Static Variables:
+ *             end_found     Whether the END0 code was seen
+ *         Printout:
+ *             A summary message at the end of FCode detokenization
+ *
+ *      Error Detection:
+ *          If the end of the FCode block, as calculated by its FCode length,
+ *              was reached without encountering END0, print a message.
+ *          Detect END0 that occurs before end of FCode block,
+ *              even if  decode_all  is in effect.
+ *
+ *      Process Explanation:
+ *          This routine dispatches to the routines that do the actual
+ *              printing of the detokenization.
+ *          The  end_found  flag is not a direct input, but more of an
+ *              intermediate input, so to speak...  Clear it at the start.
+ *          Detection of FCode-Starters in the middle of an FCode block
+ *              is handled by  decode_token()  routine.
+ *
+ **************************************************************************** */
+
+static void decode_fcode_block(void)
 {
    u16 token;
+    unsigned int fc_block_start;
+    unsigned int fc_block_end;
    
-	if (linenumbers)
-		linenum=1;
+    end_found = FALSE;
+    fc_block_start = get_streampos();
    
-	do {
-		decode_lines();
-		decode_indent();
-		token=get_token();
+    decode_fcode_header();
+
+    fc_block_end = fc_block_start + fclen;
+    
+    while ( ( !end_found || decode_all )
+         && ( get_streampos() < fc_block_end  ) )
+    {
+	token = next_token();
    	decode_token(token);
-	} while ((token||decode_all) && ((get_streampos()-1)<=fclen));
+    }
+    if ( !end_found )
+    {
+	printremark("FCode-ender not found");
+    }
+    {
+	char temp_bufr[80];
+	/*  Don't use  fclen  here, in case it got corrupted
+	 *      by an "Unexpected FCode-Block Starter"
+	 */
+	if ( get_streampos() == fc_block_end  )
+	{
+	    sprintf( temp_bufr,
+		"Detokenization finished normally after %d bytes.",
+		     fc_block_end - fc_block_start );
+	}else{
+	    sprintf( temp_bufr,
+		"Detokenization finished prematurely after %d of %d bytes.",
+		     get_streampos() - fc_block_start,
+			 fc_block_end - fc_block_start );
+	    ended_okay = FALSE;
+	}
+	printremark( temp_bufr );
+    }
+}

-	decode_lines();
-	printf ("\ detokenizing finished after %d of %d bytes.\n",
-			get_streampos(), fclen );
+/* **************************************************************************
+ *
+ *      Function name:  another_fcode_block
+ *      Synopsis:       Indicate whether there is a follow-on FCode block
+ *                      within the current PCI image.
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *         Global/Static Variables:
+ *             token_streampos             Streampos() of token just gotten
+ *             Next token in Input Stream
+ *
+ *      Outputs:
+ *         Returned Value:                  TRUE if next token shows the start
+ *                                              of a valid FCode-block
+ *         Printout:
+ *             Message if there is a follow-on FCode block
+ *
+ *      Error Detection:
+ *          If next token is neither a valid FCode-block starter nor the
+ *              start of a zero-fill field, print a message.
+ *
+ *      Process Explanation:
+ *          Extract the next token from the Input Stream but do not
+ *              consume it.  Then examine the token.
+ *
+ **************************************************************************** */

-	return 0;
+static bool another_fcode_block(void)
+{
+    bool retval = FALSE;
+    u16 token;
+
+    token = next_token();
+    set_streampos( token_streampos );
+
+    switch (token)
+    {
+	case 0x0fd: /* version1 */
+	case 0x0f0: /* start0 */
+	case 0x0f1: /* start1 */
+	case 0x0f2: /* start2 */
+	case 0x0f3: /* start4 */
+	    retval = TRUE;
+	    printremark("Subsequent FCode Block detected.  Detokenizing.");
+	    break;
+	case 0:    /* Start of a zero-fill field  */
+	    /* retval already = FALSE .  Nothing else to be done.  */
+	    break;
+	default:
+	    {
+		char temp_bufr[80];
+		sprintf( temp_bufr,
+		    "Unexpected token, 0x%02x, after end of FCode block.",
+		    token);
+		printremark( temp_bufr);
+	    }
+    }
+    return ( retval );
 }
+
+/* **************************************************************************
+ *
+ *      Function name:  detokenize
+ *      Synopsis:       Detokenize one input file 
+ *                      
+ *
+ *
+ **************************************************************************** */
+
+void detokenize(void)
+{
+    fclen = stream_max;
+
+    if ( setjmp(eof_exception) == 0 )
+    {
+	while ( more_to_go() )
+	{
+	    if ( ended_okay )
+	    {
+		init_fcode_block();
+	    }
+	    ended_okay = TRUE;
+
+	    adjust_for_pci_header();
+
+	    /*   Allow for multiple FCode Blocks within the PCI image.
+	     *   The first one had better be a valid block, but the
+	     *   next may or may not be...
+	     */
+	    do
+	    {
+		decode_fcode_block();
+	    }  while  ( another_fcode_block() );
+
+	    adjust_for_pci_filler();
+
+	}
+    }
+
+
+}

Modified: fcode-utils/detok/detok.c
===================================================================
--- fcode-utils/detok/detok.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/detok.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,58 +24,76 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+
 #ifdef __GLIBC__
 #define _GNU_SOURCE
 #include <getopt.h>
-#else
-/* Some systems seem to have an incomplete unistd.h.
- * We need to define getopt() and optind for them.
- */
-extern int optind;
-int getopt(int argc, char * const argv[], const char *optstring);
 #endif

#include "detok.h"
 #include "stream.h"
+#include "addfcodes.h"

#define DETOK_VERSION "0.6.1"

-/* prototypes for dictionary handling */
-void init_dictionary(void);
-void decode_token(u16 token);
-/* prototype for detokenizer function */
-int  detokenize(void);
+#define IBM_COPYR    "(C) Copyright 2005 IBM Corporation.  All Rights Reserved."

-extern unsigned int decode_all, verbose, linenumbers;
+bool verbose = FALSE ;
+bool decode_all = FALSE ;
+bool show_linenumbers = FALSE ;
+bool show_offsets = FALSE ;

-void print_copyright(void)
+/*   Param is FALSE when beginning to detokenize,
+ *       TRUE preceding error-exit   */
+static void print_copyright(bool is_error)
 {
-        printf( "Welcome to the OpenBIOS detokenizer v%s\ndetok Copyright"
+	typedef void (*vfunct)();  /*  Pointer to function returning void  */
+	vfunct pfunct ;
+	char buffr[512];
+
+	sprintf( buffr,
+		"Welcome to the OpenBIOS detokenizer v%s\ndetok Copyright"
    	"(c) 2001-2005 by Stefan Reinauer.\nWritten by Stefan "
    	"Reinauer, \n" "This program is "
    	"free software; you may redistribute it under the terms of\n"
    	"the GNU General Public License.  This program has absolutely"
    				" no warranty.\n\n" ,DETOK_VERSION);
+
+	pfunct = ( is_error ? (vfunct)printf : printremark );
+
+        (*pfunct) ( buffr );
+
+        (*pfunct) ( IBM_COPYR "\n" );
 }

-void usage(char *name)
+static void usage(char *name)
 {
    printf( "usage: %s [OPTION]... [FCODE-FILE]...\n\n"
    	"         -v, --verbose     print fcode numbers\n"
    	"         -a, --all         don't stop at end0\n"
    	"         -n, --linenumbers print line numbers\n"
    	"         -o, --offsets     print byte offsets\n"
+		"         -f, --fcodes      add FCodes from list-file\n"
    	"         -h, --help        print this help text\n\n", name);
 }

int main(int argc, char **argv)
 {
    int c;
-	const char *optstring="vhano?";
+	const char *optstring="vhanof:?";
+	int linenumbers = 0;
+	bool add_vfcodes = FALSE;
+	char *vfc_filnam = NULL;

while (1) {
 #ifdef __GLIBC__
@@ -86,6 +104,7 @@
    		{ "all", 0, 0, 'a' },
    		{ "linenumbers", 0, 0, 'n' },
    		{ "offsets", 0, 0, 'o' },
+			{ "fcodes", 1, 0, 'f' },
    		{ 0, 0, 0, 0 }
    	};

@@ -99,24 +118,31 @@

switch (c) {
    	case 'v':
-			verbose=1;
+			verbose=TRUE;
    		break;
    	case 'a':
-			decode_all=1;
+			decode_all=TRUE;
    		break;
    	case 'n':
    		linenumbers|=1;
+			show_linenumbers = TRUE;
    		break;
    	case 'o':
    		linenumbers|=2;
+			show_linenumbers = TRUE;
+			show_offsets = TRUE;
    		break;
+		case 'f':
+			add_vfcodes = TRUE;
+			vfc_filnam = optarg;
+			break;
    	case 'h':
    	case '?':
-			print_copyright();
+			print_copyright(TRUE);
    		usage(argv[0]);
    		return 0;		
    	default:
-			print_copyright();
+			print_copyright(TRUE);
    		printf ("%s: unknown option.\n",argv[0]);
    		usage(argv[0]);
    		return 1;
@@ -124,13 +150,14 @@
    }

if (verbose)
-		print_copyright();
+		print_copyright(FALSE);
    
    if (linenumbers>2)
-		printf("Line numbers will be disabled in favour of offsets.\n");
+		printremark(
+		    "Line numbers will be disabled in favour of offsets.\n");
    
    if (optind >= argc) {
-		print_copyright();
+		print_copyright(TRUE);
    	printf ("%s: filename missing.\n",argv[0]);
    	usage(argv[0]);
    	return 1;
@@ -138,6 +165,14 @@
    
    init_dictionary();
    
+	if ( add_vfcodes )
+	{
+	    if ( add_fcodes_from_list( vfc_filnam) )
+	    {
+	        freeze_dictionary();
+	    }
+	}
+
    while (optind < argc) {
    	
    	if (init_stream(argv[optind])) {

Modified: fcode-utils/detok/detok.h
===================================================================
--- fcode-utils/detok/detok.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/detok.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _UTILS_DETOK_DETOK_H
+#define _UTILS_DETOK_DETOK_H
+
 /*
  *                     OpenBIOS - free your system! 
  *                            ( detokenizer )
@@ -24,10 +27,49 @@
  *
  */

-#define u8 unsigned char
-#define u16 unsigned short
-#define u32 unsigned int
-#define s16 short
-#define bool int
-#define TRUE (-1)
-#define FALSE (0)
+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
+#include "types.h"
+
+/*  Prototypes for functions exported from
+ *   detok.c  decode.c  printformats.c  pcihdr.c  and  dictionary.c
+ */
+
+void add_token(u16 number, char *name);
+void init_dictionary(void);
+void reset_dictionary(void);
+void freeze_dictionary(void);
+char *lookup_token(u16 number);
+
+void detokenize(void);
+
+void printremark(char *str);
+
+int handle_pci_header ( u8  *data_ptr );
+void handle_pci_filler(u8 *filler_ptr);
+
+
+/*  External declarations for variables defined in or used by
+ *   detok.c  decode.c  printformats.c  pcihdr.c  and  dictionary.c
+ */
+extern bool verbose;
+extern bool decode_all;
+extern bool show_linenumbers;
+extern bool show_offsets;
+
+extern bool check_tok_seq;
+
+extern u16 fcode;
+extern bool offs16;
+extern bool end_found;
+extern unsigned int linenum;
+
+extern u8 *pci_image_end;
+extern unsigned int token_streampos;
+extern u16 last_defined_token;
+
+#endif   /*  _UTILS_DETOK_DETOK_H    */

Modified: fcode-utils/detok/dictionary.c
===================================================================
--- fcode-utils/detok/dictionary.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/dictionary.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,20 +24,29 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>

#include "detok.h"

+bool check_tok_seq = TRUE;
+
 typedef struct token {
-	u8  *name;
+	char *name;
    u16 fcode;
    struct token *next;
 } token_t;
+#define TOKEN_ENTRY(num, name)   { name, (u16)num, (token_t *)NULL }
+static token_t *dictionary;   /*  Initialize dynamically to accommodate AIX  */

static char *fcerror="ferror";
-token_t *dictionary=NULL;

char *lookup_token(u16 number)
 {
@@ -53,7 +62,51 @@
    return fcerror;
 }

-int add_token(u16 number, char *name)
+/* **************************************************************************
+ *
+ *      Function name:  add_token
+ *      Synopsis:       Add an entry to the FCode-Tokens vocabulary.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             number                      Numeric value of the FCode token
+ *             name                        Name of the function to display
+ *         Global/Static Variables:
+ *             dictionary                  Pointer to the "tail" of the
+ *                                             FCode-Tokens vocabulary.
+ *             check_tok_seq               TRUE = "Check Token Sequence"
+ *                                             A retro-fit to accommodate
+ *                                             adding Vendor FCodes
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Global/Static Variables:
+ *             dictionary                  Updated to point to the new entry.
+ *             last_defined_token          Updated to the given FCode token
+ *         Memory Allocated
+ *             For the new entry.
+ *         When Freed?
+ *             Never.  Retained for duration of the program.
+ *
+ *      Error Detection:
+ *          Failure to allocate memory is a fatal error.
+ *          If the given FCode token is not exactly one larger than the
+ *              previous  last_defined_token , then there's something
+ *              odd going on; print a remark to alert the user.  The
+ *              value of  last_defined_token  will be used elsewhere
+ *              for additional error-checking.
+ *
+ *      Process Explanation:
+ *          The name field pointer is presumed to already point to a stable
+ *              memory-space.
+ *          Memory will be allocated for the entry itself; its fields will
+ *              be entered and the pointer-to-the-tail-of-the-vocabulary
+ *              will be updated to point to the new entry.
+ *          Error-check and update  last_defined_token  
+ *
+ **************************************************************************** */
+
+void add_token(u16 number, char *name)
 {
    token_t *curr;

@@ -68,393 +121,451 @@
    curr->name=name;

dictionary=curr;
-	return 0;
+
+	if ( check_tok_seq )
+	{
+	    /*  Error-check, but not for first time.  */
+	    if ( (number == last_defined_token + 1)
+		   || (last_defined_token == 0) )
+	    {
+		last_defined_token = number;
+	    }else{
+		if ( number <= last_defined_token )
+		{
+		    printremark("Warning:  New token # might overlap "
+			"previously assigned token #(s).");
+		}else{
+		    printremark("Warning:  New token # out of sequence with "
+			"previously assigned token #(s).");
+		    /*  It's increasing; update it.  */
+		    last_defined_token = number;
+		}
 }
+	}

+}
+
+token_t detok_table[]  = {
+
+	TOKEN_ENTRY( 0x000, "end0" ),
+	TOKEN_ENTRY( 0x010, "b(lit)" ),
+	TOKEN_ENTRY( 0x011, "b(')" ),
+	TOKEN_ENTRY( 0x012, "b(")" ),
+	TOKEN_ENTRY( 0x013, "bbranch" ),
+	TOKEN_ENTRY( 0x014, "b?branch" ),
+	TOKEN_ENTRY( 0x015, "b(loop)" ),
+	TOKEN_ENTRY( 0x016, "b(+loop)" ),
+	TOKEN_ENTRY( 0x017, "b(do)" ),
+	TOKEN_ENTRY( 0x018, "b(?do)" ),
+	TOKEN_ENTRY( 0x019, "i" ),
+	TOKEN_ENTRY( 0x01a, "j" ),
+	TOKEN_ENTRY( 0x01b, "b(leave)" ),
+	TOKEN_ENTRY( 0x01c, "b(of)" ),
+	TOKEN_ENTRY( 0x01d, "execute" ),
+	TOKEN_ENTRY( 0x01e, "+" ),
+	TOKEN_ENTRY( 0x01f, "-" ),
+	TOKEN_ENTRY( 0x020, "*" ),
+	TOKEN_ENTRY( 0x021, "/" ),
+	TOKEN_ENTRY( 0x022, "mod" ),
+	TOKEN_ENTRY( 0x023, "and" ),
+	TOKEN_ENTRY( 0x024, "or" ),
+	TOKEN_ENTRY( 0x025, "xor" ),
+	TOKEN_ENTRY( 0x026, "invert" ),
+	TOKEN_ENTRY( 0x027, "lshift" ),
+	TOKEN_ENTRY( 0x028, "rshift" ),
+	TOKEN_ENTRY( 0x029, ">>a" ),
+	TOKEN_ENTRY( 0x02a, "/mod" ),
+	TOKEN_ENTRY( 0x02b, "u/mod" ),
+	TOKEN_ENTRY( 0x02c, "negate" ),
+	TOKEN_ENTRY( 0x02d, "abs" ),
+	TOKEN_ENTRY( 0x02e, "min" ),
+	TOKEN_ENTRY( 0x02f, "max" ),
+	TOKEN_ENTRY( 0x030, ">r" ),
+	TOKEN_ENTRY( 0x031, "r>" ),
+	TOKEN_ENTRY( 0x032, "r@" ),
+	TOKEN_ENTRY( 0x033, "exit" ),
+	TOKEN_ENTRY( 0x034, "0=" ),
+	TOKEN_ENTRY( 0x035, "0<>" ),
+	TOKEN_ENTRY( 0x036, "0<" ),
+	TOKEN_ENTRY( 0x037, "0<=" ),
+	TOKEN_ENTRY( 0x038, "0>" ),
+	TOKEN_ENTRY( 0x039, "0>=" ),
+	TOKEN_ENTRY( 0x03a, "<" ),
+	TOKEN_ENTRY( 0x03b, ">" ),
+	TOKEN_ENTRY( 0x03c, "=" ),
+	TOKEN_ENTRY( 0x03d, "<>" ),
+	TOKEN_ENTRY( 0x03e, "u>" ),
+	TOKEN_ENTRY( 0x03f, "u<=" ),
+	TOKEN_ENTRY( 0x040, "u<" ),
+	TOKEN_ENTRY( 0x041, "u>=" ),
+	TOKEN_ENTRY( 0x042, ">=" ),
+	TOKEN_ENTRY( 0x043, "<=" ),
+	TOKEN_ENTRY( 0x044, "between" ),
+	TOKEN_ENTRY( 0x045, "within" ),
+	TOKEN_ENTRY( 0x046, "drop" ),
+	TOKEN_ENTRY( 0x047, "dup" ),
+	TOKEN_ENTRY( 0x048, "over" ),
+	TOKEN_ENTRY( 0x049, "swap" ),
+	TOKEN_ENTRY( 0x04A, "rot" ),
+	TOKEN_ENTRY( 0x04b, "-rot" ),
+	TOKEN_ENTRY( 0x04c, "tuck" ),
+	TOKEN_ENTRY( 0x04d, "nip" ),
+	TOKEN_ENTRY( 0x04e, "pick" ),
+	TOKEN_ENTRY( 0x04f, "roll" ),
+	TOKEN_ENTRY( 0x050, "?dup" ),
+	TOKEN_ENTRY( 0x051, "depth" ),
+	TOKEN_ENTRY( 0x052, "2drop" ),
+	TOKEN_ENTRY( 0x053, "2dup" ),
+	TOKEN_ENTRY( 0x054, "2over" ),
+	TOKEN_ENTRY( 0x055, "2swap" ),
+	TOKEN_ENTRY( 0x056, "2rot" ),
+	TOKEN_ENTRY( 0x057, "2/" ),
+	TOKEN_ENTRY( 0x058, "u2/" ),
+	TOKEN_ENTRY( 0x059, "2*" ),
+	TOKEN_ENTRY( 0x05a, "/c" ),
+	TOKEN_ENTRY( 0x05b, "/w" ),
+	TOKEN_ENTRY( 0x05c, "/l" ),
+	TOKEN_ENTRY( 0x05d, "/n" ),
+	TOKEN_ENTRY( 0x05e, "ca+" ),
+	TOKEN_ENTRY( 0x05f, "wa+" ),
+	TOKEN_ENTRY( 0x060, "la+" ),
+	TOKEN_ENTRY( 0x061, "na+" ),
+	TOKEN_ENTRY( 0x062, "char+" ),
+	TOKEN_ENTRY( 0x063, "wa1+" ),
+	TOKEN_ENTRY( 0x064, "la1+" ),
+	TOKEN_ENTRY( 0x065, "cell+" ),
+	TOKEN_ENTRY( 0x066, "chars" ),
+	TOKEN_ENTRY( 0x067, "/w*" ),
+	TOKEN_ENTRY( 0x068, "/l*" ),
+	TOKEN_ENTRY( 0x069, "cells" ),
+	TOKEN_ENTRY( 0x06a, "on" ),
+	TOKEN_ENTRY( 0x06b, "off" ),
+	TOKEN_ENTRY( 0x06c, "+!" ),
+	TOKEN_ENTRY( 0x06d, "@" ),
+	TOKEN_ENTRY( 0x06e, "l@" ),
+	TOKEN_ENTRY( 0x06f, "w@" ),
+	TOKEN_ENTRY( 0x070, "<w@" ),
+	TOKEN_ENTRY( 0x071, "c@" ),
+	TOKEN_ENTRY( 0x072, "!" ),
+	TOKEN_ENTRY( 0x073, "l!" ),
+	TOKEN_ENTRY( 0x074, "w!" ),
+	TOKEN_ENTRY( 0x075, "c!" ),
+	TOKEN_ENTRY( 0x076, "2@" ),
+	TOKEN_ENTRY( 0x077, "2!" ),
+	TOKEN_ENTRY( 0x078, "move" ),
+	TOKEN_ENTRY( 0x079, "fill" ),
+	TOKEN_ENTRY( 0x07a, "comp" ),
+	TOKEN_ENTRY( 0x07b, "noop" ),
+	TOKEN_ENTRY( 0x07c, "lwsplit" ),
+	TOKEN_ENTRY( 0x07d, "wljoin" ),
+	TOKEN_ENTRY( 0x07e, "lbsplit" ),
+	TOKEN_ENTRY( 0x07f, "bljoin" ),
+	TOKEN_ENTRY( 0x080, "wbflip" ),
+	TOKEN_ENTRY( 0x081, "upc" ),
+	TOKEN_ENTRY( 0x082, "lcc" ),
+	TOKEN_ENTRY( 0x083, "pack" ),
+	TOKEN_ENTRY( 0x084, "count" ),
+	TOKEN_ENTRY( 0x085, "body>" ),
+	TOKEN_ENTRY( 0x086, ">body" ),
+	TOKEN_ENTRY( 0x087, "fcode-revision" ),
+	TOKEN_ENTRY( 0x088, "span" ),
+	TOKEN_ENTRY( 0x089, "unloop" ),
+	TOKEN_ENTRY( 0x08a, "expect" ),
+	TOKEN_ENTRY( 0x08b, "alloc-mem" ),
+	TOKEN_ENTRY( 0x08c, "free-mem" ),
+	TOKEN_ENTRY( 0x08d, "key?" ),
+	TOKEN_ENTRY( 0x08e, "key" ),
+	TOKEN_ENTRY( 0x08f, "emit" ),
+	TOKEN_ENTRY( 0x090, "type" ),
+	TOKEN_ENTRY( 0x091, "(cr" ),
+	TOKEN_ENTRY( 0x092, "cr" ),
+	TOKEN_ENTRY( 0x093, "#out" ),
+	TOKEN_ENTRY( 0x094, "#line" ),
+	TOKEN_ENTRY( 0x095, "hold" ),
+	TOKEN_ENTRY( 0x096, "<#" ),
+	TOKEN_ENTRY( 0x097, "u#>" ),
+	TOKEN_ENTRY( 0x098, "sign" ),
+	TOKEN_ENTRY( 0x099, "u#" ),
+	TOKEN_ENTRY( 0x09a, "u#s" ),
+	TOKEN_ENTRY( 0x09b, "u." ),
+	TOKEN_ENTRY( 0x09c, "u.r" ),
+	TOKEN_ENTRY( 0x09d, "." ),
+	TOKEN_ENTRY( 0x09e, ".r" ),
+	TOKEN_ENTRY( 0x09f, ".s" ),
+	TOKEN_ENTRY( 0x0a0, "base" ),
+	TOKEN_ENTRY( 0x0a1, "convert" ),
+	TOKEN_ENTRY( 0x0a2, "$number" ),
+	TOKEN_ENTRY( 0x0a3, "digit" ),
+	TOKEN_ENTRY( 0x0a4, "-1" ),
+	TOKEN_ENTRY( 0x0a5, "0" ),
+	TOKEN_ENTRY( 0x0a6, "1" ),
+	TOKEN_ENTRY( 0x0a7, "2" ),
+	TOKEN_ENTRY( 0x0a8, "3" ),
+	TOKEN_ENTRY( 0x0a9, "bl" ),
+	TOKEN_ENTRY( 0x0aa, "bs" ),
+	TOKEN_ENTRY( 0x0ab, "bell" ),
+	TOKEN_ENTRY( 0x0ac, "bounds" ),
+	TOKEN_ENTRY( 0x0ad, "here" ),
+	TOKEN_ENTRY( 0x0ae, "aligned" ),
+	TOKEN_ENTRY( 0x0af, "wbsplit" ),
+	TOKEN_ENTRY( 0x0b0, "bwjoin" ),
+	TOKEN_ENTRY( 0x0b1, "b(<mark)" ),
+	TOKEN_ENTRY( 0x0b2, "b(>resolve)" ),
+	TOKEN_ENTRY( 0x0b3, "set-token-table" ),
+	TOKEN_ENTRY( 0x0b4, "set-table" ),
+	TOKEN_ENTRY( 0x0b5, "new-token" ),
+	TOKEN_ENTRY( 0x0b6, "named-token" ),
+	TOKEN_ENTRY( 0x0b7, "b(:)" ),
+	TOKEN_ENTRY( 0x0b8, "b(value)" ),
+	TOKEN_ENTRY( 0x0b9, "b(variable)" ),
+	TOKEN_ENTRY( 0x0ba, "b(constant)" ),
+	TOKEN_ENTRY( 0x0bb, "b(create)" ),
+	TOKEN_ENTRY( 0x0bc, "b(defer)" ),
+	TOKEN_ENTRY( 0x0bd, "b(buffer:)" ),
+	TOKEN_ENTRY( 0x0be, "b(field)" ),
+	TOKEN_ENTRY( 0x0bf, "b(code)" ),
+	TOKEN_ENTRY( 0x0c0, "instance" ),
+	TOKEN_ENTRY( 0x0c2, "b(;)" ),
+	TOKEN_ENTRY( 0x0c3, "b(to)" ),
+	TOKEN_ENTRY( 0x0c4, "b(case)" ),
+	TOKEN_ENTRY( 0x0c5, "b(endcase)" ),
+	TOKEN_ENTRY( 0x0c6, "b(endof)" ),
+	TOKEN_ENTRY( 0x0c7, "#" ),
+	TOKEN_ENTRY( 0x0c8, "#s" ),
+	TOKEN_ENTRY( 0x0c9, "#>" ),
+	TOKEN_ENTRY( 0x0ca, "external-token" ),
+	TOKEN_ENTRY( 0x0cb, "$find" ),
+	TOKEN_ENTRY( 0x0cc, "offset16" ),
+	TOKEN_ENTRY( 0x0cd, "evaluate" ),
+	TOKEN_ENTRY( 0x0d0, "c," ),
+	TOKEN_ENTRY( 0x0d1, "w," ),
+	TOKEN_ENTRY( 0x0d2, "l," ),
+	TOKEN_ENTRY( 0x0d3, "," ),
+	TOKEN_ENTRY( 0x0d4, "um*" ),
+	TOKEN_ENTRY( 0x0d5, "um/mod" ),
+	TOKEN_ENTRY( 0x0d8, "d+" ),
+	TOKEN_ENTRY( 0x0d9, "d-" ),
+	TOKEN_ENTRY( 0x0da, "get-token" ),
+	TOKEN_ENTRY( 0x0db, "set-token" ),
+	TOKEN_ENTRY( 0x0dc, "state" ),
+	TOKEN_ENTRY( 0x0dd, "compile" ),
+	TOKEN_ENTRY( 0x0de, "behavior" ),
+	TOKEN_ENTRY( 0x0f0, "start0" ),
+	TOKEN_ENTRY( 0x0f1, "start1" ),
+	TOKEN_ENTRY( 0x0f2, "start2" ),
+	TOKEN_ENTRY( 0x0f3, "start4" ),
+	TOKEN_ENTRY( 0x0fc, "ferror" ),
+	TOKEN_ENTRY( 0x0fd, "version1" ),
+	TOKEN_ENTRY( 0x0fe, "4-byte-id" ),
+	TOKEN_ENTRY( 0x0ff, "end1" ),
+	TOKEN_ENTRY( 0x101, "dma-alloc" ),
+	TOKEN_ENTRY( 0x102, "my-address" ),
+	TOKEN_ENTRY( 0x103, "my-space" ),
+	TOKEN_ENTRY( 0x104, "memmap" ),
+	TOKEN_ENTRY( 0x105, "free-virtual" ),
+	TOKEN_ENTRY( 0x106, ">physical" ),
+	TOKEN_ENTRY( 0x10f, "my-params" ),
+	TOKEN_ENTRY( 0x110, "property" ),
+	TOKEN_ENTRY( 0x111, "encode-int" ),
+	TOKEN_ENTRY( 0x112, "encode+" ),
+	TOKEN_ENTRY( 0x113, "encode-phys" ),
+	TOKEN_ENTRY( 0x114, "encode-string" ),
+	TOKEN_ENTRY( 0x115, "encode-bytes" ),
+	TOKEN_ENTRY( 0x116, "reg" ),
+	TOKEN_ENTRY( 0x117, "intr" ),
+	TOKEN_ENTRY( 0x118, "driver" ),
+	TOKEN_ENTRY( 0x119, "model" ),
+	TOKEN_ENTRY( 0x11a, "device-type" ),
+	TOKEN_ENTRY( 0x11b, "parse-2int" ),
+	TOKEN_ENTRY( 0x11c, "is-install" ),
+	TOKEN_ENTRY( 0x11d, "is-remove" ),
+	TOKEN_ENTRY( 0x11e, "is-selftest" ),
+	TOKEN_ENTRY( 0x11f, "new-device" ),
+	TOKEN_ENTRY( 0x120, "diagnostic-mode?" ),
+	TOKEN_ENTRY( 0x121, "display-status" ),
+	TOKEN_ENTRY( 0x122, "memory-test-issue" ),
+	TOKEN_ENTRY( 0x123, "group-code" ),
+	TOKEN_ENTRY( 0x124, "mask" ),
+	TOKEN_ENTRY( 0x125, "get-msecs" ),
+	TOKEN_ENTRY( 0x126, "ms" ),
+	TOKEN_ENTRY( 0x127, "finish-device" ),
+	TOKEN_ENTRY( 0x128, "decode-phys" ),
+	TOKEN_ENTRY( 0x12b, "interpose" ),
+	TOKEN_ENTRY( 0x130, "map-low" ),
+	TOKEN_ENTRY( 0x131, "sbus-intr>cpu" ),
+	TOKEN_ENTRY( 0x150, "#lines" ),
+	TOKEN_ENTRY( 0x151, "#columns" ),
+	TOKEN_ENTRY( 0x152, "line#" ),
+	TOKEN_ENTRY( 0x153, "column#" ),
+	TOKEN_ENTRY( 0x154, "inverse?" ),
+	TOKEN_ENTRY( 0x155, "inverse-screen?" ),
+	TOKEN_ENTRY( 0x156, "frame-buffer-busy?" ),
+	TOKEN_ENTRY( 0x157, "draw-character" ),
+	TOKEN_ENTRY( 0x158, "reset-screen" ),
+	TOKEN_ENTRY( 0x159, "toggle-cursor" ),
+	TOKEN_ENTRY( 0x15a, "erase-screen" ),
+	TOKEN_ENTRY( 0x15b, "blink-screen" ),
+	TOKEN_ENTRY( 0x15c, "invert-screen" ),
+	TOKEN_ENTRY( 0x15d, "insert-characters" ),
+	TOKEN_ENTRY( 0x15e, "delete-characters" ),
+	TOKEN_ENTRY( 0x15f, "insert-lines" ),
+	TOKEN_ENTRY( 0x160, "delete-lines" ),
+	TOKEN_ENTRY( 0x161, "draw-logo" ),
+	TOKEN_ENTRY( 0x162, "frame-buffer-adr" ),
+	TOKEN_ENTRY( 0x163, "screen-height" ),
+	TOKEN_ENTRY( 0x164, "screen-width" ),
+	TOKEN_ENTRY( 0x165, "window-top" ),
+	TOKEN_ENTRY( 0x166, "window-left" ),
+	TOKEN_ENTRY( 0x16a, "default-font" ),
+	TOKEN_ENTRY( 0x16b, "set-font" ),
+	TOKEN_ENTRY( 0x16c, "char-height" ),
+	TOKEN_ENTRY( 0x16d, "char-width" ),
+	TOKEN_ENTRY( 0x16e, ">font" ),
+	TOKEN_ENTRY( 0x16f, "fontbytes" ),
+	TOKEN_ENTRY( 0x170, "fb1-draw-character" ),
+	TOKEN_ENTRY( 0x171, "fb1-reset-screen" ),
+	TOKEN_ENTRY( 0x172, "fb1-toggle-cursor" ),
+	TOKEN_ENTRY( 0x173, "fb1-erase-screen" ),
+	TOKEN_ENTRY( 0x174, "fb1-blink-screen" ),
+	TOKEN_ENTRY( 0x175, "fb1-invert-screen" ),
+	TOKEN_ENTRY( 0x176, "fb1-insert-characters" ),
+	TOKEN_ENTRY( 0x177, "fb1-delete-characters" ),
+	TOKEN_ENTRY( 0x178, "fb1-insert-lines" ),
+	TOKEN_ENTRY( 0x179, "fb1-delete-lines" ),
+	TOKEN_ENTRY( 0x17a, "fb1-draw-logo" ),
+	TOKEN_ENTRY( 0x17b, "fb1-install" ),
+	TOKEN_ENTRY( 0x17c, "fb1-slide-up" ),
+	TOKEN_ENTRY( 0x180, "fb8-draw-character" ),
+	TOKEN_ENTRY( 0x181, "fb8-reset-screen" ),
+	TOKEN_ENTRY( 0x182, "fb8-toggle-cursor" ),
+	TOKEN_ENTRY( 0x183, "fb8-erase-screen" ),
+	TOKEN_ENTRY( 0x184, "fb8-blink-screen" ),
+	TOKEN_ENTRY( 0x185, "fb8-invert-screen" ),
+	TOKEN_ENTRY( 0x186, "fb8-insert-characters" ),
+	TOKEN_ENTRY( 0x187, "fb8-delete-characters" ),
+	TOKEN_ENTRY( 0x188, "fb8-insert-lines" ),
+	TOKEN_ENTRY( 0x189, "fb8-delete-lines" ),
+	TOKEN_ENTRY( 0x18a, "fb8-draw-logo" ),
+	TOKEN_ENTRY( 0x18b, "fb8-install" ),
+	TOKEN_ENTRY( 0x1a0, "return-buffer" ),
+	TOKEN_ENTRY( 0x1a1, "xmit-packet" ),
+	TOKEN_ENTRY( 0x1a2, "poll-packet" ),
+	TOKEN_ENTRY( 0x1a4, "mac-address" ),
+	TOKEN_ENTRY( 0x201, "device-name" ),
+	TOKEN_ENTRY( 0x202, "my-args" ),
+	TOKEN_ENTRY( 0x203, "my-self" ),
+	TOKEN_ENTRY( 0x204, "find-package" ),
+	TOKEN_ENTRY( 0x205, "open-package" ),
+	TOKEN_ENTRY( 0x206, "close-package" ),
+	TOKEN_ENTRY( 0x207, "find-method" ),
+	TOKEN_ENTRY( 0x208, "call-package" ),
+	TOKEN_ENTRY( 0x209, "$call-parent" ),
+	TOKEN_ENTRY( 0x20a, "my-parent" ),
+	TOKEN_ENTRY( 0x20b, "ihandle>phandle" ),
+	TOKEN_ENTRY( 0x20d, "my-unit" ),
+	TOKEN_ENTRY( 0x20e, "$call-method" ),
+	TOKEN_ENTRY( 0x20f, "$open-package" ),
+	TOKEN_ENTRY( 0x210, "processor-type" ),
+	TOKEN_ENTRY( 0x211, "firmware-version" ),
+	TOKEN_ENTRY( 0x212, "fcode-version" ),
+	TOKEN_ENTRY( 0x213, "alarm" ),
+	TOKEN_ENTRY( 0x214, "(is-user-word)" ),
+	TOKEN_ENTRY( 0x215, "suspend-fcode" ),
+	TOKEN_ENTRY( 0x216, "abort" ),
+	TOKEN_ENTRY( 0x217, "catch" ),
+	TOKEN_ENTRY( 0x218, "throw" ),
+	TOKEN_ENTRY( 0x219, "user-abort" ),
+	TOKEN_ENTRY( 0x21a, "get-my-property" ),
+	TOKEN_ENTRY( 0x21b, "decode-int" ),
+	TOKEN_ENTRY( 0x21c, "decode-string" ),
+	TOKEN_ENTRY( 0x21d, "get-inherited-property" ),
+	TOKEN_ENTRY( 0x21e, "delete-property" ),
+	TOKEN_ENTRY( 0x21f, "get-package-property" ),
+	TOKEN_ENTRY( 0x220, "cpeek" ),
+	TOKEN_ENTRY( 0x221, "wpeek" ),
+	TOKEN_ENTRY( 0x222, "lpeek" ),
+	TOKEN_ENTRY( 0x223, "cpoke" ),
+	TOKEN_ENTRY( 0x224, "wpoke" ),
+	TOKEN_ENTRY( 0x225, "lpoke" ),
+	TOKEN_ENTRY( 0x226, "lwflip" ),
+	TOKEN_ENTRY( 0x227, "lbflip" ),
+	TOKEN_ENTRY( 0x228, "lbflips" ),
+	TOKEN_ENTRY( 0x229, "adr-mask" ),
+	TOKEN_ENTRY( 0x230, "rb@" ),
+	TOKEN_ENTRY( 0x231, "rb!" ),
+	TOKEN_ENTRY( 0x232, "rw@" ),
+	TOKEN_ENTRY( 0x233, "rw!" ),
+	TOKEN_ENTRY( 0x234, "rl@" ),
+	TOKEN_ENTRY( 0x235, "rl!" ),
+	TOKEN_ENTRY( 0x236, "wbflips" ),
+	TOKEN_ENTRY( 0x237, "lwflips" ),
+	TOKEN_ENTRY( 0x238, "probe" ),
+	TOKEN_ENTRY( 0x239, "probe-virtual" ),
+	TOKEN_ENTRY( 0x23b, "child" ),
+	TOKEN_ENTRY( 0x23c, "peer" ),
+	TOKEN_ENTRY( 0x23d, "next-property" ),
+	TOKEN_ENTRY( 0x23e, "byte-load" ),
+	TOKEN_ENTRY( 0x23f, "set-args" ),
+	TOKEN_ENTRY( 0x240, "left-parse-string" ),
+
+	/* FCodes from 64bit extension addendum */
+	TOKEN_ENTRY( 0x22e, "rx@" ),
+	TOKEN_ENTRY( 0x22f, "rx!" ),
+	TOKEN_ENTRY( 0x241, "bxjoin" ),
+	TOKEN_ENTRY( 0x242, "<l@" ),
+	TOKEN_ENTRY( 0x243, "lxjoin" ),
+	TOKEN_ENTRY( 0x244, "wxjoin" ),
+	TOKEN_ENTRY( 0x245, "x," ),
+	TOKEN_ENTRY( 0x246, "x@" ),
+	TOKEN_ENTRY( 0x247, "x!" ),
+	TOKEN_ENTRY( 0x248, "/x" ),
+	TOKEN_ENTRY( 0x249, "/x*" ),
+	TOKEN_ENTRY( 0x24a, "xa+" ),
+	TOKEN_ENTRY( 0x24b, "xa1+" ),
+	TOKEN_ENTRY( 0x24c, "xbflip" ),
+	TOKEN_ENTRY( 0x24d, "xbflips" ),
+	TOKEN_ENTRY( 0x24e, "xbsplit" ),
+	TOKEN_ENTRY( 0x24f, "xlflip" ),
+	TOKEN_ENTRY( 0x250, "xlflips" ),
+	TOKEN_ENTRY( 0x251, "xlsplit" ),
+	TOKEN_ENTRY( 0x252, "xwflip" ),
+	TOKEN_ENTRY( 0x253, "xwflips" ),
+	TOKEN_ENTRY( 0x254, "xwsplit" ),
+};
+
+static const int dictionary_indx_max = (sizeof(detok_table)/sizeof(token_t)) ;
+
+static token_t *dictionary_reset_position;
+
 void init_dictionary(void) 
 {
-	add_token( 0x000, "end0" );
-	add_token( 0x010, "b(lit)" );
-	add_token( 0x011, "b(')" );
-	add_token( 0x012, "b(")" );
-	add_token( 0x013, "bbranch" );
-	add_token( 0x014, "b?branch" );
-	add_token( 0x015, "b(loop)" );
-	add_token( 0x016, "b(+loop)" );
-	add_token( 0x017, "b(do)" );
-	add_token( 0x018, "b(?do)" );
-	add_token( 0x019, "i" );
-	add_token( 0x01a, "j" );
-	add_token( 0x01b, "b(leave)" );
-	add_token( 0x01c, "b(of)" );
-	add_token( 0x01d, "execute" );
-	add_token( 0x01e, "+" );
-	add_token( 0x01f, "-" );
-	add_token( 0x020, "*" );
-	add_token( 0x021, "/" );
-	add_token( 0x022, "mod" );
-	add_token( 0x023, "and" );
-	add_token( 0x024, "or" );
-	add_token( 0x025, "xor" );
-	add_token( 0x026, "invert" );
-	add_token( 0x027, "lshift" );
-	add_token( 0x028, "rshift" );
-	add_token( 0x029, ">>a" );
-	add_token( 0x02a, "/mod" );
-	add_token( 0x02b, "u/mod" );
-	add_token( 0x02c, "negate" );
-	add_token( 0x02d, "abs" );
-	add_token( 0x02e, "min" );
-	add_token( 0x02f, "max" );
-	add_token( 0x030, ">r" );
-	add_token( 0x031, "r>" );
-	add_token( 0x032, "r@" );
-	add_token( 0x033, "exit" );
-	add_token( 0x034, "0=" );
-	add_token( 0x035, "0<>" );
-	add_token( 0x036, "0<" );
-	add_token( 0x037, "0<=" );
-	add_token( 0x038, "0>" );
-	add_token( 0x039, "0>=" );
-	add_token( 0x03a, "<" );
-	add_token( 0x03b, ">" );
-	add_token( 0x03c, "=" );
-	add_token( 0x03d, "<>" );
-	add_token( 0x03e, "u>" );
-	add_token( 0x03f, "u<=" );
-	add_token( 0x040, "u<" );
-	add_token( 0x041, "u>=" );
-	add_token( 0x042, ">=" );
-	add_token( 0x043, "<=" );
-	add_token( 0x044, "between" );
-	add_token( 0x045, "within" );
-	add_token( 0x046, "drop" );
-	add_token( 0x047, "dup" );
-	add_token( 0x048, "over" );
-	add_token( 0x049, "swap" );
-	add_token( 0x04A, "rot" );
-	add_token( 0x04b, "-rot" );
-	add_token( 0x04c, "tuck" );
-	add_token( 0x04d, "nip" );
-	add_token( 0x04e, "pick" );
-	add_token( 0x04f, "roll" );
-	add_token( 0x050, "?dup" );
-	add_token( 0x051, "depth" );
-	add_token( 0x052, "2drop" );
-	add_token( 0x053, "2dup" );
-	add_token( 0x054, "2over" );
-	add_token( 0x055, "2swap" );
-	add_token( 0x056, "2rot" );
-	add_token( 0x057, "2/" );
-	add_token( 0x058, "u2/" );
-	add_token( 0x059, "2*" );
-	add_token( 0x05a, "/c" );
-	add_token( 0x05b, "/w" );
-	add_token( 0x05c, "/l" );
-	add_token( 0x05d, "/n" );
-	add_token( 0x05e, "ca+" );
-	add_token( 0x05f, "wa+" );
-	add_token( 0x060, "la+" );
-	add_token( 0x061, "na+" );
-	add_token( 0x062, "char+" );
-	add_token( 0x063, "wa1+" );
-	add_token( 0x064, "la1+" );
-	add_token( 0x065, "cell+" );
-	add_token( 0x066, "chars" );
-	add_token( 0x067, "/w*" );
-	add_token( 0x068, "/l*" );
-	add_token( 0x069, "cells" );
-	add_token( 0x06a, "on" );
-	add_token( 0x06b, "off" );
-	add_token( 0x06c, "+!" );
-	add_token( 0x06d, "@" );
-	add_token( 0x06e, "l@" );
-	add_token( 0x06f, "w@" );
-	add_token( 0x070, "<w@" );
-	add_token( 0x071, "c@" );
-	add_token( 0x072, "!" );
-	add_token( 0x073, "l!" );
-	add_token( 0x074, "w!" );
-	add_token( 0x075, "c!" );
-	add_token( 0x076, "2@" );
-	add_token( 0x077, "2!" );
-	add_token( 0x078, "move" );
-	add_token( 0x079, "fill" );
-	add_token( 0x07a, "comp" );
-	add_token( 0x07b, "noop" );
-	add_token( 0x07c, "lwsplit" );
-	add_token( 0x07d, "wljoin" );
-	add_token( 0x07e, "lbsplit" );
-	add_token( 0x07f, "bljoin" );
-	add_token( 0x080, "wbflip" );
-	add_token( 0x081, "upc" );
-	add_token( 0x082, "lcc" );
-	add_token( 0x083, "pack" );
-	add_token( 0x084, "count" );
-	add_token( 0x085, "body>" );
-	add_token( 0x086, ">body" );
-	add_token( 0x087, "fcode-revision" );
-	add_token( 0x088, "span" );
-	add_token( 0x089, "unloop" );
-	add_token( 0x08a, "expect" );
-	add_token( 0x08b, "alloc-mem" );
-	add_token( 0x08c, "free-mem" );
-	add_token( 0x08d, "key?" );
-	add_token( 0x08e, "key" );
-	add_token( 0x08f, "emit" );
-	add_token( 0x090, "type" );
-	add_token( 0x091, "(cr" );
-	add_token( 0x092, "cr" );
-	add_token( 0x093, "#out" );
-	add_token( 0x094, "#line" );
-	add_token( 0x095, "hold" );
-	add_token( 0x096, "<#" );
-	add_token( 0x097, "u#>" );
-	add_token( 0x098, "sign" );
-	add_token( 0x099, "u#" );
-	add_token( 0x09a, "u#s" );
-	add_token( 0x09b, "u." );
-	add_token( 0x09c, "u.r" );
-	add_token( 0x09d, "." );
-	add_token( 0x09e, ".r" );
-	add_token( 0x09f, ".s" );
-	add_token( 0x0a0, "base" );
-	add_token( 0x0a1, "convert" );
-	add_token( 0x0a2, "$number" );
-	add_token( 0x0a3, "digit" );
-	add_token( 0x0a4, "-1" );
-	add_token( 0x0a5, "0" );
-	add_token( 0x0a6, "1" );
-	add_token( 0x0a7, "2" );
-	add_token( 0x0a8, "3" );
-	add_token( 0x0a9, "bl" );
-	add_token( 0x0aa, "bs" );
-	add_token( 0x0ab, "bell" );
-	add_token( 0x0ac, "bounds" );
-	add_token( 0x0ad, "here" );
-	add_token( 0x0ae, "aligned" );
-	add_token( 0x0af, "wbsplit" );
-	add_token( 0x0b0, "bwjoin" );
-	add_token( 0x0b1, "b(<mark)" );
-	add_token( 0x0b2, "b(>resolve)" );
-	add_token( 0x0b3, "set-token-table" );
-	add_token( 0x0b4, "set-table" );
-	add_token( 0x0b5, "new-token" );
-	add_token( 0x0b6, "named-token" );
-	add_token( 0x0b7, "b(:)" );
-	add_token( 0x0b8, "b(value)" );
-	add_token( 0x0b9, "b(variable)" );
-	add_token( 0x0ba, "b(constant)" );
-	add_token( 0x0bb, "b(create)" );
-	add_token( 0x0bc, "b(defer)" );
-	add_token( 0x0bd, "b(buffer:)" );
-	add_token( 0x0be, "b(field)" );
-	add_token( 0x0bf, "b(code)" );
-	add_token( 0x0c0, "instance" );
-	add_token( 0x0c2, "b(;)" );
-	add_token( 0x0c3, "b(to)" );
-	add_token( 0x0c4, "b(case)" );
-	add_token( 0x0c5, "b(endcase)" );
-	add_token( 0x0c6, "b(endof)" );
-	add_token( 0x0c7, "#" );
-	add_token( 0x0c8, "#s" );
-	add_token( 0x0c9, "#>" );
-	add_token( 0x0ca, "external-token" );
-	add_token( 0x0cb, "$find" );
-	add_token( 0x0cc, "offset16" );
-	add_token( 0x0cd, "evaluate" );
-	add_token( 0x0d0, "c," );
-	add_token( 0x0d1, "w," );
-	add_token( 0x0d2, "l," );
-	add_token( 0x0d3, "," );
-	add_token( 0x0d4, "um*" );
-	add_token( 0x0d5, "um/mod" );
-	add_token( 0x0d8, "d+" );
-	add_token( 0x0d9, "d-" );
-	add_token( 0x0da, "get-token" );
-	add_token( 0x0db, "set-token" );
-	add_token( 0x0dc, "state" );
-	add_token( 0x0dd, "compile" );
-	add_token( 0x0de, "behavior" );
-	add_token( 0x0f0, "start0" );
-	add_token( 0x0f1, "start1" );
-	add_token( 0x0f2, "start2" );
-	add_token( 0x0f3, "start4" );
-	add_token( 0x0fc, "ferror" );
-	add_token( 0x0fd, "version1" );
-	add_token( 0x0fe, "4-byte-id" );
-	add_token( 0x0ff, "end1" );
-	add_token( 0x101, "dma-alloc" );
-	add_token( 0x102, "my-address" );
-	add_token( 0x103, "my-space" );
-	add_token( 0x104, "memmap" );
-	add_token( 0x105, "free-virtual" );
-	add_token( 0x106, ">physical" );
-	add_token( 0x10f, "my-params" );
-	add_token( 0x110, "property" );
-	add_token( 0x111, "encode-int" );
-	add_token( 0x112, "encode+" );
-	add_token( 0x113, "encode-phys" );
-	add_token( 0x114, "encode-string" );
-	add_token( 0x115, "encode-bytes" );
-	add_token( 0x116, "reg" );
-	add_token( 0x117, "intr" );
-	add_token( 0x118, "driver" );
-	add_token( 0x119, "model" );
-	add_token( 0x11a, "device-type" );
-	add_token( 0x11b, "parse-2int" );
-	add_token( 0x11c, "is-install" );
-	add_token( 0x11d, "is-remove" );
-	add_token( 0x11e, "is-selftest" );
-	add_token( 0x11f, "new-device" );
-	add_token( 0x120, "diagnostic-mode?" );
-	add_token( 0x121, "display-status" );
-	add_token( 0x122, "memory-test-issue" );
-	add_token( 0x123, "group-code" );
-	add_token( 0x124, "mask" );
-	add_token( 0x125, "get-msecs" );
-	add_token( 0x126, "ms" );
-	add_token( 0x127, "finish-device" );
-	add_token( 0x128, "decode-phys" );
-	add_token( 0x12b, "interpose" );
-	add_token( 0x130, "map-low" );
-	add_token( 0x131, "sbus-intr>cpu" );
-	add_token( 0x150, "#lines" );
-	add_token( 0x151, "#columns" );
-	add_token( 0x152, "line#" );
-	add_token( 0x153, "column#" );
-	add_token( 0x154, "inverse?" );
-	add_token( 0x155, "inverse-screen?" );
-	add_token( 0x156, "frame-buffer-busy?" );
-	add_token( 0x157, "draw-character" );
-	add_token( 0x158, "reset-screen" );
-	add_token( 0x159, "toggle-cursor" );
-	add_token( 0x15a, "erase-screen" );
-	add_token( 0x15b, "blink-screen" );
-	add_token( 0x15c, "invert-screen" );
-	add_token( 0x15d, "insert-characters" );
-	add_token( 0x15e, "delete-characters" );
-	add_token( 0x15f, "insert-lines" );
-	add_token( 0x160, "delete-lines" );
-	add_token( 0x161, "draw-logo" );
-	add_token( 0x162, "frame-buffer-adr" );
-	add_token( 0x163, "screen-height" );
-	add_token( 0x164, "screen-width" );
-	add_token( 0x165, "window-top" );
-	add_token( 0x166, "window-left" );
-	add_token( 0x16a, "default-font" );
-	add_token( 0x16b, "set-font" );
-	add_token( 0x16c, "char-height" );
-	add_token( 0x16d, "char-width" );
-	add_token( 0x16e, ">font" );
-	add_token( 0x16f, "fontbytes" );
-	add_token( 0x170, "fb1-draw-character" );
-	add_token( 0x171, "fb1-reset-screen" );
-	add_token( 0x172, "fb1-toggle-cursor" );
-	add_token( 0x173, "fb1-erase-screen" );
-	add_token( 0x174, "fb1-blink-screen" );
-	add_token( 0x175, "fb1-invert-screen" );
-	add_token( 0x176, "fb1-insert-characters" );
-	add_token( 0x177, "fb1-delete-characters" );
-	add_token( 0x178, "fb1-insert-lines" );
-	add_token( 0x179, "fb1-delete-lines" );
-	add_token( 0x17a, "fb1-draw-logo" );
-	add_token( 0x17b, "fb1-install" );
-	add_token( 0x17c, "fb1-slide-up" );
-	add_token( 0x180, "fb8-draw-character" );
-	add_token( 0x181, "fb8-reset-screen" );
-	add_token( 0x182, "fb8-toggle-cursor" );
-	add_token( 0x183, "fb8-erase-screen" );
-	add_token( 0x184, "fb8-blink-screen" );
-	add_token( 0x185, "fb8-invert-screen" );
-	add_token( 0x186, "fb8-insert-characters" );
-	add_token( 0x187, "fb8-delete-characters" );
-	add_token( 0x188, "fb8-insert-lines" );
-	add_token( 0x189, "fb8-delete-lines" );
-	add_token( 0x18a, "fb8-draw-logo" );
-	add_token( 0x18b, "fb8-install" );
-	add_token( 0x1a0, "return-buffer" );
-	add_token( 0x1a1, "xmit-packet" );
-	add_token( 0x1a2, "poll-packet" );
-	add_token( 0x1a4, "mac-address" );
-	add_token( 0x201, "device-name" );
-	add_token( 0x202, "my-args" );
-	add_token( 0x203, "my-self" );
-	add_token( 0x204, "find-package" );
-	add_token( 0x205, "open-package" );
-	add_token( 0x206, "close-package" );
-	add_token( 0x207, "find-method" );
-	add_token( 0x208, "call-package" );
-	add_token( 0x209, "$call-parent" );
-	add_token( 0x20a, "my-package" );
-	add_token( 0x20b, "ihandle>phandle" );
-	add_token( 0x20d, "my-unit" );
-	add_token( 0x20e, "$call-method" );
-	add_token( 0x20f, "$open-package" );
-	add_token( 0x210, "processor-type" );
-	add_token( 0x211, "firmware-version" );
-	add_token( 0x212, "fcode-version" );
-	add_token( 0x213, "alarm" );
-	add_token( 0x214, "(is-user-word)" );
-	add_token( 0x215, "suspend-fcode" );
-	add_token( 0x216, "abort" );
-	add_token( 0x217, "catch" );
-	add_token( 0x218, "throw" );
-	add_token( 0x219, "user-abort" );
-	add_token( 0x21a, "get-my-property" );
-	add_token( 0x21b, "decode-int" );
-	add_token( 0x21c, "decode-string" );
-	add_token( 0x21d, "get-inherited-property" );
-	add_token( 0x21e, "delete-property" );
-	add_token( 0x21f, "get-package-property" );
-	add_token( 0x220, "cpeek" );
-	add_token( 0x221, "wpeek" );
-	add_token( 0x222, "lpeek" );
-	add_token( 0x223, "cpoke" );
-	add_token( 0x224, "wpoke" );
-	add_token( 0x225, "lpoke" );
-	add_token( 0x226, "lwflip" );
-	add_token( 0x227, "lbflip" );
-	add_token( 0x228, "lbflips" );
-	add_token( 0x229, "adr-mask" );
-	add_token( 0x230, "rb@" );
-	add_token( 0x231, "rb!" );
-	add_token( 0x232, "rw@" );
-	add_token( 0x233, "rw!" );
-	add_token( 0x234, "rl@" );
-	add_token( 0x235, "rl!" );
-	add_token( 0x236, "wbflips" );
-	add_token( 0x237, "lwflips" );
-	add_token( 0x238, "probe" );
-	add_token( 0x239, "probe-virtual" );
-	add_token( 0x23b, "child" );
-	add_token( 0x23c, "peer" );
-	add_token( 0x23d, "next-property" );
-	add_token( 0x23e, "byte-load" );
-	add_token( 0x23f, "set-args" );
-	add_token( 0x240, "left-parse-string" );
+   int  indx;

-	/* FCodes from 64bit extension addendum */
-	add_token( 0x22e, "rx@" );
-	add_token( 0x22f, "rx!" );
-	add_token( 0x241, "bxjoin" );
-	add_token( 0x242, "<l@" );
-	add_token( 0x243, "lxjoin" );
-	add_token( 0x244, "wxjoin" );
-	add_token( 0x245, "x," );
-	add_token( 0x246, "x@" );
-	add_token( 0x247, "x!" );
-	add_token( 0x248, "/x" );
-	add_token( 0x249, "/x*" );
-	add_token( 0x24a, "xa+" );
-	add_token( 0x24b, "xa1+" );
-	add_token( 0x24c, "xbflip" );
-	add_token( 0x24d, "xbflips" );
-	add_token( 0x24e, "xbsplit" );
-	add_token( 0x24f, "xlflip" );
-	add_token( 0x250, "xlflips" );
-	add_token( 0x251, "xlsplit" );
-	add_token( 0x252, "xwflip" );
-	add_token( 0x253, "xwflips" );
-	add_token( 0x254, "xwsplit" );
+   dictionary = &detok_table[dictionary_indx_max-1];
+   dictionary_reset_position = dictionary;
+
+   for (indx = 1; indx < dictionary_indx_max ; indx++ )
+   {
+       detok_table[indx].next = &detok_table[indx-1];
+   }
 }
+
+void reset_dictionary(void) 
+{
+    token_t *next_t;
+
+    next_t = dictionary;
+    while ( next_t != dictionary_reset_position )
+    {
+        next_t = dictionary->next ;
+	free( dictionary->name );
+	free( dictionary );
+	dictionary = next_t ;
+    }
+}
+
+/*  If FCodes have been added by User, we need to update reset-position  */
+void freeze_dictionary(void)
+{
+   dictionary_reset_position = dictionary;
+}

Added: fcode-utils/detok/pcihdr.c
===================================================================
--- fcode-utils/detok/pcihdr.c	                        (rev 0)
+++ fcode-utils/detok/pcihdr.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,478 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Support function for de-tokenizer.
+ *
+ *      Identify and process PCI header at beginning of FCode binary file.
+ *      "Processing" consists of recognizing the PCI Header and Data Structure,
+ *      optionally printing a description thereof, and (mainly) allowing
+ *      the given pointer to be bumped to the start of the actual FCode.
+ *
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ *      Revision History:
+ *
+ *      Updated Mon, 23 May 2005 by David L. Paktor
+ *          Identify "Not Last" header.
+ *      Updated Thu, 24 Feb 2005 by David L. Paktor
+ *          Per notes after Code Review.
+ *      Updated Fri, 04 Feb 2005 by David L. Paktor
+ *      Updated Wed, 08 Jun 2005 by David L. Paktor
+ *         Added support for multiple-PCI-image files.
+ *
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Eported:
+ *          handle_pci_header
+ *              Handle all activities connected with presence of
+ *              PCI Header/Data at beginning of FCode file, and
+ *              facilitate "skipping" over to actual FCode data.
+ *
+ *          handle_pci_filler
+ *          Skip past "filler" between blocks in multi-PCI-image files.
+ *              
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Print (as remarks) full descriptions of headers' fields
+ *          Error check for wrong "Format"
+ *          Skip past non-FCode blocks, thru multiple data-blocks
+ *          Recognize PCI header in unexpected place or out-of-place
+ *
+ **************************************************************************** */
+
+#include "pcihdr.h"
+#include <stdio.h>
+
+#include "detok.h"
+
+
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *      pci_image_end           Pointer to just after end of current PCI image
+ *
+ **************************************************************************** */
+
+u8 *pci_image_end = NULL;
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *      pci_image_len           Length (in bytes) of current PCI image
+ *
+ **************************************************************************** */
+
+static int pci_image_len = 0;
+
+
+/* **************************************************************************
+ *
+ *      Function name:  is_pci_header ( rom_header_t *pci_rom_hdr )
+ *      Synopsis:   Indicate whether given pointer is pointing to
+ *                  something that might be a valid PCI header
+ *      
+ *      Inputs:
+ *          Parameters:     
+ *              pci_rom_hdr    pointer to start of data-stream to examine.
+ *                          Treat as pointer to rom_header_t
+ *
+ *      Outputs:
+ *          Returned Value: An integer.
+ *               0                  Definitely *NOT* a PCI header
+ *              Positive Number     Appears to be a valid PCI header;
+ *                                      value is offset to PCI Data Structure.
+ *              Negative Number     Appears to be a PCI header, but
+ *                                      with errors. (Not Implemented Yet.
+ *                                      See under "Still to be done".)
+ *
+ *      Error Detection:
+ *              (See under "Still to be done".)
+ *
+ *      Process Explanation:
+ *          Examine "signature" location for known value  0x55aa
+ *          If a match, return value of "dptr" (Data-pointer offset) field.
+ *
+ *      Revision History:
+ *          Created Tue, 01 Feb 2005 by David L. Paktor
+ *
+ *      Still to be done:
+ *          Error-check; look for inconsistencies:
+ *              Return a Negative Number if data-stream appears to be a PCI
+ *              header, but has erroneous or inconsistent sub-field contents.
+ *              Value and meaning of the Negative Number yet to be defined.
+ *
+ **************************************************************************** */
+
+static int is_pci_header ( rom_header_t *pci_rom_hdr )
+{
+    const u16 pci_header_signature = 0x55aa;
+    int retval ;
+
+    retval = 0;
+
+    if ( BIG_ENDIAN_WORD_FETCH(pci_rom_hdr->signature) == pci_header_signature )
+    {
+        retval = LITTLE_ENDIAN_WORD_FETCH(pci_rom_hdr->data_ptr);
+    }
+    return(retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  is_pci_data_struct ( pci_data_t *pci_data_ptr )
+ *      Synopsis:       Indicate whether given pointer is pointing to
+ *                      a valid PCI Data Structure
+ *      
+ *      Inputs:
+ *          Parameters:     
+ *              pci_data_ptr    pointer to start of data-stream to examine.
+ *                          Treat as pointer to pci_data_t 
+ *
+ *      Outputs:
+ *          Returned Value: An integer.
+ *                   0                  Definitely *NOT* a PCI Data Structure
+ *              Positive Number         Appears to be valid PCI Data Structure;
+ *                                      value is length of PCI Data Structure,
+ *                                      (presumably, offset to start of FCode).
+ *              Negative Number         Appears to be a PCI Data Structure,
+ *                                      but with errors. (Not Implemented Yet.
+ *                                      See under "Still to be done".)
+ *
+ *          Global/Static Variables:
+ *              Does not alter the poiner passed-in;
+ *              does not alter any Global/Static Variables
+ *          Printout:       NONE
+ *
+ *      Error Detection:        (Condition)     (Action)
+ *              (See under "Still to be done".)
+ *
+ *      Process Explanation:
+ *          Examine "signature" location for known value "PCIR"
+ *          If a match, return value of "dlen" (Data Structure Length) field.
+ *      
+ *      Revision History:
+ *          Created Tue, 01 Feb 2005 by David L. Paktor
+ *
+ *      Still to be done:
+ *          Error-check; look for wrong "Code Type" or other inconsistencies:
+ *              Return a Negative Number if data-stream appears to be a
+ *              valid PCI Data Structure, but has erroneous or inconsistent
+ *              sub-field contents.
+ *              Value and meaning of the Negative Number yet to be defined.
+ *          Skip past non-FCode data-blocks, even multiple blocks
+ *
+ **************************************************************************** */
+
+static int is_pci_data_struct ( pci_data_t *pci_data_ptr )
+{
+    int retval ;
+
+    retval = 0;
+
+    if (BIG_ENDIAN_LONG_FETCH(pci_data_ptr->signature) == PCI_DATA_HDR)
+    {
+        retval = LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->dlen);
+    }
+    return(retval);
+}
+ 
+
+/* **************************************************************************
+ *
+ *      Function name:  announce_pci_hdr ( rom_header_t *pci_rom_hdr )
+ *      Synopsis:       Print indication that the PCI header was found,
+ *                      and other details, formatted as FORTH remarks.
+ *      
+ *      Inputs:
+ *              Parameters:
+ *                      pci_rom_hdr        Pointer to start of PCI header.
+ *
+ *      Outputs:
+ *              Returned Value: NONE
+ *              Printout:       Announcement.  Size of  data_ptr  field.
+ *
+ **************************************************************************** */
+
+static void announce_pci_hdr ( rom_header_t *pci_rom_hdr )
+{
+    char temp_buf[80];
+    u32  temp;
+
+    printremark ( "PCI Header identified");
+    temp=(u32)LITTLE_ENDIAN_WORD_FETCH(pci_rom_hdr->data_ptr);
+    sprintf(temp_buf, "  Offset to Data Structure = 0x%04x (%d)\n",
+        temp, temp);
+    printremark ( temp_buf );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  announce_pci_data_struct ( pci_data_t *pci_data_ptr )
+ *      Synopsis:       Print indication that the PCI Data Structure
+ *                      was found, and some additional details.
+ *                      Format as FORTH remarks.
+ *      
+ *      Inputs:
+ *          Parameters:
+ *              pci_data_ptr        Pointer to start of PCI Data Structure.
+ *
+ *      Outputs:
+ *          Returned Value: NONE    
+ *          Global/Static Variables:    
+ *              pci_image_len      Updated to byte-length of current PCI image
+ *          Printout:       (See Synopsis)
+ *      
+ *      Process Explanation:
+ *          Extract some details, format and print them,
+ *              using the syntax of FORTH remarks.
+ *
+ *      Revision History:
+ *          Created Tue, 01 Feb 2005 by David L. Paktor
+ *          Updated Wed, 25 May 2005 by David L. Paktor
+ *               Added printout of several fields...
+ *
+ **************************************************************************** */
+
+static void announce_pci_data_struct ( pci_data_t *pci_data_ptr )
+{
+    char temp_buf[80];
+    u32  temp;
+
+    printremark ( "PCI Data Structure identified");
+
+    temp=(u32)LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->dlen);
+    sprintf(temp_buf, "  Data Structure Length = 0x%04x (%d)\n", temp, temp);
+    printremark ( temp_buf );
+
+    sprintf(temp_buf, "  Vendor ID: 0x%04x\n",
+            LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->vendor));
+    printremark ( temp_buf );
+
+    sprintf(temp_buf, "  Device ID: 0x%04x\n",
+            LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->device));
+    printremark ( temp_buf );
+
+    temp=(u32)CLASS_CODE_FETCH(pci_data_ptr->class_code);
+    sprintf(temp_buf, "  Class Code: 0x%06x  (%s)",
+            temp,  pci_device_class_name(temp));
+    printremark ( temp_buf );
+
+    temp=(u32)LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->vpd);
+    if ( temp != 0 )
+    {
+	sprintf(temp_buf, "  Vital Prod Data: 0x%02x\n", temp);
+	printremark ( temp_buf );
+    }
+
+    temp=(u32)LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->irevision);
+    if ( temp != 0 )
+    {
+	sprintf(temp_buf, "  Image Revision: 0x%02x\n", temp);
+	printremark ( temp_buf );
+    }
+
+    sprintf(temp_buf, "  Code Type: 0x%02x (%s)\n", pci_data_ptr->code_type,
+             pci_code_type_name(pci_data_ptr->code_type) );
+    printremark ( temp_buf );
+
+    temp=(u32)LITTLE_ENDIAN_WORD_FETCH(pci_data_ptr->ilen);
+    pci_image_len = temp*512;
+    sprintf(temp_buf, "  Image Length: 0x%04x blocks (%d bytes)\n",
+                    temp, pci_image_len);
+    printremark ( temp_buf );
+
+    sprintf(temp_buf, "  %sast PCI Image.\n",
+        pci_data_ptr->last_image_flag&&0x80 != 0 ? "L" : "Not l");
+    printremark ( temp_buf );
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_pci_header
+ *      Synopsis:       Handle PCI Header/Data at beginning of FCode file;
+ *                      facilitate "skipping" over to actual FCode data.
+ *      
+ *      Inputs:
+ *          Parameters:     
+ *              data_ptr           Pointer to start of data-stream to examine.
+ *          Global/Static Variables:        
+ *              pci_image_len      Length (in bytes) of current PCI image
+ *
+ *      Outputs:
+ *          Returned Value:
+ *              Positive Number.        Offset to start of FCode.
+ *              Zero                    If no PCI header; may be treated as
+ *                                          a valid offset.
+ *              Negative Number         PCI header or PCI Data Structure test
+ *                                          returned error indication.
+ *                                         (Not Implemented Yet.  See
+ *                                          under "Still to be done".)
+ *          Global/Static Variables:        
+ *              pci_image_end           Pointer to just after the end of
+ *                                          the current PCI image
+ *          Printout:       As FORTH remarks, print indications that the
+ *                          PCI header was found, and maybe later more data.
+ *
+ *      Error Detection:        (Condition)     (Action)
+ *              (See under "Still to be done".)
+ *      
+ *      Process Explanation:
+ *          Use the various support routines defined below.
+ *      
+ *      
+ *      Revision History:
+ *
+ *      Updated Wed, 09 Feb 2005 by David L. Paktor
+ *          Extracted assignments from within  if( )  statements.
+ *
+ *      Created Tue, 01 Feb 2005 by David L. Paktor
+ *
+ *      Still to be done:
+ *          Handle error cases.  At present, neither  is_pci_header()
+ *              nor  is_pci_data_struct()  returns a negative number,
+ *              but when they are modified to do so, we must handle it.
+ *
+ **************************************************************************** */
+
+int handle_pci_header ( u8  *data_ptr )
+{
+    int hdrlen;
+    int data_struc_len;
+    /*  int retval;  */  /*  Not needed until we handle error cases...  */
+
+    data_struc_len = 0;
+
+    hdrlen = is_pci_header( (rom_header_t *)data_ptr );
+    /*  retval = hdrlen;  */  /*  Not needed yet...  */
+    if ( hdrlen < 0 )
+    {
+        /*  Handle error case...  */
+        /*  Leave null for now...  */
+        /*  It might need to do a premature EXIT here...  */
+    } else {
+        /* if hdrlen == 0 then we don't need to check a Data Structure  */
+        if ( hdrlen > 0 )
+        {
+            announce_pci_hdr ( (rom_header_t *)data_ptr );
+            data_struc_len = is_pci_data_struct(
+                        (pci_data_t *) &data_ptr[hdrlen] );
+            /*
+             *  A Data Structure Length of Zero would be an error
+             *  that could be detected by  is_pci_data_struct()
+             */ 
+            if ( data_struc_len <= 0 )
+            {
+                /*  Handle error case...  */
+                /*  Leave null for now...  */
+                /*  It might need to do a premature EXIT here...  */
+                /*  retval = -1;   */  /*  Not needed yet...  */
+            } else {
+                announce_pci_data_struct ( (pci_data_t *) &data_ptr[hdrlen] );
+		pci_image_end = data_ptr + pci_image_len;
+                /* retval = hdrlen+data_struc_len; */ /*  Not needed yet... */
+            }
+        }
+    }
+    return (hdrlen+data_struc_len);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_pci_filler
+ *      Synopsis:       Examine and report on the "filler" padding after the
+ *                      end of an FCode-block but still within a PCI-image
+ *
+ *      Inputs:
+ *         Parameters:
+ *             filler_ptr         Pointer to start of PCI-filler in data-stream
+ *         Global/Static Variables:    
+ *             pci_image_end      Pointer to just after the end of
+ *                                          the current PCI image
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *         Printout:
+ *             Descriptive message.
+ *
+ *      Error Detection:
+ *          Non-zero filler field.  Different message.
+ *
+ *      Process Explanation:
+ *          The calling routine has checked that there was, indeed, a PCI
+ *              header present, so we know that pci_image_end is valid.
+ *          If the entire filler is zero-bytes, print a simple message and
+ *              we're out'a here!
+ *          If there are non-zero bytes, identify loc'n of first non-zero.
+ *
+ *      Still to be done:
+ *          Come up with something more elegant for non-zero filler.
+ *
+ **************************************************************************** */
+
+void handle_pci_filler(u8 *filler_ptr)
+{
+    u8 *scan_ptr;
+    int filler_len;
+    char temp_buf[80];
+    bool all_zero = TRUE;
+    u8 filler_byte = *filler_ptr;
+
+    filler_len = pci_image_end - filler_ptr;
+
+    for ( scan_ptr = filler_ptr;
+        scan_ptr < pci_image_end;
+	    filler_byte = *(++scan_ptr) )
+    {
+	if ( filler_byte != 0 )
+	{
+	    all_zero = FALSE;
+	    break;
+	}    
+    }
+    
+    if ( all_zero )
+    {
+	sprintf(temp_buf, "PCI Image padded with %d bytes of zero", filler_len);
+    }else{
+	sprintf(temp_buf, "PCI Image padding-field of %d bytes "
+	    "had first non-zero byte at offset %ld",
+		 filler_len, scan_ptr - filler_ptr );
+    }
+    printremark ( temp_buf );
+}

Added: fcode-utils/detok/printformats.c
===================================================================
--- fcode-utils/detok/printformats.c	                        (rev 0)
+++ fcode-utils/detok/printformats.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,140 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Print, in various controlled formats, for the detokenizer.
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Ultimately, our goal is to produce output that can be run back
+ *      through the tokenizer and produce the same binary.  So, any
+ *      extra text will have to be in a form that the tokenizer will
+ *      treat as comments.
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "detok.h"
+
+
+/* **************************************************************************
+ *
+ *      Function name:  printremark ( string )
+ *      Synopsis:       Print the given string as a series of "Remark" lines,
+ *                      (i.e., preceded by backslash-space)
+ *      
+ *      Inputs:
+ *          Parameters:     
+ *              str     Pointer to the start of the string,
+ *                      however long it may be.  The string
+ *                      may have any number of embedded new-lines.
+ *
+ *      Outputs:
+ *          Returned Value: *** NONE***
+ *          Printout:    Each line of the string will be preceded by
+ *                       a backslash and two spaces.  Backslash-space
+ *                       is the standard delimiter for a "remark", i.e.
+ *                       the entire line is ignored as a comment.  The
+ *                       second space is "just" for aesthetics.
+ *      
+ *      Process Explanation:
+ *          Parse the input string for new-lines.  Print each separately.
+ *          Do not alter the input string.
+ *      
+ *
+ *      Still to be done:
+ *          Define a routine, call it  PrintComment , to print the given
+ *              string surrounded by open-paren-space ... space-close-paren
+ *          Define a single central routine, call it  safe_malloc ,
+ *              to do the test for null and print "No Memory" and exit.
+ *          Define a single central routine, call it  PrintError , to:
+ *              Print the given error message
+ *              Show the input file and line number
+ *              Collect error-flags for failure-exit at end of operation.
+ *
+ **************************************************************************** */
+
+void printremark ( char *str)
+{
+    char *strtmp ;          /*  Temporary pointer to current substring    */
+    int substrlen ;         /*  Length of current substring               */
+    char *substrend ;       /*  Pointer to end of current substring       */
+    char *strend ;          /*  Pointer to end of given string            */
+
+    char *strbfr ;          /*  Temporary substring buffer                */
+
+    /*  Guarantee that the malloc will be big enough.  */
+    strbfr = (char *)malloc(strlen((char *)str)+1);
+    if ( !strbfr )
+    {
+        printf ("No memory.\n");
+        exit(-1);
+    }
+
+
+    strtmp = str;
+    strend = &str[strlen(str)];
+
+    /* ******************************************************************
+     *
+     *      Isolate the current substring; allow that the given
+     *      string might not be terminated with a new-line. 
+     *
+     *      The  strend  pointer provides a convenient means to
+     *      test for when we've reached the end.
+     *
+     ******************************************************************** */
+
+   while ( strtmp < strend  )
+    {
+        substrend = strchr(strtmp , '\n');
+        substrlen = ( substrend ? (substrend-strtmp) : strlen(strtmp) );
+
+        strncpy (strbfr, strtmp, substrlen);
+        /* **********************************************************
+         * 
+         *  strncpy() does not append a terminating null character,
+         *  so we have to.
+         *
+         ************************************************************ */
+        strbfr[substrlen] = (char)0;
+
+        printf("\  %s\n",strbfr);
+
+        strtmp = &strtmp[ substrlen + ( substrend ? 1 : 0 ) ] ;
+
+    }
+
+    free(strbfr) ;
+}
+

Modified: fcode-utils/detok/stream.c
===================================================================
--- fcode-utils/detok/stream.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/stream.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,20 +24,60 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/stat.h>
+#include <setjmp.h>

+#include "stream.h"
 #include "detok.h"
+#include "pcihdr.h"

-extern bool offs16;
-extern u32  fcpos;
+extern jmp_buf eof_exception;

+/* **************************************************************************
+ *
+ *      Global Variables Exported:
+ *        Name            Value
+ *      fcode          The FCode-token last read.  Not necessarily the byte
+ *                          last read, if its function has in-line parameters
+ *      stream_max     The maximum position -- length -- of the input stream
+ *      pc             Pointer to "Current byte" in the input-data image.
+ *      max            Just after the end of the input-data image.
+ *                         This is *NOT* a byte-count.
+ *
+ **************************************************************************** */
+
 u16 fcode;
-u8  inbyte;
+unsigned int stream_max;
+u8 *pc;
+u8 *max;

-static u8 *indata, *pc, *max;
+/* **************************************************************************
+ *
+ *      Local/Static Variables:
+ *        Name            Pointer to:
+ *      indata          Start of input-data image taken from input file.
+ *                          This memory was "malloc"ed; keep it around
+ *                          for when we "free" that memory.
+ *      fc_start        Start of the FCode.  This might not be the same
+ *                          as the start of the input file data, especially
+ *                          if the input file data starts with a PCI header.
+ *      pci_image_found     TRUE iff a valid PCI header was found
+ *
+ **************************************************************************** */
+u8 *indata;
+static u8 *fc_start;
+static bool pci_image_found = FALSE;

+
 int init_stream(char *name)
 {
    FILE *infile;
@@ -62,119 +102,455 @@
    fclose(infile);

pc=indata; 
+	fc_start = indata;
    max=pc+finfo.st_size;
    
+	stream_max = finfo.st_size;
+	
    return 0;
 }

-int close_stream(void)
+/* **************************************************************************
+ *
+ *      Function name:  init_fcode_block
+ *      Synopsis:       Initialize all pointers and variables, etcetera,
+ *                          for an FCode-block.
+ *
+ **************************************************************************** */
+
+void init_fcode_block(void)
 {
+	fc_start = pc;
+	linenum = 1 ;
+}
+
+
+void close_stream(void)
+{
    free(indata);
-	return 0;	
+	stream_max = 0;
 }

int get_streampos(void)
 {
-	return (int)((long)pc-(long)indata);
+	return (int)( pc - fc_start );
 }

-void set_streampos(long pos)
+void set_streampos(int pos)
 {
-	pc=indata+pos;
+	pc = fc_start + pos;
 }

-static int get_byte(void)
+
+/* **************************************************************************
+ *
+ *      Function name:  throw_eof
+ *      Synopsis:       Analyze and print the cause of the end-of-file
+ *                      and throw an exception.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             premature              TRUE iff end-of-file was out-of-sync 
+ *         Global/Static Variables:        
+ *                  end_found         Indicates if normal end of fcode was read.
+ *                  eof_exception     Long-Jump environment to which to jump.
+ *
+ *      Outputs:                      Does a Long-Jump
+ *         Returned Value:            NONE
+ *         Printout:
+ *             "End-of-file" message, along with a descriptor, if applicable:
+ *                 Premature, Unexpected.
+ *          The calling routine notifies us if the number of bytes requested
+ *              overflowed the input buffer, by passing us a TRUE for the
+ *              input parameter.  That is a "Premature" end-of-file.
+ *          If  end_found  is FALSE, it means the normal end of FCode
+ *               wasn't seen.  That is an "Unexpected" end-of-file.
+ *
+ **************************************************************************** */
+
+static void throw_eof(bool premature)
 {
-	inbyte=*pc;
-	pc++;
+    char yoo = 'U';
+    char eee = 'E';
+    if ( premature)
+    {
+        printf("Premature ");
+	yoo = 'u';
+	eee = 'e';
+    }
+    if ( ! end_found )
+    {
+        printf("%cnexpected ",yoo);
+	eee = 'e';
+    }
+    printf("%cnd of file.\n",eee);
+    longjmp(eof_exception, -1);
+}
    
-        if (pc>max) {
-                printf ("\nUnexpected end of file.\n");
-                return 0;
+/* **************************************************************************
+ *
+ *      Function name:  get_bytes
+ *      Synopsis:       Return the next string of bytes, as requested, from
+ *                      the FCode input-stream.  Detect end-of-file
+ *
+ *      Inputs:
+ *         Parameters:
+ *             nbytes            The number of bytes requested
+ *         Global/Static Variables:
+ *             pc                Pointer to "where we are" in the FCode
+ *             max               Pointer to just after end of input file data.
+ *
+ *      Outputs:
+ *         Returned Value:
+ *             Pointer to the requested bytes in the FCode input-stream.
+ *         Global/Static Variables:
+ *             pc                Incremented by the number of bytes requested
+ *
+ *      Exception:
+ *          When end-of-file is reached, or is about to be exceeded,    
+ *              throw an end-of-file exception.
+ *
+ *      Process Explanation:
+ *          End-of-file is not exactly an error, so its detection is more
+ *              a part of normal processing.
+ *          If we entered this routine with PC pointing exactly at MAX,
+ *              we are probably at the end the way we expect to be, so we
+ *              call our EOF-handling routine with a "non-premature" flag.
+ *          If the requested number of bytes puts us just even with MAX,
+ *              we have neither reached nor over-run the input stream, so
+ *              there's no call to our EOF-handling routine needed at all.
+ *          Only if the requested number of bytes puts us past MAX have we 
+ *              over-run our input stream with a "premature" condition.
+ *
+ *      Extraneous Remarks:
+ *          This is another one where it was easier to write the code
+ *              than the explanation ...  ;-}
+ *
+ **************************************************************************** */
+
+static u8 *get_bytes(int nbytes)
+{
+   u8 *retval = pc;
+   if ( pc == max )
+   {
+       throw_eof(FALSE);
+   }
+   if ( pc + nbytes > max )
+   {
+       throw_eof(TRUE);
+   }
+   pc += nbytes;
+   return( retval);
         }

-	return 1;
+
+/* **************************************************************************
+ *
+ *       Function name:  more_to_go
+ *       Synopsis:       Return FALSE when the last byte has been
+ *                           read from the input-stream.
+ *
+ **************************************************************************** */
+
+bool more_to_go(void)
+{
+    bool retval;
+    retval = INVERSE( pc == max );
+    return( retval);
 }

-u16 get_token(void)
+
+/* **************************************************************************
+ *
+ *      Function name:  next_token
+ *      Synopsis:       Retrieve the next FCode-token from the input-stream.
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *
+ *      Outputs:
+ *         Returned Value:                 The next FCode-token
+ *         Global/Static Variables:
+ *             fcode                       The FCode-token last read.
+ *             token_streampos             Streampos() of token just gotten
+ *
+ **************************************************************************** */
+ 
+u16 next_token(void)
 {
         u16 tok;
-        get_byte();
-        tok=inbyte;
-        if (tok != 0x00 && tok < 0x10) {
-                get_byte();
+        token_streampos = get_streampos();
+        tok = *(get_bytes(1));
+	if ( tok != 0x00 &&  tok < 0x10)
+	{
                 tok<<=8;
-                tok|=inbyte;
+              tok |= *(get_bytes(1));     
         }
         fcode=tok;
-        return tok;
+    return(tok);
 }

u32 get_num32(void)
 {
-        u32 ret;
+        u32 retval;
+	u8 *num_str;

-        get_byte();
-        ret=inbyte<<24;
-        get_byte();
-        ret|=(inbyte<<16);
-        get_byte();
-        ret|=(inbyte<<8);
-        get_byte();
-        ret|=inbyte;
+	num_str = get_bytes(4);
+	retval = BIG_ENDIAN_LONG_FETCH(num_str);

-        return ret;
+        return ( retval);
 }

u16 get_num16(void)
 {
-        u16 ret;
+        u16 retval;
+	u8 *num_str;

-        get_byte();
-        ret=inbyte<<8;
-        get_byte();
-        ret|=inbyte;
+        num_str = get_bytes(2);
+	retval = BIG_ENDIAN_WORD_FETCH(num_str);

-        return ret;
+        return ( retval);
 }

u8 get_num8(void)
 {
-        get_byte();
+        u8  inbyte;
+
+	inbyte = *(get_bytes(1));
         return(inbyte);
 }

-u16 get_offset(void)
+s16 get_offset(void)
 {
+        s16 retval;
         if (offs16)
-                return (get_num16());
+        {
+	    retval = (s16)get_num16();
+	}else{
+	    retval = (s16)get_num8();
+	    /*  Make sure it's sign-extended  */
+	    retval |= ( retval & 0x80 ) ? 0xff00 : 0 ;
+	}

-        return (get_num8());
+        return (retval);
 }

+/* **************************************************************************
+ *
+ *      Function name:  get_string
+ *      Synopsis:       Return a pointer to a Forth-Style string within the
+ *                          input stream.  Note: this cannot be used to create
+ *                          a new token-name; use  get_name()  for that.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             *len                       Pointer to where the length will go
+ *
+ *      Outputs:
+ *         Returned Value:
+ *             Pointer to the string within the input stream.
+ *         Supplied Pointers:
+ *             *len                      Length of the string
+ *
+ *      Process Explanation:
+ *          Get one byte representing the length of the FORTH-style string.
+ *          Get as many bytes as the length indicates.
+ *          That's the string.  The pointer returned by  get_bytes()  is
+ *              our return value.
+ *
+ **************************************************************************** */

-int scnt=0;
-u8 *get_string(void)
+u8 *get_string(u8 *len)
 {
-        u8 *data;
-        u8 size;
-        unsigned int i;
+    char *retval;
+
+    *len = get_num8();
+    retval = get_bytes((int)*len);

-        get_byte();
-        size=inbyte;
+    return (retval); 
+}

-	scnt++;

-        data=malloc(size+2);
-	if (!data) printf ("No more memory.\n");
-        data[0]=size;
+/* **************************************************************************
+ *
+ *      Function name:  get_name
+ *      Synopsis:       Retrieve a copy of the next string in the input-stream
+ *                      when it is expected to be a new function-name.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             *len                       Pointer to where the length will go
+ *         Global/Static Variables:
+ *             pc                        "Where we are" in the file-stream
+ *
+ *      Outputs:
+ *         Returned Value: 
+ *             Pointer to allocated memory containing a copy of the string
+ *         Supplied Pointers:
+ *             *len                      Length of the name
+ *         Memory Allocated
+ *             Memory for the copy of the string is allocated by  strdup()
+ *         When Freed?
+ *             Never.  Retained for duration of the program.
+ *
+ *      Process Explanation:
+ *          Get the FORTH-style string.
+ *          At this point, PC points to the byte that follows the string;
+ *              we are going to save that byte and replace it with a zero,
+ *              thus creating a C-style string.
+ *          We will pass that C-style string as an argument to  strdup() ,
+ *              which will give us our return value.
+ *          Then, of course, we restore the byte we so rudely zeroed, and
+ *              proceed merrily on our way.
+ *
+ **************************************************************************** */
+
+char *get_name(u8 *len)
+{
+    char *str_start;
+    char *retval;
+    u8 sav_byt;

-        for (i=1; i<size+1; i++) {
-                get_byte();
-                data[i]=inbyte;
+    str_start = get_string( len );
+
+    sav_byt = *pc;
+    *pc = 0;
+
+    retval = strdup(str_start);
+    *pc = sav_byt;
+ 
+    return (retval);
         }
-        data[i]=0;
-        return data;

+/* **************************************************************************
+ *
+ *      Function name:  calc_checksum
+ *      Synopsis:       Calculate the checksum.
+ *                          Leave the input position unchanged.
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Global/Static Variables:     
+ *             pc      Pointer to "where we are" in the file-stream
+ *
+ *      Outputs:
+ *         Returned Value:        Calculated checksum.
+ *         Global/Static Variables:  
+ *             pc     Reset to value upon entry
+ *
+ *      Process Explanation:
+ *          When this routine is entered, the PC is presumed to be pointing
+ *              just after the stored checksum, and just before the length
+ *              field in the FCode header.  This is the point at which we
+ *              will preserve the PC
+ *          Extract the length from the FCode header.  It includes the eight
+ *              bytes of the FCode header, so we will need to adjust for that.
+ *          The first byte after the FCode header is where the checksum
+ *              calculation begins.
+ *
+ **************************************************************************** */
+
+u16 calc_checksum(void)
+{
+    u16 retval = 0;
+    u8 *cksmptr;
+    u8 *save_pc;
+    u32 fc_blk_len;
+    int indx;
+
+    save_pc = pc;
+
+    fc_blk_len = get_num32();            /* Read len */
+    cksmptr = get_bytes(fc_blk_len-8);   /*  Make sure we have all our data  */
+
+    for ( indx = 8; indx < fc_blk_len; indx++)
+    {
+	retval += *cksmptr++;
 }

+    pc = save_pc;
+    return( retval);
+} 
+
+
+/* **************************************************************************
+ *
+ *      Function name:  adjust_for_pci_header
+ *      Synopsis:   Skip the PCI Header.  Adjust the pointer to 
+ *                  the start of FCode in the file-stream,
+ *                  and our pointer to "where we are" in the FCode,
+ *                  by the size of the PCI header.
+ *      
+ *      Inputs:
+ *              Parameters:     NONE
+ *              Global/Static Variables:    (Pointer to:)
+ *                  pc                     "where we are" in the file-stream
+ *                  fc_start               start of FCode in the file-stream
+ *
+ *      Outputs:
+ *              Returned Value: NONE
+ *              Global/Static Variables:
+ *                  pc                     Advanced past PCI header
+ *                  fc_start               Likewise.
+ *                  pci_image_found        Set or cleared as appropriate.
+ *                  last_defined_token     Re-initialized
+ *      
+ *      Process Explanation:
+ *          Call handle_pci_header to get the size of the PCI header,
+ *              if any, and increment  pc  and  fc_start  by the number
+ *              of bytes it returns; also, set  pci_image_found  based
+ *              on whether the "size of the PCI header" was non-zero.
+ *          (Re-)Initialize overlap detection here.  Images with multiple
+ *              PCI blocks can safely re-cycle FCode numbers; this is
+ *              not necessarily  true of multiple FCode blocks within
+ *              the same PCI block...
+ *
+ **************************************************************************** */
+
+void adjust_for_pci_header(void)
+{
+	int pci_header_size;
+	
+	pci_header_size = handle_pci_header(pc);
+	pci_image_found = pci_header_size > 0 ? TRUE : FALSE;
+	pc += pci_header_size;
+	fc_start += pci_header_size;
+	last_defined_token = 0;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  adjust_for_pci_filler
+ *      Synopsis:       Dispatch call to pci-filler-handler
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Global/Static Variables:
+ *             pci_image_found           Whether to proceed...
+ *             pci_image_end             Pointer to just after the end of
+ *                                           the current PCI image
+ *
+ *      Outputs:
+ *          Returned Value:                NONE
+ *          Global/Static Variables:
+ *              pci_image_found            Reset to FALSE
+ *
+ *      Error Detection:
+ *          Confirm that the data-stream has the complete filler,
+ *              via a call to get_bytes()
+ *
+ **************************************************************************** */
+
+void adjust_for_pci_filler(void)
+{
+    if ( pci_image_found )
+    {
+        int pci_filler_len;
+	u8 *pci_filler_ptr;
+
+	pci_filler_len = pci_image_end - pc;
+	pci_filler_ptr = get_bytes(pci_filler_len);
+	handle_pci_filler( pci_filler_ptr );
+	pci_image_found = FALSE;
+    }
+}

Modified: fcode-utils/detok/stream.h
===================================================================
--- fcode-utils/detok/stream.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/detok/stream.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _UTILS_DETOK_STREAM_H
+#define _UTILS_DETOK_STREAM_H
+
 /*
  *                     OpenBIOS - free your system! 
  *                            ( detokenizer )
@@ -24,16 +27,42 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
+#include "types.h"
+
+/*  Prototypes for functions exported from  stream.c     */
+
+
 int init_stream(char *name);
-int close_stream(void);
+void close_stream(void);
+bool more_to_go(void);

+void adjust_for_pci_header(void);
+void adjust_for_pci_filler(void);
+void init_fcode_block(void);
+
 int  get_streampos(void);
-void set_streampos(long pos);
+void set_streampos(int pos);

-u16 get_token(void);
+u16 next_token(void);
 u8  get_num8(void);
 u16 get_num16(void);
 u32 get_num32(void);
-u16 get_offset(void);
-u8 *get_string(void);
+s16 get_offset(void);
+u8 *get_string(u8 *len);
+char *get_name(u8 *len);
+u16 calc_checksum(void);

+/*  External declarations for variables defined in   stream.c   */
+
+extern unsigned int stream_max;
+extern u8 *indata;
+extern u8 *pc;
+extern u8 *max;
+
+#endif   /*  _UTILS_DETOK_STREAM_H    */

Modified: fcode-utils/romheaders/Makefile
===================================================================
--- fcode-utils/romheaders/Makefile	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/romheaders/Makefile	2006-08-18 12:47:12 UTC (rev 76)
@@ -21,20 +21,21 @@
 #  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
 #

-
 CC  = gcc
-CFLAGS= -O2 -Wall -W -ansi
+CFLAGS= -O2 -Wall -W -ansi -I../shared

-.SUFFIXES: .c .o
+SOURCES = romheaders.c ../shared/classcodes.c

+.SUFFIXES: .c
+
 all: romheaders

-romheaders: romheaders.o
-	$(CC) $(CFLAGS) romheaders.o -o romheaders
+romheaders: $(SOURCES)
+	$(CC) $(CFLAGS) $(SOURCES) -o $@
    strip romheaders
    
 clean:
-	rm -f *.o
+	rm -f *~

distclean: clean
    rm -f romheaders

Modified: fcode-utils/romheaders/romheaders.c
===================================================================
--- fcode-utils/romheaders/romheaders.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/romheaders/romheaders.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -23,6 +23,11 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */

#include <stdio.h>
 #include <stdlib.h>
@@ -30,60 +35,32 @@
 #include <sys/stat.h>
 #include <unistd.h>

-#define u8  unsigned char
-#define u16 unsigned short
-#define u32 unsigned int
+#include "pcihdr.h"

-#define PCI_DATA_HDR (u32) ( ('R' << 24) | ('I' << 16) | ('C' << 8) | 'P' )
-
-typedef struct {
-	u16	signature;
-	u8	reserved[0x16];
-	u16	dptr;
-} rom_header_t;
-
-typedef struct {
-	u32	signature;
-	u16	vendor;
-	u16	device;
-	u16	reserved_1;
-	u16	dlen;
-	u8	drevision;
-	u8	class_hi;
-	u16	class_lo;
-	u16	ilen;
-	u16	irevision;
-	u8	type;
-	u8	indicator;
-	u16	reserved_2;
-} pci_data_t;
-
 char *rom=NULL;
 size_t romlen=0;

-/* make this endian safe without fancy system headers */
-static u16 little_word(u16 val)
-{
-	u8 *ptr=(u8 *)&val;
-	return (ptr[0]|(ptr[1]<<8));
-}
+/*  Prototypes for functions exported from  shared/classcodes.c   */

-static u16 little_dword(u16 val)
-{
-	u8 *ptr=(u8 *)&val;
-	return (ptr[0]|(ptr[1]<<8)|(ptr[2]<<16)|(ptr[3]<<24));
-}
+char *pci_device_class_name(u32 code);
+char *pci_code_type_name(unsigned char code);

-/* dump the rom headers */
+/*  Functions local to this file:
+	    int dump_rom_header(rom_header_t *data);
+	    int dump_pci_data(pci_data_t *data);
+	    void dump_platform_extensions(u8 type, rom_header_t *data);
+ */
+
 static int dump_rom_header(rom_header_t *data)
-{
-	u16 sig=little_word(data->signature);
+{   /*  Return TRUE for "no problem"  */
+	const u16 pci_header_signature = 0x55aa;
+	u16 sig=BIG_ENDIAN_WORD_FETCH(data->signature);
    int i;
    
    printf ("PCI Expansion ROM Header:\n");
    
-	printf ("  Signature: 0x%02x%02x (%s)\n", 
-			sig&0xff,sig>>8,sig==0xaa55?"Ok":"Not Ok");
+	printf ("  Signature: 0x%04x (%s)\n", 
+			sig, sig == pci_header_signature ? "Ok":"Not Ok");
    
    printf ("  CPU unique data:");
    for (i=0;i<16;i++) {
@@ -92,97 +69,41 @@
    }
    
    printf ("\n  Pointer to PCI Data Structure: 0x%04x\n\n",
-						little_word(data->dptr));
+				LITTLE_ENDIAN_WORD_FETCH(data->data_ptr));

-	return (sig==0xaa55);
+	return (sig == pci_header_signature);
 }

static int dump_pci_data(pci_data_t *data)
-{
-	u32 sig=little_dword(data->signature);
-	u32 classcode=(data->class_hi<<16)|(little_word(data->class_lo));
+{   /*  Return TRUE for "no problem"  */
+	const u32 pci_data_hdr = PCI_DATA_HDR ;
+
+	u32 sig      = BIG_ENDIAN_LONG_FETCH(data->signature);
+	u32 classcode= CLASS_CODE_FETCH(data->class_code);
+	u32 dlen     = (u32)LITTLE_ENDIAN_WORD_FETCH(data->dlen);
+	u32 ilen     = (u32)LITTLE_ENDIAN_WORD_FETCH(data->ilen);
    
    printf("PCI Data Structure:\n");
-	printf("  Signature: '%c%c%c%c' (%s)\n", sig&0xff,(sig>>8)&0xff,
-			(sig>>16)&0xff, sig>>24, sig==PCI_DATA_HDR?"Ok":"Not Ok");
-	printf("  Vendor ID: 0x%04x\n", little_word(data->vendor));
-	printf("  Device ID: 0x%04x\n", little_word(data->device));
-	printf("  Reserved: 0x%04x\n", little_word(data->reserved_1));
-	printf("  PCI Data Structure Length: 0x%04x (%d bytes)\n", 
-			little_word(data->dlen), little_word(data->dlen));
+	printf("  Signature: 0x%04x '%c%c%c%c' ", sig,
+			sig>>24,(sig>>16)&0xff, (sig>>8)&0xff, sig&0xff);
+	printf("(%s)\n", sig == pci_data_hdr ?"Ok":"Not Ok");
+
+	printf("  Vendor ID: 0x%04x\n", LITTLE_ENDIAN_WORD_FETCH(data->vendor));
+	printf("  Device ID: 0x%04x\n", LITTLE_ENDIAN_WORD_FETCH(data->device));
+	printf("  Vital Product Data:  0x%04x\n",
+	                                   LITTLE_ENDIAN_WORD_FETCH(data->vpd));
+	printf("  PCI Data Structure Length: 0x%04x (%d bytes)\n", dlen, dlen);
    printf("  PCI Data Structure Revision: 0x%02x\n", data->drevision);
-	printf("  Class Code: 0x%06x (",classcode);
-	switch (classcode) {
-	case 0x0100:
-		printf("SCSI Storage");
-		break;
-	case 0x0101:
-		printf("IDE Storage");
-		break;
-	case 0x0103:
-		printf("IPI Storage");
-		break;
-	case 0x0104:
-		printf("RAID Storage");
-		break;
-	case 0x0180:
-		printf("Storage");
-		break;
-		
-	case 0x0200:
-		printf("Ethernet");
-		break;
-	case 0x0201:
-		printf("Token Ring");
-		break;
-	case 0x0202:
-		printf("FDDI");
-		break;
-	case 0x0203:
-		printf("ATM");
-		break;
-	case 0x0280:
-		printf("Network");
-		
-	case 0x0300:
-		printf("VGA Display");
-		break;
-	case 0x0301:
-		printf("XGA Display");
-		break;
-	case 0x0302:
-		printf("3D Display");
-		break;
-	case 0x0380:
-		printf("Display");
-		break;
-		
-	default:
-		printf("unkown");
-	}
-	printf(")\n  Image Length: 0x%04x blocks (%d bytes)\n", 
-			little_word(data->ilen), little_word(data->ilen)*512);
+	printf("  Class Code: 0x%06x (%s)\n",classcode,
+					 pci_device_class_name(classcode));
+	printf("  Image Length: 0x%04x blocks (%d bytes)\n", ilen, ilen*512);
    printf("  Revision Level of Code/Data: 0x%04x\n",
-			little_word(data->irevision));
-	printf("  Code Type: 0x%02x (", data->type);
-	switch (data->type) {
-	case 0:
-		printf("Intel x86");
-		break;
-	case 1: 
-		printf("Open Firmware");
-		break;
-	case 2:
-		printf("HP PA Risc");
-		break;
-	case 3:
-		printf("Intel EFI (unofficial)");
-		break;
-	default:
-		printf("unknown as of PCI specs 2.2");
-	}
-	printf(")\n  Indicator: 0x%02x %s\n", data->indicator,
-			data->indicator&0x80?"(last image in rom)":"");
+			(u32)LITTLE_ENDIAN_WORD_FETCH(data->irevision));
+	printf("  Code Type: 0x%02x (%s)\n", data->code_type,
+					  pci_code_type_name(data->code_type) );
+	printf("  Last-Image Flag: 0x%02x (%slast image in rom)\n",
+			data->last_image_flag,
+			data->last_image_flag&0x80?"":"not ");
    printf("  Reserved: 0x%04x\n\n", little_word(data->reserved_2));

return (sig==PCI_DATA_HDR);
@@ -225,7 +146,7 @@
    		printf( "  Entry point for INIT function:"
    			" 0x%x\n\n",entry);
    	} else
-			printf( "  Unable to determin entry point for INIT"
+			printf( "  Unable to determine entry point for INIT"
    			" function. Please report.\n\n");
    	
    	break;
@@ -252,9 +173,9 @@

if (argc!=2) {
    	printf ("\nUsage: %s <romimage.img>\n",argv[0]);
-		printf ("\nromheaders dumps pci option rom headers "
+		printf ("\n  romheaders dumps pci option rom headers "
    			"according to PCI \n"
-				"specs 2.2 in human readable form\n\n");
+				"  specs 2.2 in human readable form\n\n");
    	return -1;
    }
    
@@ -290,23 +211,25 @@
    do {
    	printf("\nImage %d:\n",i);
    	if (!dump_rom_header(rom_header)) {
-			printf("Error occured. Bailing out.\n");
+			printf("Rom Header error occured. Bailing out.\n");
    		break;
    	}
    	
-		pci_data=(pci_data_t *)(rom+little_word(rom_header->dptr));
+		pci_data = (pci_data_t *)((char *)rom_header +
+			     LITTLE_ENDIAN_WORD_FETCH(rom_header->data_ptr));
    	
    	if (!dump_pci_data(pci_data)) {
-			printf("Error occured. Bailing out.\n");
+			printf("PCI Data error occured. Bailing out.\n");
    		break;
    	}
    	
-		dump_platform_extensions(pci_data->type, rom_header);
+		dump_platform_extensions(pci_data->code_type, rom_header);
    	
-		rom_header+=little_word(pci_data->ilen)*512;
+		rom_header = (rom_header_t *)((char *)rom_header +
+				LITTLE_ENDIAN_WORD_FETCH(pci_data->ilen)*512);
    	i++;
-	} while ((pci_data->indicator&0x80)!=0x80 &&
-			romlen<(unsigned long)rom_header-(unsigned long)romlen);
+	} while ((pci_data->last_image_flag&0x80)!=0x80 &&
+			(char *)rom_header < rom+(int)romlen );

return 0;
 }

Modified: fcode-utils/toke/Makefile
===================================================================
--- fcode-utils/toke/Makefile	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/Makefile	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,11 +1,12 @@
 #
-#                     OpenBIOS - free your system! 
-#                         ( FCode tokenizer )
-#                          
-#  This program is part of a free implementation of the IEEE 1275-1994 
+#                     OpenBIOS - free your system!
+#                             ( Utilities )
+#
+#  This program is part of a free implementation of the IEEE 1275-1994
 #  Standard for Boot (Initialization Configuration) Firmware.
 #
-#  Copyright (C) 2001-2005  Stefan Reinauer, 
+#  Copyright (C) 2001-2006  Stefan Reinauer 
+#  Copyright (C) 2006 coresystems GmbH 
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -21,52 +22,39 @@
 #  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
 #

-ARCH     := $(shell uname -m | sed -e s/i.86/x86/ -e s/sun4u/sparc64/ -e s/arm.*/arm/ -e s/sa110/arm/ -e s/x86_64/amd64/)
-TOPDIR   := $(shell /bin/pwd)
-BUILDDIR ?= $(TOPDIR)/obj-$(ARCH)
-VPATH    := $(BUILDDIR)
+PROGRAM = toke

-include $(TOPDIR)/Rules.make
+CC      = gcc
+STRIP	= strip
+INCLUDES = -I../shared
+#CFLAGS  = -O2 -g -Wall 
+CFLAGS  = -Os -Wall -Wno-pointer-sign -DSYS_IS_GNU_Linux
+LDFLAGS =

-CC     = gcc
-CFLAGS = -O2 -Wall -W -ansi -pedantic
-# CFLAGS = -O -g -Wall -ansi -pedantic
+OBJS  = clflags.o conditl.o devnode.o dictionary.o emit.o errhandler.o     \
+        flowcontrol.o macros.o nextfcode.o parselocals.o scanner.o stack.o \
+	stream.o strsubvocab.o ticvocab.o toke.o tokzesc.o tracesyms.o     \
+	usersymbols.o ../shared/classcodes.o

-# CFLAGS := $(CFLAGS) -DDEBUG_SCANNER
-# CFLAGS := $(CFLAGS) -DDEBUG_DSTACK
+all: .dependencies $(PROGRAM)

-OBJECTS= toke.o stack.o stream.o dictionary.o macros.o scanner.o emit.o
+$(PROGRAM): $(OBJS)
+	$(CC) -o $(PROGRAM) $(OBJS) $(LDFLAGS)
+	$(STRIP) -s $(PROGRAM)

-.SUFFIXES: .c .o
+clean:
+	rm -f $(OBJS) *~

-all: main toke
-	@cd $(BUILDDIR) && strip toke
-	@echo -e "\nOpenBIOS tokenizer toke build finished\n"
+distclean: clean
+	rm -f $(PROGRAM) .dependencies
+	
+.dependencies: *.c 
+	@$(CC) $(CFLAGS) $(INCLUDES) -MM *.c > .dependencies

-# main should go to rules.make
-main:
-	@echo -e "\nWelcome to toke, the OpenBIOS fcode tokenizer.."
-	@test -r $(BUILDDIR) || ( mkdir -p $(BUILDDIR); \
-	  echo -e "\nCreating build directory $(BUILDDIR)" )
+.PHONY: all clean distclean

-toke: $(OBJECTS)
-	@echo -n "  linking tokenizer executable... "
-	@cd $(BUILDDIR) && $(CC) $(CFLAGS) -o $@ $^
-	@echo -e "\tok"
-	
-clean:
-	@test ! -d $(BUILDDIR) && \
-		echo "Architecture $(ARCH) is already clean." || \
-		( 						 \
-			echo "Cleaning up architecture $(ARCH)"; \
-			rm -rf $(BUILDDIR)			 \
-			rm forth.dict.core			 \
-		)
+-include .dependencies

-toke.o:       toke.c toke.h stack.h stream.h emit.h
-dictionary.o: dictionary.c toke.h dictionary.h
-emit.o:       emit.c toke.h stack.h
-macros.o:     macros.c toke.h
-scanner.o:    scanner.c toke.h stack.h stream.h dictionary.h
-stack.o:      stack.c toke.h stack.h
-stream.o:     stream.c toke.h
+.c.o:
+	$(CC) -c $(CFLAGS) $(INCLUDES) $< -o $@ 
+

Deleted: fcode-utils/toke/Rules.make
===================================================================
--- fcode-utils/toke/Rules.make	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/Rules.make	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,17 +0,0 @@
-# tag: Makefile rules
-
-VPATH    := $(VPATH):.
-
-.S.o:
-	echo -n "  assembling $<... "
-	$(CC) -c -nostdlib $(INCLUDES) $(CFLAGS) $< -o $(BUILDDIR)/$@ && \
-		echo -e "    \t\tok" || \
-		echo -e "    \t\tfailed"
-
-.c.o:
-	@echo -n "  compiling $<... "
-	@$(CC) -c $(CFLAGS) $(INCLUDES) $< -o $(BUILDDIR)/$@ && \
-		echo -e "   \t\tok" || \
-		echo -e "   \t\failed"
-		
-

Added: fcode-utils/toke/clflags.c
===================================================================
--- fcode-utils/toke/clflags.c	                        (rev 0)
+++ fcode-utils/toke/clflags.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,729 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Command-Line Flags are used to control certain non-Standard
+ *          variant behaviors of the Tokenizer.
+ *      Support Functions for setting, clearing, displaying, etc.
+ *      Call them "Special-Feature Flags" in messages to the User
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          For a given CLFlag name, the user may enter either:
+ *
+ *                   -f CLFlagName
+ *              or
+ *                   -f noCLFlagName
+ *
+ *          to either enable or disable the associated function
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          set_cl_flag                 Set (or clear) a CL Flag Variable
+ *          show_all_cl_flag_settings   Show CL Flags' settings unconditionally.
+ *          list_cl_flag_settings       Display CL Flags' settings if changed.
+ *          list_cl_flag_names          Display just the names of the CL Flags.
+ *          cl_flags_help               Help Message for CL Flags
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Revision History:
+ *          Updated Mon, 08 Aug 2005 by David L. Paktor
+ *          They're not just for setting from the Command-Line anymore,
+ *              but let's still keep these names for internal use....
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          The CL_FLAGS data structure has a field for the CLFlagName,
+ *          one for a text explanation of the function it controls, and
+ *          one for the address of the boolean variable ("flag")
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "clflags.h"
+#include "errhandler.h"
+
+
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *              (The "flags" controlled by this means)
+ *
+ **************************************************************************** */
+
+bool ibm_locals = FALSE ;
+bool ibm_locals_legacy_separator = TRUE ;
+bool ibm_legacy_separator_message = TRUE ;
+bool enable_abort_quote = TRUE ;
+bool sun_style_abort_quote = TRUE ;
+bool abort_quote_throw = TRUE ;
+bool string_remark_escape = TRUE ;
+bool hex_remark_escape = TRUE ;
+bool c_style_string_escape = TRUE ;
+bool always_headers = FALSE ;
+bool always_external = FALSE ;
+bool verbose_dup_warning = TRUE ;
+bool obso_fcode_warning = TRUE ;
+bool trace_conditionals = FALSE ;
+bool big_end_pci_image_rev = FALSE ;
+
+/*  And one to trigger a "help" message  */
+bool clflag_help = FALSE;
+
+/* **************************************************************************
+ *
+ *     The addition of the "upper/lower-case-tokens" flags introduces
+ *         some complications.  These are the variables we will actually
+ *         be exporting:
+ *
+ **************************************************************************** */
+
+bool force_tokens_case       = FALSE ;
+bool force_lower_case_tokens = FALSE ;
+
+/* **************************************************************************
+ *
+ *         but we will be entering two static variables into the table,
+ *         and keep two more to detect when a change is made...
+ *
+ **************************************************************************** */
+static bool upper_case_tokens = FALSE ;
+static bool lower_case_tokens = FALSE ;
+static bool was_upper_case_tk = FALSE ;
+static bool was_lower_case_tk = FALSE ;
+
+/* **************************************************************************
+ *
+ *              Internal Static Variables
+ *     cl_flag_change        A change was made to any of the CL Flags
+ *              Internal Static Constants
+ *     cl_flags_list         List of CL Flags and their data.
+ *
+ **************************************************************************** */
+
+static bool cl_flag_change = FALSE;
+
+static const cl_flag_t cl_flags_list[] = {
+  /*  The clflag_tabs field takes at least one tab.
+   *  If the name has fewer than 16 characters,
+   *  stick in an extra tab, and yet another tab
+   *  if the name is shorter than 8 characters
+   *  to make the formatting of the "explanation"
+   *  come out prettier.
+   */
+  { "Local-Values",
+        &ibm_locals,
+	"\t\t",
+	    "Support IBM-style Local Values ("LV"s)"     } ,
+
+  { "LV-Legacy-Separator",
+        &ibm_locals_legacy_separator,
+	"\t",
+	    "Allow Semicolon for Local Values Separator ("Legacy")"     } ,
+
+  { "LV-Legacy-Message",
+        &ibm_legacy_separator_message,
+	"\t",
+	    "Display a Message when Semicolon is used as the "
+		"Local Values Separator" } ,
+
+  { "ABORT-Quote",
+        &enable_abort_quote,
+	"\t\t",
+	    "Allow ABORT" macro"     } ,
+
+  { "Sun-ABORT-Quote",
+        &sun_style_abort_quote,
+	"\t\t",
+	    "ABORT" with implicit IF ... THEN"     } ,
+
+  { "ABORT-Quote-Throw",
+        &abort_quote_throw,
+	"\t",
+	    "Use -2 THROW in an Abort" phrase, rather than ABORT"     } ,
+
+  { "String-remark-escape",
+        &string_remark_escape,
+	"\t",
+	    "Allow "\ (Quote-Backslash) to interrupt string parsing"     } ,
+
+  { "Hex-remark-escape",
+        &hex_remark_escape,
+	"\t",
+	    "Allow \ (Backslash) to interrupt "
+		"hex-sequence parsing within a string"     } ,
+
+  { "C-Style-string-escape",
+        &c_style_string_escape ,
+	"\t",
+	    "Allow \n \t and \xx\ for special chars in string parsing"  } ,
+
+  { "Always-Headers",
+        &always_headers ,
+	"\t\t",
+	    "Override "headerless" and force to "headers"" } ,
+
+  { "Always-External",
+        &always_external ,
+	"\t\t",
+	    "Override "headerless" and "headers" and "
+		"force to "external"" } ,
+
+  { "Warn-if-Duplicate",
+        &verbose_dup_warning ,
+	"\t",
+	    "Display a WARNING message when a duplicate definition is made" } ,
+
+  { "Obsolete-FCode-Warning",
+        &obso_fcode_warning ,
+	"\t",
+	    "Display a WARNING message when an "obsolete" "
+		"(per the Standard) FCode is used" } ,
+
+  { "Trace-Conditionals",
+        &trace_conditionals,
+	"\t",
+	    "Display ADVISORY messages about the state of "
+		"Conditional Tokenization" } ,
+
+  { "Upper-Case-Token-Names",
+        &upper_case_tokens,
+	"\t",
+	    "Convert Token-Names to UPPER-Case" } ,
+
+
+  { "Lower-Case-Token-Names",
+        &lower_case_tokens,
+	"\t",
+	    "Convert Token-Names to lower-Case" } ,
+
+
+  { "Big-End-PCI-Rev-Level",
+        &big_end_pci_image_rev,
+	"\t",
+	    "Save the Vendor's Rev Level field of the PCI Header"
+		" in Big-Endian format" } ,
+
+
+  /*  Keep the "help" pseudo-flag last in the list  */
+  { "help",
+        &clflag_help,
+	    /*  Two extra tabs if the name is shorter than 8 chars  */
+	"\t\t\t",
+	    "Print this "Help" message for the Special-Feature Flags" }
+
+};
+
+static const int number_of_cl_flags =
+	 sizeof(cl_flags_list)/sizeof(cl_flag_t);
+
+
+/* **************************************************************************
+ *
+ *          CL Flags whose settings are changed in the source file should
+ *              not stay in effect for the duration of the entire batch of
+ *              tokenizations (i.e., if multiple input files are named on
+ *              the command line) the way command-line settings should.
+ *          To accomplish this we will collect the state of the flags into
+ *              a bit-mapped variable after the command line has been parsed
+ *              and restore them to their collected saved state before an
+ *              input file is processed.
+ *
+ **************************************************************************** */
+
+static long int cl_flags_bit_map;
+/*  If the number of CL Flags ever exceeds the number of bits in a long
+ *      (presently 32), we will need to change both this variable and
+ *      the routines that use it.  Of course, if the number of CL Flags
+ *      ever gets that high, it will be *seriously* unwieldy...   ;-}
+ */
+
+/* **************************************************************************
+ *
+ *      Function name:  adjust_case_flags
+ *      Synopsis:       If the last CL Flag Variable setting changed one of
+ *                          the "upper/lower-case-tokens" flags, make the 
+ *                          appropriate adjustments.
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Local Static Variables:
+ *             was_upper_case_tk      State of "upper-case-tokens" flag before
+ *                                        last CL Flag Variable was processed
+ *             was_lower_case_tk      State of "lower-case-tokens" flag, before
+ *             upper_case_tokens      State of "upper-case-tokens" flag after
+ *                                        last CL Flag V'ble was processed
+ *             lower_case_tokens      State of "lower-case-tokens" flag, after
+ *         Global Variables:
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Global Variables:
+ *             force_tokens_case          TRUE if "upper/lower-case-tokens"
+ *                                            flag is in effect
+ *             force_lower_case_tokens    If force_tokens_case is TRUE, then
+ *                                            this switches between "upper"
+ *                                            or "lower" case
+ *
+ *      Process Explanation:
+ *          We cannot come out of this with both  upper_case_tokens  and
+ *               lower_case_tokens  being TRUE, though they may both be FALSE.
+ *          If neither has changed state, we need not do anything here.
+ *              If one has gone to TRUE, we must force the other to FALSE and
+ *                  we will set  force_tokens_case  to TRUE.
+ *              If one has gone to FALSE, turn  force_tokens_case  to FALSE.
+ *              If  force_tokens_case  is TRUE after all this, we must adjust
+ *                   force_lower_case_tokens  according to  lower_case_tokens
+ *
+ **************************************************************************** */
+
+static void adjust_case_flags( void)
+{
+    static bool *case_tokens[2] = { &upper_case_tokens, &lower_case_tokens };
+    static bool *was_case_tk[2] = { &was_upper_case_tk, &was_lower_case_tk };
+    int the_one = 0;
+    int the_other = 1;
+
+    for ( ; the_one < 2 ; the_one++ , the_other-- )
+    {
+	/*  If one has changed state   */
+	if ( *(case_tokens[the_one]) != *(was_case_tk[the_one]) )
+	{
+	    if ( *(case_tokens[the_one]) )
+	    {
+	        /*  If it has gone to TRUE, force the other to FALSE.  */
+		*(case_tokens[the_other]) = FALSE;
+	        /*      and set  force_tokens_case  to TRUE  */
+		force_tokens_case = TRUE;
+	    }else{
+		/*  If it has gone to FALSE turn  force_tokens_case FALSE  */
+		force_tokens_case = FALSE;
+	    }
+	    if ( force_tokens_case )
+	    {
+	        force_lower_case_tokens = lower_case_tokens;
+	    }
+	    break;  /*  Only one can have changed state.   */
+	}
+    }
+}
+
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  set_cl_flag
+ *      Synopsis:       Set (or clear) the named CL Flag Variable
+ *
+ *      Inputs:
+ *         Parameters:
+ *             flag_name           The name as supplied by the user
+ *             from_src            TRUE if called from source-input file
+ *         Static Constants:
+ *             cl_flags_list
+ *             number_of_cl_flags
+ *
+ *      Outputs:
+ *         Returned Value:         TRUE if supplied name is not valid
+ *         Global Variables:
+ *             The CL Flag Variable associated with the supplied name will
+ *                 be set or cleared according to the leading "no"
+ *         Local Static Variables:
+ *             cl_flag_change      TRUE if associated variable has changed
+ *         Printout:
+ *             If  from_src  is TRUE, print "En" or "Dis" abling:
+ *                 followed by the explanation
+ *
+ *      Error Detection:
+ *          If the supplied name is not a valid CL Flag name, or if
+ *              it's too short to be a valid CL Flag name, return TRUE.
+ *          Print a message;  either a simple print if this function was
+ *              called from a command-line argument, or an ERROR if it
+ *              was called from a line in the from source-input file.
+ *
+ *      Process Explanation:
+ *          Save the current state of the "upper/lower-case-tokens" flags
+ *          If the given name has a leading "no", make note of that fact
+ *              and remove the leading "no" from the comparison.
+ *          Compare with the list of valid CL Flag names.
+ *          If no match was found, Error.  See under Error Detection.
+ *          If a match:
+ *              Change the associated variable according to the leading "no"
+ *              Set  cl_flag_change  to TRUE  unless the variable is the one
+ *                  associated with the "help" flag; this permits the
+ *                  "Default" vs "Setting" part of  cl_flags_help() to
+ *                  work properly...
+ *              Do the conditional Printout (see above)
+ *          Adjust the "upper/lower-case-tokens" flags if one has changed.
+ *
+ **************************************************************************** */
+static bool first_err_msg = TRUE;  /*  Need extra carr-ret for first err msg  */
+bool set_cl_flag(char *flag_name, bool from_src)
+{
+    bool retval = TRUE;
+
+    was_upper_case_tk = upper_case_tokens;
+    was_lower_case_tk = lower_case_tokens;
+
+    if ( strlen(flag_name) > 3 )
+    {
+	int indx;
+	bool flagval = TRUE;
+	char *compar = flag_name;
+
+	if ( strncasecmp( flag_name, "no", 2) == 0 )
+	{
+	    flagval = FALSE;
+	    compar += 2;
+	}
+	for ( indx = 0 ; indx < number_of_cl_flags ; indx++ )
+	{
+	    if ( strcasecmp( compar, cl_flags_list[indx].clflag_name ) == 0 )
+	    {
+		retval = FALSE;
+		*(cl_flags_list[indx].flag_var) = flagval;
+
+		/*  The "help" flag is the last one in the list  */
+		if ( indx != number_of_cl_flags - 1 )
+		{
+		    cl_flag_change = TRUE;
+		}
+		if ( from_src )
+		{
+		    tokenization_error(INFO,
+		    "%sabling:  %s\n",
+		    flagval ? "En" : "Dis", cl_flags_list[indx].clflag_expln);
+		}
+		break;
+	    }
+	}
+    }
+
+    if ( retval )
+    {
+       const char* msg_txt = "Unknown Special-Feature Flag:  %s\n" ;
+       if ( from_src )
+       {
+           tokenization_error( TKERROR, (char *)msg_txt, flag_name);
+       }else{
+	   if ( first_err_msg )
+	   {
+	       printf( "\n");
+	       first_err_msg = FALSE;
+	   }
+	   printf( msg_txt, flag_name);
+       }
+    }
+
+    adjust_case_flags();
+
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  show_all_cl_flag_settings
+ *      Synopsis:       Display the settings of the CL Flags, (except "help")
+ *                          regardless of whether they have been changed.
+ *
+ *      Associated Tokenizer directive(s):        [FLAGS]
+ *                                                #FLAGS
+ *                                                [#FLAGS]
+ *                                                SHOW-FLAGS
+ *          This routine may also be invoked by a combination of
+ *              options on the command-line.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             from_src                TRUE if called from source-input file
+ *         Macro:
+ *             ERRMSG_DESTINATION        Error message destination;
+ *                                           (Development-time switch)
+ *         Static Constants:
+ *             cl_flags_list
+ *             number_of_cl_flags
+ *
+ *      Outputs:
+ *         Returned Value:                  NONE
+ *         Printout:    Directed to  stdout or to stderr 
+ *                          (see definition of ERRMSG_DESTINATION)
+ *             A header line, followed by the names of the CL Flags,
+ *                  with "No" preceding name if value is FALSE, one to a line.
+ *
+ *      Process Explanation:
+ *          If from_src is TRUE, print the header line as a Message, and
+ *              then direct output to  ERRMSG_DESTINATION .
+ *          Don't print the "help" trigger (the last flag in the array).
+ *
+ **************************************************************************** */
+
+void show_all_cl_flag_settings(bool from_src)
+{
+    const char* hdr_txt = "Special-Feature Flag settings:" ;
+    int indx;
+
+    if ( from_src )
+    {
+	tokenization_error(MESSAGE, (char *)hdr_txt);
+    }else{
+	printf("\n%s\n", hdr_txt);
+    }
+
+    for ( indx = 0 ; indx < (number_of_cl_flags - 1) ; indx++ )
+    {
+	fprintf( from_src ? ERRMSG_DESTINATION : stdout ,
+	    "\t%s%s\n",
+		*(cl_flags_list[indx].flag_var) ? "  " : "No" ,
+		    cl_flags_list[indx].clflag_name );
+    }
+    if ( from_src )   fprintf( ERRMSG_DESTINATION, "\n");
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  list_cl_flag_settings
+ *      Synopsis:       Display the settings of the CL Flags, (except "help")
+ *                          if any of them have been changed
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Local Static Variables:        
+ *             cl_flag_change          TRUE if a Flag setting has been changed.
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Printout:
+ *             Settings of the CL Flags.  See   show_all_cl_flag_settings()
+ *
+ *      Process Explanation:
+ *          Don't print anything if  cl_flag_change  is not TRUE
+ *
+ **************************************************************************** */
+
+void list_cl_flag_settings(void)
+{
+
+    if ( cl_flag_change )
+    {
+	show_all_cl_flag_settings( FALSE);
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  list_cl_flag_names
+ *      Synopsis:       Display just the names of the CL Flags
+ *                      for the Usage message
+ *
+ *      Inputs:
+ *         Parameters:                      NONE
+ *         Static Constants:
+ *             cl_flags_list                
+ *             number_of_cl_flags           
+ *
+ *      Outputs:
+ *         Returned Value:                  NONE
+ *         Printout:
+ *             A header line, followed by the names of the CL Flags,
+ *
+ **************************************************************************** */
+
+void list_cl_flag_names(void)
+{
+    int indx;
+
+    printf("Valid Special-Feature Flags are:\n");
+    for ( indx = 0 ; indx < number_of_cl_flags ; indx++ )
+    {
+        printf("\t%s\n", cl_flags_list[indx].clflag_name );
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  cl_flags_help
+ *      Synopsis:       Display Usage of the CL Flags and their defaults
+ *                      
+ *
+ *      Inputs:
+ *         Parameters::                      NONE
+ *         Static Constants:
+ *             cl_flags_list
+ *             number_of_cl_flags
+ *         Local Static Variables:
+ *             cl_flag_change                TRUE if setting has been changed.
+ *
+ *      Outputs:
+ *         Returned Value:                   NONE
+ *         Printout:
+ *             A few lines of header, followed by the default, the name
+ *             and the "explanation" of each of the CL Flags, one to a line.
+ *
+ *      Extraneous Remarks:
+ *          We take advantage of the facts that this routine is called
+ *              (1) only from the command-line, before any settings
+ *              have been changed, and (2) via changing the flag for
+ *              "help" to TRUE.  (Technically, I suppose, the default
+ *              for the "help" feature is "no", but showing will, I
+ *              think be more confusing than enlightening to the user.)
+ *          Also, I suppose a perverse user could change setting(s) on
+ *              the same command-line with a "-f help" request; we cannot
+ *              stop users from aiming at their boot and pulling the
+ *              trigger.  As my buddies in Customer Support would say:
+ *              "KMAC YOYO"  (Approximately, "You're On Your Own, Clown")...
+ *
+ *      Revision History:
+ *          Oh, all right.  If the user changed setting(s), we can do
+ *              them the minor courtesy of printing "Setting" instead
+ *              of "Default".
+ *
+ *
+ **************************************************************************** */
+
+void cl_flags_help(void )
+{
+    int indx;
+
+    printf("\n"
+           "Special-Feature Flags usage:\n"
+           "  -f   FlagName   to enable the feature associated with FlagName,\n"
+           "or\n"
+           "  -f noFlagName   to disable the feature.\n\n"
+                "%s   Flag-Name\t\t  Feature:\n\n",
+	            cl_flag_change ? "Setting" : "Default" );
+
+   for ( indx = 0 ; indx < number_of_cl_flags ; indx++ )
+    {
+	printf("  %s    %s%s%s\n",
+	    *(cl_flags_list[indx].flag_var) ? "  " : "no" ,
+	    cl_flags_list[indx].clflag_name,
+	    cl_flags_list[indx].clflag_tabs,
+	    cl_flags_list[indx].clflag_expln);
+    }
+
+}
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  save_cl_flags
+ *      Synopsis:       Collect the state of the CL Flags
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *         Local Static Variables:
+ *             cl_flags_list
+ *         Static Constants:
+ *             number_of_cl_flags
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Local Static Variables:
+ *             cl_flags_bit_map            Will be set to reflect the state
+ *                                             of the CL Flags in the list.
+ *
+ *      Process Explanation:
+ *          The correspondence of bits to the list is that the first item
+ *              in the list corresponds to the low-order bit, and so on
+ *              moving toward the high-order with each successive item.
+ *          Do not save the "help" flag (last item on the list).
+ *          This routine is called after the command line has been parsed.
+ *
+ **************************************************************************** */
+
+void save_cl_flags(void)
+{
+    int indx;
+    long int moving_bit = 1;
+
+    cl_flags_bit_map = 0;
+    for ( indx = 0 ; indx < (number_of_cl_flags - 1) ; indx++ )
+    {
+	if ( *(cl_flags_list[indx].flag_var) )
+	{
+	    cl_flags_bit_map |= moving_bit;  /*  The moving finger writes,  */
+	}
+	moving_bit <<= 1;                    /*  and having writ, moves on. */
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_cl_flags
+ *      Synopsis:       Restore the CL Flags to the state that was saved.
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *         Local Static Variables:
+ *             cl_flags_bit_map            Reflects the state of the CL Flags
+ *         Static Constants:
+ *             number_of_cl_flags
+ *
+ *      Outputs:
+ *         Returned Value:                  NONE
+ *         Local Static Variables:
+ *             cl_flags_list
+ *         Global Variables:
+ *             The CL Flag Variables will be set or cleared
+ *                 to their saved state
+ *
+ *      Process Explanation:
+ *          This routine is called before starting a new input file.
+ *              Any changes made in the source file will not stay
+ *              in effect for the next tokenization.
+ *
+ **************************************************************************** */
+
+void reset_cl_flags(void)
+{
+    int indx;
+    long int moving_bit = 1;
+
+    for ( indx = 0 ; indx < (number_of_cl_flags - 1) ; indx++ )
+    {
+	*(cl_flags_list[indx].flag_var) =
+	    BOOLVAL( cl_flags_bit_map & moving_bit) ;
+	moving_bit <<= 1;
+    }
+}

Added: fcode-utils/toke/clflags.h
===================================================================
--- fcode-utils/toke/clflags.h	                        (rev 0)
+++ fcode-utils/toke/clflags.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,144 @@
+#ifndef _TOKE_CLFLAGS_H
+#define _TOKE_CLFLAGS_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Function Prototypes and External declarations
+ *          for Command-Line Flags Support
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Structure-Types:
+ *          cl_flag_t            Data for recognizing and setting Command-Line
+ *                                   Flags, and for displaying the help message
+ *
+ *      Function Prototypes / Functions Exported:
+ *          set_cl_flag
+ *          list_cl_flag_settings
+ *          list_cl_flag_names          Display just the names of the CL Flags
+ *          cl_flags_help
+ *
+ *      Global Variables Exported
+ *                (These flags, which are set by the set_cl_flag() routine,
+ *                     are used throughout the Tokenizer to control certain
+ *                     non-Standard variant behaviors.)
+ *          ibm_locals
+ *          ibm_locals_legacy_separator
+ *          ibm_legacy_separator_message
+ *          enable_abort_quote
+ *          sun_style_abort_quote
+ *          string_remark_escape
+ *          hex_remark_escape
+ *          c_style_string_escape
+ *          always_headers
+ *          always_external
+ *          verbose_dup_warning
+ *          obso_fcode_warning
+ *          trace_conditionals
+ *          force_tokens_case
+ *          force_lower_case_tokens
+ *          big_end_pci_image_rev
+ *          clflag_help
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *          Structure-Type Name:    cl_flag_t
+ *               Data for recognizing and setting Command-Line Flags,
+ *                   and for displaying the help message
+ *
+ *   Fields:
+ *       clflag_name     *char          CL Flag name, as entered by the user
+ *       flag_var        *bool          Address of boolean ("flag") variable
+ *       clflag_tabs     *char          Tabs to align the explanations, in
+ *                                          the help message display
+ *       clflag_expln    *char          Explanation, used in help message
+ *
+ *   Since this structure will be initialized by the program, and will not
+ *       be added-to, we can structure it as purely an array, and have no
+ *       need to treat it as a linked list, hence no link-field.
+ *
+ **************************************************************************** */
+
+#include "types.h"
+
+typedef struct cl_flag
+    {
+	char             *clflag_name;
+	bool             *flag_var;
+	char             *clflag_tabs;
+	char             *clflag_expln;
+    }  cl_flag_t ;
+
+
+/* **************************************************************************
+ *
+ *          Exported Global Variables
+ *
+ **************************************************************************** */
+
+extern bool ibm_locals;
+extern bool ibm_locals_legacy_separator;
+extern bool ibm_legacy_separator_message;
+extern bool enable_abort_quote;
+extern bool sun_style_abort_quote;
+extern bool abort_quote_throw;
+extern bool string_remark_escape;
+extern bool hex_remark_escape;
+extern bool c_style_string_escape;
+extern bool always_headers;
+extern bool always_external;
+extern bool verbose_dup_warning;
+extern bool obso_fcode_warning;
+extern bool trace_conditionals;
+extern bool big_end_pci_image_rev;
+
+extern bool force_tokens_case;
+extern bool force_lower_case_tokens;
+
+extern bool clflag_help;
+
+/* **************************************************************************
+ *
+ *          Exported Functions
+ *
+ **************************************************************************** */
+
+bool set_cl_flag(char *flag_name, bool print_message);
+void cl_flags_help(void);
+void list_cl_flag_names(void);
+void show_all_cl_flag_settings(bool from_src);
+void list_cl_flag_settings(void);
+void save_cl_flags(void);
+void reset_cl_flags(void);
+
+#endif   /*  _TOKE_CLFLAGS_H    */

Added: fcode-utils/toke/conditl.c
===================================================================
--- fcode-utils/toke/conditl.c	                        (rev 0)
+++ fcode-utils/toke/conditl.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,772 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Conditional-Compilation support for Tokenizer
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          init_conditionals_vocab    Initialize the "Conditionals" Vocabulary.
+ *          handle_conditional         Confirm whether a given name is a valid
+ *                                         Conditional, and, if so, perform its
+ *                                         function and return an indication.
+ *          create_conditional_alias   Add an alias to "Conditionals" vocab
+ *          reset_conditionals         Reset the "Conditionals" Vocabulary
+ *                                         to its "Built-In" position.
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <string.h>
+#include <errno.h>
+
+#include "scanner.h"
+#include "errhandler.h"
+#include "ticvocab.h"
+#include "conditl.h"
+#include "stack.h"
+#include "dictionary.h"
+#include "vocabfuncts.h"
+#include "usersymbols.h"
+#include "stream.h"
+#include "clflags.h"
+
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              statbuf               Start of input-source buffer
+ *              pc                    Input-source Scanning pointer
+ *              iname                 Current Input File name
+ *              lineno                Current Line Number in Input File
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *              Local Static Variables
+ *      already_ignoring      Location from which to pass a parameter,
+ *                            called "alr_ign" to the main routine.  Each
+ *                            "Conditional" will have an associated routine
+ *                            that takes the pointer to this as its argument.
+ *                            The pointer to this will satisfy the "param-field"
+ *                            requirement of a TIC_HDR-style "Vocabulary"-list.
+ *      conditionals_tbl      TIC_HDR-style "Vocabulary"-list table, initialized
+ *                            as an array.
+ *      conditionals          Pointer to "tail" of Conditionals Vocabulary-list
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      The lists of synonymous forms of the #ELSE and #THEN operators
+ *          are incorporated into the "Shared Words" Vocabulary.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Function name:  is_a_then  /  is_an_else
+ *      Synopsis:       Indicate whether the given name is one of the
+ *                      [then] / [else]  synonyms
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             a_word                        Word to test
+ *
+ *      Outputs:
+ *         Returned Value:       TRUE if the given name is one of the synonyms
+ *
+ *      Process Explanation:
+ *          The functions are twins, hence bundling them like this...
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Support function:  is_a_type
+ *      Synopsis:          Indicate whether the given name is a "shared word"
+ *                         whose FWord Token matches the one given
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                        Target name to look for
+ *             fw_type                      The FWord Token type to match
+ *
+ *      Outputs:
+ *         Returned Value:                  TRUE if it matches
+ *
+ **************************************************************************** */
+
+
+static bool is_a_type( char *tname, fwtoken fw_type)
+{
+    bool retval = FALSE;
+    tic_fwt_hdr_t *found = (tic_fwt_hdr_t *)lookup_shared_f_exec_word( tname );
+    if ( found != NULL )
+    {
+        if ( found->pfield.fw_token == fw_type )  retval = TRUE;
+    }
+    return ( retval );
+}
+
+static bool is_a_then( char *a_word)
+{
+    bool retval = is_a_type( a_word, CONDL_ENDER);
+    return ( retval );
+}
+
+static bool is_an_else( char *a_word)
+{
+    bool retval = is_a_type( a_word, CONDL_ELSE);
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *     This is a somewhat roundabout way of passing an "already ignoring"
+ *         parameter to the various Conditional Operators.  Each operator's
+ *         Parameter-Field Pointer points to this.  The calling routine
+ *         must set it (or rely on the default); the routine that handles
+ *         nesting of Conditionals must save and restore a local copy.
+ *
+ **************************************************************************** */
+
+static bool already_ignoring = FALSE;
+
+/* **************************************************************************
+ *
+ *      Further down, will define and initialize a word-list table with
+ *          all the functions that implement the Conditional Operators,
+ *          and we'll link it in with the "Global Vocabulary" pointer.
+ *
+ *      We'll call the word-list the "Conditionals Vocabulary Table",
+ *          and refer to its entries as the "Conditionals Vocabulary",
+ *          even though it isn't really a separate vocabulary...
+ * 
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      We also need a few common routines to pass as "Ignoring" functions
+ *          for a few occasional words that take another word or two from
+ *          the input stream as their arguments.  For example, if the user
+ *          were to write:   alias [otherwise] [else]   and that were to
+ *          occur within a segment already being ignored, we need to make
+ *          sure that this doesn't get processed as an occurrence of  [else]
+ *          Similarly with macro-definitions.
+ *
+ *      Since we are using the term "ignore a word" to mean "look it up and
+ *          process it in Ignoring-state", we need a different term to name
+ *          this class of routine; let's use the term "skip a word" where
+ *          the "word" is strictly an input token, delimited by whitespace.
+ *
+ **************************************************************************** */
+ 
+/* **************************************************************************
+ *
+ *      Function name:  skip_a_word
+ *      Synopsis:       Consume one input-token ("word") from the
+ *                          Input Stream, with no processing
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield             "Parameter field" pointer, to satisfy
+ *                                    the calling convention, but not used
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *
+ **************************************************************************** */
+
+void skip_a_word( tic_bool_param_t pfield )
+{
+    /* signed long wlen = */ get_word();
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_a_word_in_line
+ *      Synopsis:       Consume one input-token ("word") on the same line
+ *                          as the current line of input, from the Input
+ *                          Stream, with no processing.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield             "Parameter field" pointer, to satisfy
+ *                                    the calling convention, but not used
+ *         Global Variables:
+ *             statbuf            The word being processed, which expects
+ *                                    another word on the same line; used
+ *                                    for the Error message.
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *
+ *      Error Detection:
+ *          get_word_in_line() will check and report if no word on same line.
+ *
+ **************************************************************************** */
+void skip_a_word_in_line( tic_bool_param_t pfield )
+{
+    /* bool isokay = */ get_word_in_line( statbuf);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_two_words_in_line
+ *      Synopsis:       Consume two input-tokens ("words") on the same line
+ *                          as the current line of input, from the Input
+ *                          Stream, with no processing.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield             "Parameter field" pointer, to satisfy
+ *                                    the calling convention, but not used
+ *         Global Variables:
+ *             statbuf            The word being processed, which expects
+ *                                    two words on the same line; used for
+ *                                    the Error message.
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Memory Allocated
+ *             Copy of  statbuf  for Error message.
+ *         When Freed?
+ *             End of this routine
+ *
+ *      Error Detection:
+ *          get_word_in_line() will check and report
+ *
+ **************************************************************************** */
+
+void skip_two_words_in_line( tic_bool_param_t pfield )
+{
+    char *func_cpy = strupr( strdup( statbuf));
+    if ( get_word_in_line( func_cpy) )
+    {
+        /* bool isokay = */ get_word_in_line( func_cpy);
+    }
+    free( func_cpy);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  ignore_one_word
+ *      Synopsis:       Handle a word that needs processing while "ignoring"
+ *                          Ignore the rest.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                  Target name to test
+ *         Local Static Variables:
+ *             already_ignoring       The "Already Ignoring" flag
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Global Variables:
+ *             tic_found              Set to the TIC-entry that has just been
+ *                                        found, in case it's a Macro.
+ *         Local Static Variables:
+ *             already_ignoring       Intermediately set to TRUE, then
+ *                                        returned to former state.
+ *
+ *      Process Explanation:
+ *          When we are ignoring source input, we still need to be
+ *              sensitive to the nesting of Conditional Operators, to
+ *              consume comments and user -message text-bodies, and to
+ *              expand Macros, among other things.
+ *         Rather than create special cases here for each one, we have
+ *              added an "ign_funct" pointer to those words where this
+ *              is relevant, including Conditional Operators. 
+ *          Save the state of  already_ignoring  and set it to TRUE
+ *              Execute the "Ignoring" Function associated with the entry
+ *              Restore  already_ignoring  to its previous state.
+ *          This is necessary if the word is a Conditional Operator and
+ *              is harmless otherwise.
+ *
+ **************************************************************************** */
+
+static void ignore_one_word( char *tname)
+{
+    tic_bool_hdr_t *found = (tic_bool_hdr_t *)lookup_word( tname, NULL, NULL);
+    if ( found != NULL )
+    {
+        if ( found->ign_func != NULL )
+	{
+	    bool save_already_ignoring = already_ignoring;
+	    already_ignoring = TRUE ;
+	    tic_found = (tic_hdr_t *)found;
+
+	    found->ign_func( found->pfield);
+
+	    already_ignoring = save_already_ignoring;
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  conditionally_tokenize
+ *      Synopsis:       Conduct tokenization while a Conditional-Tokenization
+ *                          operator is in effect.  This is the core of the
+ *                          implementation of Conditional-Tokenization.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             cond             The state of the Condition-Flag that the
+ *                                  immediate Conditional Operator acquired.
+ *                                  TRUE means "do not ignore".  Its sense
+ *                                  is reversed when [ELSE] is encountered.
+ *             alr_ign          TRUE means we are Already Ignoring source input,
+ *                                  except for Conditional Operators...
+ *         Global Variables:
+ *             statbuf       The symbol (word) just retrieved from input stream.
+ *             iname         Current Input File name (for Error Messages)
+ *             lineno        Current Line Number in Input File (ditto)
+ *             trace_conditionals   Whether to issue ADVISORY messages about
+ *                                      the state of Conditional Tokenization.
+ *
+ *      Outputs:
+ *         Returned Value:       NONE
+ *         Global Variables:
+ *             statbuf           Will be advanced to the balancing [THEN] op'r.
+ *             already_ignoring  Set to TRUE if nested Conditional encountered;
+ *                                  restored to previous state when done.
+ *         Memory Allocated
+ *             Duplicate of Input File name, for Error Messages
+ *         When Freed?
+ *             Just prior to exit from routine.
+ *         Printout:
+ *             ADVISORY messages, if "Trace-Conditionals" flag was selected.
+ *             
+ *         Global Behavior:
+ *            Tokenization happens, or inputs are ignored, as necessary.
+ *
+ *      Error Detection:
+ *          End-of-file encountered on reading a word    
+ *              ERROR.  Conditional Operators must be balanced within a file
+ *          More than one [ELSE] encountered:  ERROR if processing segment;
+ *              if ignoring, WARNING.
+ *
+ *      Process Explanation:
+ *          Read a word at a time.   Allow Macros to "pop" transparently,
+ *              but not source files.
+ *          If the word is a [THEN], we are done.
+ *          If the word is an [ELSE], then, if we are not Already Ignoring,
+ *                  invert the sense of whether we are ignoring source input. 
+ *              If this is not the only [ELSE] in the block, report an Error
+ *                  and disregard it.
+ *          If we are ignoring source input, for whatever reason, we still
+ *              need to be sensitive to the nesting of Conditional Operators:
+ *              If the word is a Conditional Operator, activate it with the
+ *                  "Already Ignoring" parameter set to TRUE; doing so will
+ *                  result in a nested call to this routine.
+ *              Otherwise, i.e., if the word is not a Conditional Operator,
+ *                  we may still need to process it in "ignoring" mode:
+ *                  we need, for instance, to consume strings, comments
+ *                  and the text-bodies of user-messages in their entirety,
+ *                  in case there is a reference to an [ELSE] or suchlike.
+ *                  The words that need processing while "ignoring" will
+ *                  have a valid function-pointer in their ign_func field.
+ *          If we are not ignoring source input, pass the word along to the
+ *              tokenize_one_word routine and process it.  If the word is
+ *              a Conditional Operator, it will be handled in the context
+ *              of normal (i.e., non-ignored) tokenization, and, again, a
+ *              nested call to this routine will result...
+ *
+ *      Revision History:
+ *          Updated Thu, 23 Feb 2006 by David L. Paktor
+ *              Conditional Blocks may begin with a Conditional Operator in
+ *                  a Macro definition and do not need to be concluded in
+ *                  the body of the Macro.
+ *          Updated Fri, 10 Mar 2006 by David L. Paktor
+ *              Recognize aliased string, comment and user-message delimiters
+ *                  in a segment that is being ignored; Conditional Operators
+ *                  within the text body of any of these are always consumed
+ *                  and never unintentionally processed.  Macros are always
+ *                  processed; Conditional Operators inside a Macro body are
+ *                  recognized, so the Macro continues to function as intended.
+ *
+ **************************************************************************** */
+
+static void conditionally_tokenize( bool cond, bool alr_ign )
+{
+    
+    signed long wlen;
+
+    /*  Note:  The following variables *must* remain within
+     *      the scope of this routine; a distinct instance
+     *      is needed each time this routine is re-entered
+     *     (aka "a nested call").
+     */
+    bool ignoring;
+    bool first_else = TRUE;  /*  The "else" we see is the first.  */
+    bool not_done = TRUE;
+    unsigned int cond_strt_lineno = lineno;
+    char *cond_strt_ifile_nam = strdup( iname);
+
+    ignoring = BOOLVAL( ( cond == FALSE ) || ( alr_ign != FALSE ) );
+
+    if ( trace_conditionals )
+    {
+        char *cond_val = cond ? "True" : "False" ;
+	char *cond_junct = alr_ign ? ", but Already " : "; ";
+	char *processg = ignoring ? "Ignoring" : "Processing" ;
+	tokenization_error( INFO,
+	    "Tokenization-Condition is %s%s%s.\n",
+		cond_val, cond_junct, processg);
+    }
+
+    while ( not_done )
+    {
+        wlen = get_word();
+	if ( wlen == 0 )
+	{
+	    continue;
+	}
+
+	if ( wlen < 0 )
+	{
+	    tokenization_error( TKERROR,
+	        "Conditional without conclusion; started");
+	    just_where_started( cond_strt_ifile_nam, cond_strt_lineno);
+	    not_done = FALSE ;
+	    continue;
+	}
+
+	if ( is_a_then ( statbuf ) )
+	{
+	    if ( trace_conditionals )
+	    {
+		tokenization_error( INFO,
+		    "Concluding Conditional");
+		just_started_at( cond_strt_ifile_nam, cond_strt_lineno);
+	    }
+	    not_done = FALSE ;
+	    continue;
+	}
+
+	if ( is_an_else( statbuf ) )
+	{
+	    if ( ! alr_ign )
+	    {
+		if ( first_else )
+		{
+		    ignoring = INVERSE( ignoring);
+		}
+	    }
+
+	    if ( ! first_else )
+	    {
+		int severity = ignoring ? WARNING : TKERROR ;
+		char *the_scop = ignoring ? "(ignored)" : "the" ;
+		tokenization_error( severity, "Multiple %s directives "
+		    "within %s scope of the Conditional",
+			 strupr(statbuf), the_scop);
+		just_started_at( cond_strt_ifile_nam, cond_strt_lineno);
+	    }else{
+		first_else = FALSE;
+		if ( trace_conditionals )
+		{
+		    char *when_enc = alr_ign ? "While already" : "Now" ;
+		    char *processg = alr_ign ? "ignoring" :
+				    ignoring ? "Ignoring" : "Processing" ;
+		    char *enc       = alr_ign ? ", e" : ".  E" ;
+
+		    tokenization_error( INFO,
+			"%s %s%sncountered %s belonging to Conditional",
+			    when_enc, processg, enc, strupr(statbuf) );
+		    just_started_at( cond_strt_ifile_nam, cond_strt_lineno);
+		}
+	    }
+
+	    continue;
+	}
+
+	/*  If we are ignoring source input, for whatever reason, we still
+	 *      need to be sensitive to the nesting of Conditional Operators
+	 *      and some other commands and directives, as indicated...
+	 */
+	if ( ignoring )
+	{
+	    ignore_one_word( statbuf );
+	}else{
+	    /*  And if we're not ignoring source input, process it! */
+	    tokenize_one_word ( wlen );
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      We will now define a series of fairly simple functions that
+ *          will be performed by the various Conditional Operators in
+ *          the "Conditionals Vocabulary".
+ *
+ *      Each one takes, as an argument, the "parameter field" pointer,
+ *          which, in all cases, points to the local  already_ignoring 
+ *          flag, passed as an int to satisfy C's strong-typing.  The
+ *          routine will internally recast it as a  bool .
+ *
+ *      If it is TRUE, the routine will bypass the test for its particular
+ *          type of condition, and go directly to  conditionally_tokenize
+ *          In most cases, testing for the condition would be harmless,
+ *          but in the case where the test is for an item on the stack,
+ *          it would be harmful because the sequence that put the item
+ *          on the stack was also being ignored...
+ *
+ *      We'll give these functions short prologs.  Synonyms will simply
+ *          have separate entries in the Vocabulary Table, associated
+ *          with the same function.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      But first, a support routine...
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Function name:  conditional_word_in_line
+ *      Synopsis:       Common code for the types of conditionals that
+ *                          require a word on the same line.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             alr_ign         TRUE if we are already ignoring
+ *             exist_test      TRUE if the test is for "existence" of the word
+ *             exist_funct     Name of the function to call for the test
+ *         Global Variables:
+ *             stat_word       Word for which to test
+ *
+ *      Outputs:
+ *         Returned Value:     NONE
+ *
+ *      Error Detection:
+ *          The word in question must appear on the same line as the directive;
+ *              the call to  get_word_in_line()  checks for that and reports.
+ *          If the word did not appear on the same line, then the directive
+ *              will be disregarded and processing will proceed as though it
+ *              were absent.  This may lead to a cascade of errors...
+ *
+ *      Process Explanation:
+ *          The supplied  exist_funct()  will test for the existence of
+ *              the word, now read into  statbuf , in the appropriate 
+ *              venue.
+ *          We only call the  exist_funct()  if we are not already ignoring.
+ *
+ **************************************************************************** */
+
+static void conditional_word_in_line( bool alr_ign,
+                                          bool exist_test,
+                                              bool (*exist_funct)() )
+{
+    if ( get_word_in_line( statbuf) )
+    {
+    	bool cond = FALSE;
+	if ( INVERSE( alr_ign) )
+	{
+	    bool exists = exist_funct( statbuf);
+	    cond = BOOLVAL( exists == exist_test);
+	}
+	conditionally_tokenize( cond, alr_ign );
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  if_exists
+ *      Synopsis:       Test for existence of a given word, in the dictionary.
+ *
+ *      Associated Tokenizer directives:        [ifexist]
+ *                                              #ifexist
+ *                                              [#ifexist]
+ *                                              [ifexists]
+ *                                              #ifexists
+ *                                              [#ifexists]
+ *        (Note variants with and without final 's'
+ *
+ **************************************************************************** */
+
+static void if_exists( tic_param_t pfield )
+{
+    bool alr_ign = *pfield.bool_ptr;
+    conditional_word_in_line( alr_ign, TRUE, exists_in_current );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  if_not_exist
+ *      Synopsis:       Test for Non-existence, in the appropriate dictionary,)
+ *                          of the given word.
+ *
+ *      Associated Tokenizer directives:        [ifnexist]
+ *                                              #ifnexist
+ *                                              [#ifnexist]
+ *        (Note:  Variants with final 's' didn't make sense here.)
+ *
+ *      Explanatory Notes:
+ *          This is the exact inverse of  if_exists
+ *
+ **************************************************************************** */
+
+static void if_not_exist( tic_bool_param_t pfield )
+{
+    bool alr_ign = *pfield.bool_ptr;
+    conditional_word_in_line( alr_ign, FALSE, exists_in_current );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  if_defined
+ *      Synopsis:       Test for existence of a user-defined symbol
+ *
+ *      Associated Tokenizer directives:        [ifdef]
+ *                                              #ifdef
+ *                                              [#ifdef]
+ *
+ **************************************************************************** */
+
+static void if_defined( tic_bool_param_t pfield )
+{
+    bool alr_ign = *pfield.bool_ptr;
+    conditional_word_in_line( alr_ign, TRUE, exists_as_user_symbol );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  if_not_defined
+ *      Synopsis:       Test for NON-existence of a user-defined symbol
+ *
+ *      Associated Tokenizer directives:        [ifndef]
+ *                                              #ifndef
+ *                                              [#ifndef]
+ *
+ **************************************************************************** */
+
+static void if_not_defined( tic_bool_param_t pfield )
+{
+    bool alr_ign = *pfield.bool_ptr;
+    conditional_word_in_line( alr_ign, FALSE, exists_as_user_symbol );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  if_from_stack
+ *      Synopsis:       Test the number on top of the run-time stack
+ *
+ *      Associated Tokenizer directive:         [if]
+ *
+ *      Process Explanation:
+ *          When we are ignoring source input, and we still need to be
+ *              sensitive to the nesting of Conditional Operators, we
+ *              will not consume the number on the stack; this function
+ *              is after all, being ignored and should not perform any
+ *              action other than making sure the [else]s and [then]s
+ *              get properly counted.
+ *
+ **************************************************************************** */
+
+static void if_from_stack( tic_bool_param_t pfield )
+{
+    bool alr_ign = *pfield.bool_ptr;
+    bool cond = FALSE;
+
+    if ( ! alr_ign )
+    {
+        long num = dpop();
+	if (num != 0)
+	{
+	    cond = TRUE;
+	}
+    }
+    conditionally_tokenize( cond, alr_ign );
+}
+
+/*  For future functions, use  conditl.BlankTemplate.c  */
+
+/* **************************************************************************
+ *
+ *      Here, at long last, we define and initialize the structure containing
+ *          all the functions we support for Conditional Operators.
+ *
+ **************************************************************************** */
+
+#define ADD_CONDL(str, func )   BUILTIN_BOOL_TIC(str, func, already_ignoring )
+
+static tic_bool_hdr_t conditionals_vocab_tbl[] = {
+    ADD_CONDL ("[ifexist]"   , if_exists      ) ,
+    ADD_CONDL ("[ifexists]"  , if_exists      ) ,
+    ADD_CONDL ("#ifexist"    , if_exists      ) ,
+    ADD_CONDL ("#ifexists"   , if_exists      ) ,
+    ADD_CONDL ("[#ifexist]"  , if_exists      ) ,
+    ADD_CONDL ("[#ifexists]" , if_exists      ) ,
+    ADD_CONDL ("[ifnexist]"  , if_not_exist   ) ,
+    ADD_CONDL ("#ifnexist"   , if_not_exist   ) ,
+    ADD_CONDL ("[#ifnexist]" , if_not_exist   ) ,
+    ADD_CONDL ("[ifdef]"     , if_defined     ) ,
+    ADD_CONDL ("#ifdef"      , if_defined     ) ,
+    ADD_CONDL ("[#ifdef]"    , if_defined     ) ,
+    ADD_CONDL ("[ifndef]"    , if_not_defined ) ,
+    ADD_CONDL ("#ifndef"     , if_not_defined ) ,
+    ADD_CONDL ("[#ifndef]"   , if_not_defined ) ,
+    ADD_CONDL ("[if]"        , if_from_stack  )
+};
+
+
+/* **************************************************************************
+ *
+ *      Function name:  init_conditionals_vocab
+ *      Synopsis:       Initialize the "Conditionals Vocabulary Table"
+ *                          link-pointers dynamically, and link it in
+ *                          with the given ("Global") Vocabulary pointer.
+ *
+ **************************************************************************** */
+
+void init_conditionals_vocab( tic_hdr_t **tic_vocab_ptr )
+{
+    static const int conditionals_vocab_max_indx =
+	 sizeof(conditionals_vocab_tbl)/sizeof(tic_bool_hdr_t);
+
+    init_tic_vocab( (tic_hdr_t *)conditionals_vocab_tbl,
+                        conditionals_vocab_max_indx,
+                            tic_vocab_ptr );
+}
+

Added: fcode-utils/toke/conditl.h
===================================================================
--- fcode-utils/toke/conditl.h	                        (rev 0)
+++ fcode-utils/toke/conditl.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,46 @@
+#ifndef _TOKE_CONDITL_H
+#define _TOKE_CONDITL_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Function Prototypes for Conditional-Compilation support for Tokenizer
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "types.h"
+#include "ticvocab.h"
+
+void init_conditionals_vocab( tic_hdr_t **tic_vocab_ptr );
+
+void skip_a_word( tic_bool_param_t pfield );
+void skip_a_word_in_line( tic_bool_param_t pfield );
+void skip_two_words_in_line( tic_bool_param_t pfield );
+
+#endif   /* _TOKE_CONDITL_H    */

Added: fcode-utils/toke/devnode.c
===================================================================
--- fcode-utils/toke/devnode.c	                        (rev 0)
+++ fcode-utils/toke/devnode.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,595 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Support routines for managing device-node vocabularies
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      The vocabulary that is created for a device-node must not remain
+ *          available outside of that node.  Also, nodes may be nested,
+ *          child within parent.
+ *      An attempt within a child-node to access directly a method defined
+ *          in the parent must be flagged as an error.  (Consider what would
+ *          happen if the method in the parent-node used instance data, and
+ *          the child-node has an instance of its own.)
+ *      The correct way is to invoke the method via "$call-parent" or the like.
+ *
+ *      We will, however, allow the user to specify a group of exceptions,
+ *          words whose scope will be "global" within the tokenization.
+ *          When "global" scope is initiated, definitions will be made to
+ *          the "core" vocabulary until "device" scope is resumed.
+ *          That will (mostly) all be handled in  dictionary.c 
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          new_device_vocab        Create the new device-node's data-structure
+ *          delete_device_vocab     Remove device-node's data-structure
+ *          finish_device_vocab     Remove struct and give messages when
+ *                                      device is "finish"ed.
+ *          exists_in_ancestor      Issue a Message if the given word exists
+ *                                      in an ancestor of the current dev-node.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Add a pair of fields to the data-structure for the Input File and
+ *              Line Number where "finish-device" occurred.  When a device-node
+ *              is "finish"ed, do not delete it, but instead fill in those
+ *              fields and the move the node to a separate linked-list.
+ *          When looking whether a word exists in an ancestor-node, also
+ *              check whether it was in a device-node that was finished and
+ *              print both where it was started and where it was finished.
+ *
+ **************************************************************************** */
+
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "devnode.h"
+#include "errhandler.h"
+#include "scanner.h"
+#include "vocabfuncts.h"
+#include "flowcontrol.h"
+#include "stream.h"
+#include "ticvocab.h"
+
+
+/* **************************************************************************
+ *
+ *      Tokenization starts with an implicit "new-device" in effect.
+ *          The top-level device-node is never removed.
+ *
+ *      Initialize it here
+ *
+ **************************************************************************** */
+char default_top_dev_ifile_name[] = "Start of tokenization";
+
+static device_node_t top_level_dev_node = {
+     	NULL ,                          /*  parent_node    */
+	default_top_dev_ifile_name ,	/*  ifile_name.
+					 *     Something to show, Just In Case
+					 */
+	0 ,                             /*  line_no        */
+	NULL ,                          /*  tokens_vocab   */
+};
+
+/* **************************************************************************
+ *
+ *          Global Variables Exported.
+ *                                  Pointers to:
+ *     current_device_node      data-structure of current device-node
+ *     current_definitions      vocab into which to add def'ns.
+ *
+ **************************************************************************** */
+
+device_node_t *current_device_node = &top_level_dev_node;
+tic_hdr_t **current_definitions = &(top_level_dev_node.tokens_vocab);
+
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *               These are used to support the routines  in_what_node() 
+ *                   and  show_node_start() , which are used to facilitate
+ *                   certain kinds of Messaging, as described later.
+ *
+ *     in_what_buffr       Buffer for the  in_what_node()  string
+ *     show_where          TRUE if the string needs to be followed-up
+ *     show_which          TRUE if follow-up should be  just_where_started()
+ *                             rather than  just_started_at()
+ *     in_what_line        Line Number to use in the follow-up
+ *     in_what_file        File Name to use in the follow-up
+ *
+ **************************************************************************** */
+
+static char in_what_buffr[50];   /*  Ought to be more than enough.  */
+static bool show_where = FALSE;
+static bool show_which;
+static int in_what_line;
+static char *in_what_file;
+
+
+/* **************************************************************************
+ *
+ *      Function name:  dev_vocab_control_struct_check
+ *      Synopsis:       Issue Warnings for unresolved flow-control constructs
+ *                          at start or end of a device-node.
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *         Global Variables:
+ *             statbuf                     The command being processed.
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Printout:
+ *             Handled by  announce_control_structs() routine
+ *
+ *      Error Detection:
+ *          Handled by  announce_control_structs()  routine
+ *
+ *      Process Explanation:
+ *          Set up a buffer with the error message, based on  statbuf
+ *              and pass it to  announce_control_structs()
+ *          Release it when done.
+ *
+ **************************************************************************** */
+
+static void dev_vocab_control_struct_check( void)
+{
+    char *ccs_messg;
+
+    ccs_messg = safe_malloc(strlen(statbuf) + 32,
+        "Device-Node control-structure check");
+    
+    strcpy( ccs_messg, statbuf );
+    strupr( ccs_messg);
+    strcat( ccs_messg, " encountered");
+    announce_control_structs( WARNING, ccs_messg, 0 );
+    free( ccs_messg);
+}
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  new_device_vocab
+ *      Synopsis:       Create and initialize the data-structure for a
+ *                      new device-node when a "new-device" is created,
+ *                      with messages as appropriate.
+ *
+ *      Inputs:
+ *         Parameters:                     NONE
+ *         Global Variables:
+ *             statbuf                     The word that was just read.
+ *             iname                       Current Input-File Name
+ *             lineno                      Current line-number
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Global Variables:
+ *             current_device_node         Will point to the new data-structure
+ *         Memory Allocated
+ *             Space for the new  device_node_t  data-structure
+ *             Space for a copy of the current input file name
+ *         When Freed?
+ *             By delete_device_vocab(), when the device-node is "finish"ed.
+ *         Printout:
+ *             Advisory message.
+ *
+ *      Error Detection:
+ *          In immediate-execution mode, Control Structures that have not
+ *              been completed are questionable;  Issue WARNINGS via the
+ *               dev_vocab_control_struct_check()  routine.
+ *
+ *      Process Explanation:
+ *          This routine is called when "new-device" is invoked, but only
+ *              if we are in immediate-execution mode.
+ *          Later on, in ERROR- or INFOrmative messages, we will want to
+ *              be able to refer to the file and line-number in which this
+ *              was encountered, so we include them in the structure.
+ *
+ **************************************************************************** */
+
+void new_device_vocab( void )
+{
+    device_node_t *new_node_data;
+
+    dev_vocab_control_struct_check();
+
+    /*  Advisory message will mention previous device-node
+     *      if there was one.  Either way starts out the same:
+     */
+#define NEW_DEV_MSG_START  "Encountered %s.  Starting new device-node."
+
+    if ( current_device_node == &top_level_dev_node )
+    {
+	tokenization_error(INFO, NEW_DEV_MSG_START "\n", statbuf );
+    }else{
+	tokenization_error(INFO, NEW_DEV_MSG_START
+	    "  Suspending definitions of parent-device node", statbuf );
+	started_at( current_device_node->ifile_name,
+	     current_device_node->line_no );
+    }
+
+    /*  Now to business...   */
+    new_node_data = safe_malloc( sizeof(device_node_t),
+        "creating new-device vocab data" );
+    new_node_data->parent_node = current_device_node;
+    new_node_data->ifile_name = strdup(iname);
+    new_node_data->line_no = lineno;
+    new_node_data->tokens_vocab = NULL;
+
+    current_device_node = new_node_data;
+    
+    current_definitions = &(current_device_node->tokens_vocab);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  delete_device_vocab
+ *      Synopsis:       Remove the vocabularies of the current device-node,
+ *                          along with its data-structure, when the device
+ *                          is "finish"ed; do not print messages.
+ *                      Do not remove the top-level device-node data-struct.
+ *
+ *      Associated FORTH words:              FINISH_DEVICE  (interpretive state)
+ *                                           END0  END1
+ *      Associated Tokenizer directives:     RESET-SYMBOLS  (in "Normal" mode)
+ *                                           FCODE-END
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Global Variables:
+ *             current_device_node        Points to current device's struct
+ *                                            Leads to chain of dev-node structs
+ *
+ *      Outputs:
+ *         Returned Value: 
+ *         Global Variables:
+ *             current_device_node        Parent-device's struct becomes current
+ *         Memory Freed
+ *             All that was allocated for the tokens and the definers
+ *                 vocabs in the current device-node
+ *             The copy of the input file name, except the top-level
+ *             The current_device_node data-structure, except the top-level
+ *
+ **************************************************************************** */
+
+void delete_device_vocab( void )
+{
+    reset_tic_vocab( current_definitions, NULL );
+
+    if ( current_device_node != &top_level_dev_node )
+    {
+	device_node_t *temp_node = current_device_node;
+	current_device_node = current_device_node->parent_node;
+	free( temp_node->ifile_name );
+	free(temp_node);
+    }
+
+    current_definitions = &(current_device_node->tokens_vocab);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  finish_device_vocab
+ *      Synopsis:       Remove the device-node data-structure and all its
+ *                          vocabularies when the device is "finish"ed,
+ *                          with appropriate messages.
+ *                      Do not remove the top-level device node data-struct.
+ *
+ *      Associated FORTH word:                 FINISH_DEVICE
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Global Variables:
+ *             current_device_node        Current device's struct pointer
+ *
+ *      Outputs:
+ *         Returned Value:                NONE
+ *         Global Variables:
+ *             current_device_node        Parent-device's struct becomes current
+ *         Printout:
+ *             Advisory message.
+ *
+ *      Error Detection:
+ *          If current_device_node is already pointing at the top-level
+ *              device node, it means there was no corresponding NEW-DEVICE
+ *              Issue an ERROR.
+ *          In immediate-execution mode, Control Structures that have not
+ *              been completed are questionable;  Issue WARNINGS via the
+ *               dev_vocab_control_struct_check()  routine.
+ *
+ *      Process Explanation:
+ *          This routine is called when "finish-device" is invoked, but only
+ *              if we are in immediate-execution mode.
+ *
+ **************************************************************************** */
+
+void finish_device_vocab( void )
+{
+    bool at_top_level;
+
+    dev_vocab_control_struct_check();
+
+    /*   We never remove the top-level device-node vocabulary,
+     *       so we need to test whether we're about to.
+     */
+
+    at_top_level = BOOLVAL( current_device_node == &top_level_dev_node );
+    if ( at_top_level )
+    {
+        tokenization_error( TKERROR,
+	    "Encountered %s without corresponding NEW-DEVICE.  "
+	    "Resetting definitions since start of tokenization.\n",
+		statbuf );
+    }else{    
+	tokenization_error(INFO,
+	    "Encountered %s.  Resetting definitions of device node",
+		statbuf );
+	started_at( current_device_node->ifile_name,
+	     current_device_node->line_no );
+    }
+
+    /*  Now to business...   */
+    delete_device_vocab();
+
+    /*   Did we just get to the top-level device-node vocabulary
+     *       when we weren't before?
+     */
+    if ( INVERSE(at_top_level) )
+    {
+	if ( current_device_node == &top_level_dev_node )
+	{
+	    tokenization_error(INFO,
+		"Resuming definitions since start of tokenization.\n" );
+	}else{
+	    tokenization_error(INFO,
+		"Resuming definitions of parent device-node" );
+	    started_at( current_device_node->ifile_name,
+		 current_device_node->line_no );
+	}
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  in_what_node
+ *      Synopsis:       Format a string for use in a Message that might
+ *                          identify the start of the given device-node.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             the_node               The device-node vocabulary about which
+ *                                        to construct the identifying phrase.
+ *         Local Static Variables:
+ *             in_what_buffr          Buffer in which to format the string.
+ *         Global Variables:
+ *             current_definitions    Device-node vocabulary currently
+ *                                        in effect.
+ *
+ *      Outputs:
+ *         Returned Value:           Pointer to buffer w/ formatted string
+ *         Local Static Variables:
+ *             in_what_buffr         Will contain the formatted string.
+ *             show_where            TRUE if the string needs to be followed-up
+ *                                        (i.e., did not contain a terminating
+ *                                        new-line) by  just_where_started()
+ *                                        or by  just_started_at()
+ *             show_which            TRUE if the follow-up call should be
+ *                                       to  just_where_started()  rather 
+ *                                       than to  just_started_at() 
+ *             in_what_line          Copy of line_no field from the_node
+ *             in_what_file          Copy of ifile_name field from the_node
+ *
+ *      Process Explanation:
+ *          Calling routine must ascertain that Global-scope is not in effect.
+ *          The returned phrase can be used as a string argument in a Message.
+ *          Set  show_where  TRUE if the_node->line_no is non-zero.
+ *          Set  show_which  TRUE if the_node is either the Current or the
+ *              Top-Level device-node
+ *          If the originating line-number in the given Node structure is zero,
+ *              the returned phrase will contain a terminating new-line.
+ *              (This only happens if the given Node is the top-level Node,
+ *              and it's the Current Node, and the "official" starting-point
+ *              hasn't yet been established by an "FCode-Starter" such as 
+ *               FCODE-VERSION2 .  Once that command has been given, even
+ *              definitions that were made prior to it belong to the Node
+ *              that started there.)
+ *          Otherwise,  show_where  is returned TRUE, and  show_which  becomes
+ *              relevant.  If the given node is the Current or the Top-Level
+ *              node, text about the originating file-name and line-number
+ *              merely describes a node that is already uniquely identified,
+ *              so the message appended to the buffer will have the phrase
+ *              "which began" (which introduces what is known in grammar as
+ *              an Appositive Subordinate Clause) and  show_which  will be
+ *              returned TRUE.  If the given node is not uniquely identifiable
+ *              without the file- and line- phrase, then the Subordinate Clause
+ *              is Indicative, and should be introduced with "that" (and no
+ *              comma); in that case, we will return  show_which  as FALSE.
+ *          After the calling routine displays the message in which the
+ *              returned phrase is used, it must call  show_node_start()
+ *              to display the followe-up message, if any.
+ *
+ **************************************************************************** */
+
+char *in_what_node(device_node_t *the_node)
+{
+    bool top_node    = BOOLVAL( the_node == &top_level_dev_node);
+    bool curr_node   = BOOLVAL( the_node == current_device_node);
+    bool known_node  = BOOLVAL( top_node || curr_node );
+    bool no_line     = BOOLVAL( the_node->line_no == 0);
+
+    show_where   = INVERSE( no_line  );
+    show_which   = known_node;
+    in_what_line = the_node->line_no;
+    in_what_file = the_node->ifile_name;
+
+    sprintf( in_what_buffr, "in the%s device-node%s",
+	INVERSE( known_node )  ? ""
+	        :  top_node    ?    " top-level"   :  " current" ,
+	
+	no_line                ?  ".\n"
+	        :  known_node  ?  ", which began"  :   ""         );
+
+    
+    return( in_what_buffr);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  show_node_start
+ *      Synopsis:       Follow-up to the  in_what_node()  call.  Print out,
+ *                          if applicable, the text about the originating
+ *                          file-name and line-number
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Local Static Variables:
+ *             show_where          Nothing to do if not TRUE
+ *             show_which          TRUE if should call  just_where_started()
+ *                                     rather than  just_started_at()
+ *             in_what_line        Line Number to use in the follow-up
+ *             in_what_file        File Name to use in the follow-up
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Local Static Variables:
+ *             show_where          Force to FALSE
+ *         Printout:
+ *             Follow-up to  the in_what_node() call.  Applicable text
+ *                 about the originating file-name and line-number.
+ *
+ *      Process Explanation:
+ *          By forcing  show_where  to FALSE after this is called, we
+ *              can safely allow routines that might or might not have
+ *              called  in_what_node()  to call this routine, without
+ *              needing any additional "bookkeeping".
+ *
+ **************************************************************************** */
+
+void show_node_start( void)
+{
+    if ( show_where)
+    {
+	if ( show_which )
+	{
+	    just_where_started( in_what_file, in_what_line);
+	}else{
+	    just_started_at( in_what_file, in_what_line);
+	}
+	show_where = FALSE;
+    }
+}
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_in_ancestor
+ *      Synopsis:       Issue a Message and return an indication if
+ *                          the given word exists in an ancestor of
+ *                          the current device-node.
+ *                      Used for additional error-message information.
+ *                      
+ *
+ *      Inputs:
+ *         Parameters:
+ *             m_name                   "Method" name
+ *         Global Variables:
+ *             current_device_node      Leads to chain of dev-node data-structs
+ *             scope_is_global          TRUE if "global" scope is in effect
+ *
+ *      Outputs:
+ *         Returned Value:              TRUE if word found
+ *         Printout:
+ *             If  m_name  exists in an ancestor-node, print an ADVISORY
+ *                 giving the location where the ancestor originated.
+ *
+ *      Error Detection:
+ *          None here.  Calling routine detected error; see below.
+ *
+ *      Process Explanation:
+ *          This routine was called as the result of detecting an error:
+ *              viz.,  m_name  was not found in either the current node
+ *              or the base vocabulary.  (Except:  If "global" scope is
+ *              in effect, we didn't search the current device-node).
+ *
+ **************************************************************************** */
+
+bool exists_in_ancestor( char *m_name)
+{
+    tic_hdr_t *found;
+    bool retval = FALSE;
+    if ( current_device_node != NULL )
+    {
+	device_node_t *grandpa = current_device_node->parent_node;
+
+	if ( scope_is_global )    grandpa = current_device_node;
+
+	for ( ; grandpa != NULL; grandpa = grandpa->parent_node )
+	{
+	    found = lookup_tic_entry( m_name, grandpa->tokens_vocab);
+	    if ( found != NULL )
+	    {
+		retval = TRUE;
+		break;
+	    }
+	}
+	if ( grandpa != NULL )
+	{
+	    char as_what_buf[32] = "";
+	    if ( as_a_what( found->fword_defr, as_what_buf) )
+	    {
+		strcat( as_what_buf, " ");
+	    }
+	    tokenization_error(INFO, "%s is defined %s%s", m_name,
+		as_what_buf, in_what_node( grandpa) );
+	    show_node_start();
+	}
+    }
+
+    return(retval );
+}

Added: fcode-utils/toke/devnode.h
===================================================================
--- fcode-utils/toke/devnode.h	                        (rev 0)
+++ fcode-utils/toke/devnode.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,86 @@
+#ifndef _TOKE_DEVNODE_H
+#define _TOKE_DEVNODE_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      External/Prototype/Structure definitions for device-node management
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "types.h"
+#include "ticvocab.h"
+
+/* **************************************************************************
+ *          Structure Name:    device_node_t
+ *                        Data for managing a device node; pointers
+ *                            to vocabs, data for messaging.
+ *                            
+ *   Fields:
+ *       parent_node         Pointer to similar data for parent node 
+ *       line_no             Copy of Line Number where "new-device" was invoked
+ *       ifile_name          Name of Input File where "new-device" was invoked
+ *       tokens_vocab        Pointer to vocab for this device's tokens
+ *
+ **************************************************************************** */
+
+typedef struct device_node {
+        struct device_node *parent_node ;
+	char *ifile_name ;
+	unsigned int line_no ;
+	tic_hdr_t *tokens_vocab ;
+} device_node_t;
+
+
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */
+
+extern char default_top_dev_ifile_name[];
+extern device_node_t *current_device_node;
+extern tic_hdr_t **current_definitions;
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+void new_device_vocab( void );
+void delete_device_vocab( void );
+void finish_device_vocab( void );
+char *in_what_node(device_node_t *the_node);
+void show_node_start( void);
+bool exists_in_ancestor( char *m_name);
+
+#endif   /*  _TOKE_DEVNODE_H    */

Modified: fcode-utils/toke/dictionary.c
===================================================================
--- fcode-utils/toke/dictionary.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/dictionary.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,6 +24,12 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #if defined(__linux__) && ! defined(__USE_BSD)
@@ -32,543 +38,2008 @@
 #include <string.h>
 #include <errno.h>

-#include "toke.h"
+#include "emit.h"
+#include "macros.h"
+#include "scanner.h"
+#include "ticvocab.h"
 #include "dictionary.h"
+#include "vocabfuncts.h"
+#include "devnode.h"
+#include "clflags.h"
+#include "parselocals.h"
+#include "errhandler.h"
+#include "tokzesc.h"
+#include "conditl.h"

-typedef struct token {
-	u8  *name;
-	u16 fcode;
-	struct token *next;
-} token_t;
+/* **************************************************************************
+ *
+ *      Revision History:
+ *          Updated Fri, 29 Jul 2005 by David L. Paktor
+ *          Retrofit to handle "Current Device" as a separate vocabulary;
+ *              if one is in effect, searches through it occur first, as
+ *              do definitions to it, ahead of the general vocabulary.  This
+ *              is to support managing device-node vocabularies correctly.
+ *          Updated Mon, 12 Dec 2005 by David L. Paktor
+ *          Allow the user to specify a group of exceptions, words whose
+ *              scope will be "global" within the tokenization.  Under "global"
+ *              scope, definitions will be made to the "core" vocabulary.
+ *
+ *          Wed, 14 Dec 2005 by David L. Paktor
+ *          Found a problem with the current approach.  Need to be able to
+ *              temporarily suspend meaning of "instance".  Define:  (1) an
+ *              alias for  INSTANCE  called  GENERIC_INSTANCE  (2) a macro
+ *              called  INSTANCE  that effectively no-ops it out; and, when
+ *              it is time to restore "INSTANCE" to normal functionality,
+ *              (3) an alias for  GENERIC_INSTANCE  called  INSTANCE .
+ *          Problem is that macros are treated as a separate vocabulary
+ *              from FWords (and their aliases) and searching one before the
+ *              other (either way) renders the second one searched unable to
+ *              supercede the first one:  If macros are searched first, (2)
+ *              will be found ahead of the built-in FWord (which is what we
+ *              want) but later, when we search for (3) among the FWords, it
+ *              will not be found ahead of (2).  If, on the other hand, we
+ *              search FWords first, the macro defined in (2) will never be
+ *              found.
+ *          We need a way to define both (all?) types of definitions in a
+ *              single vocabulary that will honor the LIFO order of def'ns.
+ *
+ *          Mon, 19 Dec 2005 by David L. Paktor
+ *          Begin development of implementation of a way to define both (all?)
+ *              types of definitions in a single  tic_hdr_t  type vocabulary.
+ *
+ **************************************************************************** */

-static token_t *dictionary=NULL;
-static token_t *forthwords=NULL;

-static u16 lookup_token_dict(char *name, token_t *dict)
+
+/* **************************************************************************
+ *
+ *      We will be creating several different lists of initial built-in
+ *          definitions; together, they constitute the Global Vocabulary.
+ *          (We will avoid the term "dictionary", since, in classical
+ *          Forth terminology, it refers to the complete collection of
+ *          vocabularies in an application.)  The usage of the pointer
+ *          to the Global Vocabulary is central to the operation of this
+ *          program and the maintenance programmer needs to understand it.
+ *          We may also refer to the Global Vocabulary as the "core" vocab.
+ *
+ *      Each initial list will be created as an array of TIC-header entries.
+ *          Because the global vocabulary is expandable by the user,
+ *          we will not be searching the lists as arrays but rather as
+ *          linked-lists; the run-time initialization routine will fill
+ *          in their link-fields and also will link the various lists
+ *          together, so we can group their initial declarations according
+ *          to our convenience.
+ *
+ *      A single pointer, called the "Global Vocabulary Dictionary Pointer"
+ *          (okay, so classical Forth terminology isn't completely rigorous...)
+ *          and abbreviated GV_DP, will point to the "tail" of the "thread".
+ *          Similar vocabularies will be created for the device-nodes; look
+ *          in the file  devnode.fth  for a more detailed discussion of those.
+ *
+ *      The "FC-Tokens" list contains the names and FCode numeric tokens
+ *          of the straightforward FORTH words that simply write a token
+ *          directly to the tokenized output.  We need to access these
+ *          without being confused by aliases or other distractions, so
+ *          we will keep a pointer to them especially for that purpose.
+ *      Therefore it is IMPORTANT:  that the "FC-Tokens" list MUST be the
+ *          first table linked by the initialization routine, so that its
+ *          last-searched entry's link-field is NULL.
+ *
+ *      The "FWords" list contains FORTH words that require additional
+ *          special action at tokenization-time.  Their numeric values
+ *          are derived from the  fword_token  enumeration declaration,
+ *          and are used as the control-expression for a SWITCH statement
+ *          with a large number of CASE labels in the  handle_internal()
+ *          function.
+ *
+ *      The "Shared Words" list contains FORTH words that can be executed
+ *          similarly both during "Tokenizer Escape" mode (i.e., the scope
+ *          of the special brackets:  tokenizer[  ...   ]tokenizer ) and
+ *          also within "Normal Tokenization" mode.  Their numeric values
+ *          are derived and used the same way as the "FWords".  Since we
+ *          will be needing to do a separate search through them at times,
+ *          we will also need a lower-bracket pointer for them.  (An upper
+ *          bracket is irrelevant for these, because aliases can be added.
+ *          This is not the case for the "FC-Tokens" list, because searches
+ *          through those will be conducted from within this program.)
+ *
+ *      The "definer" field in the TIC-header structure is primarily used to
+ *          detect attempts to apply the  TO  directive to an inappropriate
+ *          target.  Its numeric values are a subset of the "FWord tokens".
+ *          Certain "FC-Token" names are specified to be valid  TO  targets;
+ *          their entries' "definer" fields will be initialized accordingly.
+ *          Entries in FWord Token lists that are "shared" between "Normal
+ *          Tokenization" and "Tokenizer Escape" modes will have their
+ *          "definer" fields initialized to  COMMON_FWORD .  All other
+ *          entries' "definer" fields will be initialized to  UNSPECIFIED .
+ *
+ *      Other files maintain and support additional lists with the same
+ *          structure, which need to be linked together with the lists
+ *          declared here.  We prefer to keep the  GV_DP  private to this
+ *          file, so it will be passed as a parameter where needed.  (I'm
+ *          not pleased to note, however, that it can't be kept completely
+ *          private; it's needed for add_user_macro() and possibly other
+ *          functions outside this file.)
+ *
+ *      The words that can only be used during "Tokenizer Escape" mode and
+ *          the IBM-style "Locals", as well as the device-node vocabularies,
+ *          will need to be separate and will not be linked together with
+ *          the Global Vocabulary.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      We'll be initializing the lists later, but will be referencing
+ *          the pointers sooner, so we need to declare the pointers here.
+ *
+ *      We will keep all of these pointers private to this file.
+ *
+ **************************************************************************** */
+
+static tic_hdr_t *global_voc_dict_ptr  = NULL;  /*  The Global Vocabulary    */
+static tic_hdr_t *fc_tokens_list_ender = NULL;  /*  Tokens search ends here  */
+static tic_hdr_t *fc_tokens_list_start = NULL;  /*  Start the search here    */
+static tic_hdr_t *shared_fwords_ender  = NULL;  /*  Shared FWords search end */
+static tic_hdr_t *global_voc_reset_ptr = NULL;  /*  Reset-point for G.V.     */
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_core_word
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                      word in the "Global" Vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to look up
+ *         Local Static Variables:
+ *             global_voc_dict_ptr       "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to the data-structure, or
+ *                                            NULL if not found.
+ *
+ **************************************************************************** */
+
+tic_hdr_t *lookup_core_word( char *tname)
 {
-	token_t *curr;
-	
-	for (curr=dict; curr!=NULL; curr=curr->next)
-		if (!strcasecmp(name,(char *)curr->name))
-			break;
+    tic_hdr_t *found ;

-	if (curr)
-		return curr->fcode;
-#ifdef DEBUG_TOKE
-	printf("warning: token '%s' does not exist.\n", name);
-#endif
-	return 0xffff;
+    found = lookup_tic_entry( tname, global_voc_dict_ptr);
+    return ( found ) ;
 }

-u16 lookup_token(char *name)
+/* **************************************************************************
+ *
+ *      Function name:  exists_in_core
+ *      Synopsis:       Confirm whether the given name exists in the
+ *                      Global (aka "core") Vocabulary.  Search the
+ *                      Global Vocabulary exclusively.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             name                      The name for which to look
+ *         Local Static Variables:
+ *             global_voc_dict_ptr       "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:               TRUE if name is found.
+ *
+ **************************************************************************** */
+
+bool exists_in_core( char *name)
 {
-	return lookup_token_dict(name, dictionary);
+    return exists_in_tic_vocab( name, global_voc_dict_ptr );
 }

-u16 lookup_fword(char *name)
+/* **************************************************************************
+ *
+ *      Function name:  handle_core_word
+ *      Synopsis:       Perform a function in the "Global" Vocabulary and
+ *                      indicate whether the name is valid.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to handle
+ *         Local Static Variables:
+ *             global_voc_dict_ptr       "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:   TRUE if the given name is valid in Global Vocab
+ *
+ *      Error Detection:
+ *          If the name is not in the "Global" Vocabulary, let the calling
+ *              routine determine whether to print an error message or to
+ *              try it out as a number.
+ *
+ **************************************************************************** */
+
+bool handle_core_word( char *tname )
 {
-	return lookup_token_dict(name, forthwords);
+    bool retval;
+
+    retval = handle_tic_vocab( tname, global_voc_dict_ptr );
+
+    return ( retval ) ;
 }

-static int add_token_dict(u16 number, char *name, token_t **dict)
+
+/* **************************************************************************
+ *
+ *      Function name:  create_core_alias
+ *      Synopsis:       Create, in the "Global" ("core") Vocabulary, an entry
+ *                          for NEW_NAME that behaves the same as the latest
+ *                          definition of OLD_NAME, and whose behavior will
+ *                          not change even if a new definition of OLD_NAME
+ *                          is overlaid.  Indicate if successful.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             new_name          The name for the new alias to be created
+ *             old_name          The name of the old function to be duplicated
+ *         Local Static Variables:
+ *             global_voc_dict_ptr        "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:                TRUE if OLD_NAME was found.
+ *         Local Static Variables:
+ *             global_voc_dict_ptr        Updated with the new entry
+ *         Memory Allocated
+ *             By support routine.
+ *
+ *      Process Explanation:
+ *          Both the "old" and "new" names are presumed to already point to
+ *              stable, freshly allocated memory-spaces.
+ *
+ **************************************************************************** */
+
+bool create_core_alias( char *new_name, char *old_name)
 {
-	token_t *curr;
+    bool retval = create_tic_alias( new_name, old_name, &global_voc_dict_ptr);
+    return ( retval );
+}

-	curr=malloc(sizeof(token_t));
-	if(!curr) {
-		printf("Out of memory while adding token.\n");
-		exit(-ENOMEM);
+/* **************************************************************************
+ *
+ *      The functions that go into the various lists' FUNCT field may be
+ *           defined below, or might be defined externally.
+ *
+ *      Often, we will need a function that merely recasts the type of the
+ *           parameter field before passing it to the function that does
+ *           the actual work.
+ *
+ *      Prologs will be brief or even non-existent.
+ *
+ *      Initialization macro definitions will accompany the functions.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      For the "FCode-Tokens" list, simply generate the token directly.
+ *      We need this layer for param type conversion.
+ *      In case we're ever able to eliminate it, (or just on General
+ *          Principles) we'll invoke it via a macro...
+ *
+ **************************************************************************** */
+
+static void emit_fc_token( tic_param_t pfield)
+{
+    u16 fc_tok = (u16)pfield.deflt_elem;
+    emit_fcode( fc_tok);
+}
+
+#define FC_TOKEN_FUNC  emit_fc_token
+
+#define BUILTIN_FCODE( tok, nam)   \
+     VALPARAM_TIC(nam, FC_TOKEN_FUNC, tok , UNSPECIFIED )
+
+/*  Built-in FCodes with known definers:  */
+#define BI_FCODE_VALUE( tok, nam)   \
+     VALPARAM_TIC(nam, FC_TOKEN_FUNC, tok , VALUE )
+
+#define BI_FCODE_VRBLE( tok, nam)   \
+     VALPARAM_TIC(nam, FC_TOKEN_FUNC, tok , VARIABLE )
+
+#define BI_FCODE_DEFER( tok, nam)   \
+     VALPARAM_TIC(nam, FC_TOKEN_FUNC, tok , DEFER )
+
+#define BI_FCODE_CONST( tok, nam)   \
+     VALPARAM_TIC(nam, FC_TOKEN_FUNC, tok , CONST )
+
+/* **************************************************************************
+ *
+ *      The "FCode-Tokens" list includes tokens that are identified
+ *         in the Standard as Obsolete.  We will define a function
+ *         that issues a WARNING before generating the token, and
+ *         assign it to those elements of the list.
+ *
+ *      Control the message via a command-line flag.
+ *
+ **************************************************************************** */
+
+static void obsolete_warning( void)
+{
+    if ( obso_fcode_warning )
+    {
+	tokenization_error( WARNING, "%s is an Obsolete FCode.\n",
+	    strupr(statbuf) );
+    }
+}
+
+static void obsolete_fc_token( tic_param_t pfield)
+{
+    obsolete_warning();
+    emit_fc_token( pfield);
+}
+
+#define OBSO_FC_FUNC  obsolete_fc_token
+
+#define OBSOLETE_FCODE( tok, nam)   \
+     VALPARAM_TIC(nam, OBSO_FC_FUNC, tok , UNSPECIFIED )
+
+#define OBSOLETE_VALUE( tok, nam)   \
+     VALPARAM_TIC(nam, OBSO_FC_FUNC, tok , VALUE )
+
+
+/* **************************************************************************
+ *
+ *      The function for most of the "FWords" list,  handle_internal() ,
+ *          is defined externally, but not exported in a  .h  file,
+ *          because we want to keep it as private as possible.
+ *      We will declare its prototype here.
+ *
+ *      Initialization macros for both "Normal Mode"-only and
+ *          "Shared" entries are also defined here.
+ *
+ *   Arguments:
+ *       fwt      (fword_token)    Value of the FWord Token (from Enum list)
+ *       nam      (string)         Name of the entry as seen in the source
+ *
+ **************************************************************************** */
+
+void handle_internal( tic_param_t pfield);
+/*  "Skip-a-string when Ignoring" function.  Same args and limited-proto ...  */
+void skip_string( tic_param_t pfield);
+
+#define FWORD_EXEC_FUNC  handle_internal
+
+#define BUILTIN_FWORD( fwt, nam)   \
+     FWORD_TKN_TIC(nam, FWORD_EXEC_FUNC, fwt, BI_FWRD_DEFN )
+
+#define SHARED_FWORD( fwt, nam)   \
+     FWORD_TKN_TIC(nam, FWORD_EXEC_FUNC, fwt, COMMON_FWORD )
+
+/*  Variants:  When Ignoring, SKip One Word  */
+#define SHR_FWD_SKOW( fwt, nam)   \
+     DUALFUNC_FWT_TIC(nam, FWORD_EXEC_FUNC, fwt, skip_a_word, COMMON_FWORD )
+
+/*  Variants:  When Ignoring, SKip one Word in line  */
+#define SH_FW_SK_WIL( fwt, nam)   \
+     DUALFUNC_FWT_TIC(nam, FWORD_EXEC_FUNC, fwt,     \
+         skip_a_word_in_line, COMMON_FWORD )
+
+/*  When Ignoring, SKip Two Words in line  */
+#define SH_FW_SK2WIL( fwt, nam)   \
+     DUALFUNC_FWT_TIC(nam, FWORD_EXEC_FUNC, fwt,     \
+         skip_two_words_in_line, COMMON_FWORD )
+
+/* **************************************************************************
+ *
+ *      Some of the entries in the "FWords" list -- both "Normal" (Built-in)
+ *          and "Shared" also act as an "Ignore-handler".
+ *          
+ *   Arguments:
+ *       nam      (string)         Name of the entry as seen in the source
+ *       afunc    (routine-name)   Name of internal "active" function
+ *       pval     (integer)        The "param field" item
+ *       ifunc    (routine-name)   Name of "ignore-handling" function
+ *
+ **************************************************************************** */
+
+#define SHARED_IG_HDLR(nam, afunc, pval, ifunc)     \
+    DUFNC_FWT_PARM(nam, afunc, pval, ifunc, COMMON_FWORD )
+
+/*  A "Shared" entry that uses the same routine for both of its functions  */
+#define SHR_SAMIG_FWRD( fwt, nam)   \
+    DUFNC_FWT_PARM(nam, FWORD_EXEC_FUNC, fwt, FWORD_EXEC_FUNC, COMMON_FWORD )
+
+/* **************************************************************************
+ *
+ *     But the "Normal" (Built-in) FWord Ignore-handler uses the same
+ *          routine as the  BUILTIN_FWORD for both of its functions.
+ *
+ *   Arguments:
+ *       fwt      (fword_token)    Value of the FWord Token (from Enum list)
+ *       nam      (string)         Name of the entry as seen in the source
+ *
+ **************************************************************************** */
+#define BI_IG_FW_HDLR( fwt, nam)   \
+    DUALFUNC_FWT_TIC(nam, FWORD_EXEC_FUNC, fwt, FWORD_EXEC_FUNC, BI_FWRD_DEFN )
+
+/*  A variant:  A "Built-In FWorD that SKiPs One Word", when Ignoring  */
+#define BI_FWD_SKP_OW( fwt, nam)   \
+     DUALFUNC_FWT_TIC(nam, FWORD_EXEC_FUNC, fwt, skip_a_word, BI_FWRD_DEFN )
+
+/*  Another variant:  A "Built-In FWorD String".  skip_string when Ignoring  */
+#define BI_FWD_STRING( fwt, nam)   \
+     DUALFUNC_FWT_TIC(nam, FWORD_EXEC_FUNC, fwt, skip_string, BI_FWRD_DEFN )
+
+/* **************************************************************************
+ *
+ *      In order to protect device-nodes' methods from being accessed 
+ *          by other device-nodes (with the attendant potential for
+ *          disastrous consequences), we must establish a few rules:
+ *
+ *      Each device-node has a separate vocabulary for its methods.
+ *          New definitions are made to the "current" device's vocabulary.
+ *          Searches for names go through the "current" device-node's
+ *              vocabulary first, then through the core dictionary.
+ *
+ *      A new-device (in interpretation-mode) creates a new device-node
+ *          vocabulary.  The node that had been current (presumably its
+ *          parent) remains in memory but inactive.
+ *
+ *      A finish-device (again, only in interpretation-mode) removes the
+ *          current device-node's vocabulary from memory; its presumed
+ *          parent once again becomes current.
+ *
+ *      Tokenization starts with an implicit "new-device" in effect.
+ *          The top-level device-node is never removed.
+ *
+ *      The Global Variable  current_definitions  points to the vocabulary
+ *          to which we will add and through which we will search first.
+ *
+ *      If "global" scope is in effect, then  current_definitions  will
+ *           point to the "Global" (also called "core") vocabulary.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Support for operations in "current" -- i.e., "global" vis-a-vis
+ *          "device" -- scope.
+ *      "Global" scope will not recognize words defined in "device" scope,
+ *           but "device" scope will recognize "global" words.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions to enter "global" scope and resume "device" scope.
+ *
+ **************************************************************************** */
+
+static tic_hdr_t **save_device_definitions;
+/* Export the indication that "global" scope is in effect  */
+bool scope_is_global = FALSE;
+
+
+void enter_global_scope( void )
+{
+    if ( scope_is_global )
+{
+	tokenization_error( WARNING,
+	    "%s -- Global Scope already in effect; ignoring.\n",
+		strupr(statbuf) );
+    }else{
+	tokenization_error( INFO,
+	    "Initiating Global Scope definitions.\n" );
+	scope_is_global = TRUE;
+	save_device_definitions = current_definitions;
+	current_definitions = &global_voc_dict_ptr;
+    }
+}
+
+void resume_device_scope( void )
+{
+    if ( scope_is_global )
+    {
+	tokenization_error( INFO,
+	    "Terminating Global Scope definitions; "
+		"resuming Device-node definitions.\n" );
+	current_definitions = save_device_definitions;
+	scope_is_global = FALSE;
+    }else{
+	tokenization_error( WARNING,
+	    "%s -- Device-node Scope already in effect; ignoring.\n",
+		strupr(statbuf) );
+    }
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_current
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                          word, either in the Current Device-Node vocab,
+ *                          or in the Global ("core") Vocabulary.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to look for
+ *         Global Variables:
+ *             current_definitions       Current vocabulary:  Device-Node, or
+ *                                           "core" if "global" scope in effect.
+ *             scope_is_global           TRUE if "global" scope is in effect
+ *         Local Static Variables:
+ *             global_voc_dict_ptr       "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:               Pointer to the data-structure, or
+ *                                           NULL if not found.
+ *
+ *      Process Explanation:
+ *          If a current Device-Node Vocabulary in effect, search through it.
+ *          If the given name was not found, and "global" scope is not in
+ *              effect (i.e., "core" was not already searched), make a
+ *              separate search through the Global ("core") Vocabulary
+ *
+ *      Extraneous Remarks:
+ *          This is the central routine for doing general word-searches that
+ *              make use of the "normal"-mode search-list.
+ *
+ **************************************************************************** */
+
+tic_hdr_t *lookup_current( char *tname)
+{
+    /*  Search Current Device Vocabulary ahead of global (core) vocabulary  */
+    tic_hdr_t *retval;
+    retval = lookup_tic_entry( tname, *current_definitions);
+    if ( (retval == NULL) && INVERSE(scope_is_global) )
+{
+	retval = lookup_core_word( tname);
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_in_current
+ *      Synopsis:       Confirm whether the given name exists either
+ *                          in the Current Device-Node vocab,
+ *                          or in the Global ("core") Vocabulary,
+ *                          or in Tokenizer Escape Mode, if that's current.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to look for
+ *
+ *      Outputs:
+ *         Returned Value:               TRUE if name is found
+ *
+ **************************************************************************** */
+
+bool exists_in_current( char *tname)
+{
+    tic_hdr_t *found = lookup_word( tname, NULL, NULL);
+    bool retval = BOOLVAL ( found != NULL);
+    return( retval);
    }

-	curr->next=*dict;
-	curr->fcode=number;
-	curr->name=(u8 *)name;

-	*dict=curr;
-	return 0;
+/* **************************************************************************
+ *
+ *      Function name:  handle_current
+ *      Synopsis:       Perform a function in the current device-node vocab,
+ *                      if one is in effect, or in the "Global" Vocabulary.
+ *                      Indicate whether the name is valid.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                      The name to handle
+ *         Global Variables:
+ *             current_definitions        Device-Node (or Global) Vocabulary
+ *                                            currently in effect.
+ *             scope_is_global            TRUE if "global" scope is in effect
+ *         Local Static Variables:
+ *
+ *      Outputs:
+ *         Returned Value:                TRUE if the given name is valid
+ *
+ **************************************************************************** */
+
+bool handle_current( char *tname )
+{
+    bool retval = handle_tic_vocab( tname, *current_definitions );
+
+    if ( INVERSE(retval) && INVERSE(scope_is_global) )
+    {
+	retval = handle_core_word( tname );
+    }
+    return ( retval );
+
 }

-int add_token(u16 number, char *name)
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_in_dev_node
+ *      Synopsis:       Return a pointer to the data-structure of the
+ *                          named word in the Current device node, if
+ *                          in "Device" scope.  Used for error-reporting.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to look for
+ *         Global Variables:
+ *             current_definitions        Device-Node (or Global) Vocabulary
+ *                                            currently in effect.
+ *             scope_is_global            FALSE if "Device" scope is in effect
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to the data-structure, or
+ *                                            NULL if not found.
+ *
+ **************************************************************************** */
+
+tic_hdr_t *lookup_in_dev_node( char *tname)
 {
-	return add_token_dict(number, name, &dictionary);
+    tic_hdr_t *retval = NULL;
+
+    if ( INVERSE(scope_is_global) )
+{
+	retval = lookup_tic_entry( tname, *current_definitions);
 }
+    return ( retval );
+}

-static int add_special(u16 number, char *name)
+
+/* **************************************************************************
+ *
+ *     In order to avoid unintentional "recursive"-ness, we need a way
+ *          to render a newly created colon-definition non-findable
+ *          until it's completed.
+ *
+ *      We will accomplish this by saving and reverting the pointer to
+ *          the newest entry, when we call the  hide_last_colon() , and
+ *          by restoring the pointer when we call  reveal_last_colon()
+ *
+ *      We need, therefore, to save the pointer to the last entry before
+ *          we create the new entry.
+ *
+ **************************************************************************** */
+
+/*  Update this each time a new definition is entered  */
+static tic_hdr_t *save_current = NULL;
+
+/* **************************************************************************
+ *
+ *      Function name:  add_to_current
+ *      Synopsis:       Add a new entry to the "current" scope of definitions,
+ *                          which may be either the Global Vocabulary or the
+ *                          current Device-Node Vocabulary.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             name                      The name of the new entry
+ *             fc_token                  The new entry's assigned FCode-number
+ *             fw_definer                The new entry's definer
+ *             define_token              If FALSE, suppress adding the entry,
+ *                                           but preserve the side-effect of
+ *                                           setting  save_current 
+ *         Global Variables:
+ *             current_definitions       Pointer to pointer to "tail" of the
+ *                                           Vocabulary currently in effect;
+ *                                           either Device-node or Global.
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:
+ *             *current_definitions    Updated with the new entry
+ *         Local Static Variables:
+ *             save_current            Pointer to previous entry
+ *         Memory Allocated
+ *             For the new entry's copy of the name.
+ *         When Freed?
+ *             When the Device-Node is "finish"ed or the Global Vocabulary
+ *                 is reset, or when the program exits.
+ *
+ *      Process Explanation:
+ *          Because  current_definitions  points to the Global Vocabulary
+ *              pointer during "global" scope, this routine is extremely
+ *              straightforward.
+ *          All user-defined words have the same action, i.e., emitting
+ *              the assigned FCode-number.  The new entry's "parameter
+ *              field" size is, of course, zero; the "ignore-function"
+ *              is NULL.
+ *
+ *      Extraneous Remarks:
+ *          The  define_token  parameter is a late addition, necessitated
+ *              by the decision to continue processing after an erroneous
+ *              attempt to create a definition inside a control-structure,
+ *              in order to catch other errors. 
+ *            
+ *
+ **************************************************************************** */
+
+void add_to_current( char *name,
+                           TIC_P_DEFLT_TYPE fc_token,
+			       fwtoken definer,
+			           bool define_token)
 {
-	return add_token_dict(number, name, &forthwords);
+    save_current = *current_definitions;
+    if ( define_token )
+{
+	char *nu_name = strdup( name);
+	add_tic_entry( nu_name, FC_TOKEN_FUNC, fc_token,
+			   definer, 0 , NULL, current_definitions );
+    }
 }

-void init_dictionary(void) 
+
+void hide_last_colon ( void )
 {
-	add_token( 0x000, "end0" );
-	add_token( 0x010, "b(lit)" );
-	add_token( 0x011, "b(')" );
-	add_token( 0x012, "b(")" );
-	add_token( 0x013, "bbranch" );
-	add_token( 0x014, "b?branch" );
-	add_token( 0x015, "b(loop)" );
-	add_token( 0x016, "b(+loop)" );
-	add_token( 0x017, "b(do)" );
-	add_token( 0x018, "b(?do)" );
-	add_token( 0x019, "i" );
-	add_token( 0x01a, "j" );
-	add_token( 0x01b, "b(leave)" );
-	add_token( 0x01c, "b(of)" );
-	add_token( 0x01d, "execute" );
-	add_token( 0x01e, "+" );
-	add_token( 0x01f, "-" );
-	add_token( 0x020, "*" );
-	add_token( 0x021, "/" );
-	add_token( 0x022, "mod" );
-	add_token( 0x023, "and" );
-	add_token( 0x024, "or" );
-	add_token( 0x025, "xor" );
-	add_token( 0x026, "invert" );
-	add_token( 0x027, "lshift" );
-	add_token( 0x028, "rshift" );
-	add_token( 0x029, ">>a" );
-	add_token( 0x02a, "/mod" );
-	add_token( 0x02b, "u/mod" );
-	add_token( 0x02c, "negate" );
-	add_token( 0x02d, "abs" );
-	add_token( 0x02e, "min" );
-	add_token( 0x02f, "max" );
-	add_token( 0x030, ">r" );
-	add_token( 0x031, "r>" );
-	add_token( 0x032, "r@" );
-	add_token( 0x033, "exit" );
-	add_token( 0x034, "0=" );
-	add_token( 0x035, "0<>" );
-	add_token( 0x036, "0<" );
-	add_token( 0x037, "0<=" );
-	add_token( 0x038, "0>" );
-	add_token( 0x039, "0>=" );
-	add_token( 0x03a, "<" );
-	add_token( 0x03b, ">" );
-	add_token( 0x03c, "=" );
-	add_token( 0x03d, "<>" );
-	add_token( 0x03e, "u>" );
-	add_token( 0x03f, "u<=" );
-	add_token( 0x040, "u<" );
-	add_token( 0x041, "u>=" );
-	add_token( 0x042, ">=" );
-	add_token( 0x043, "<=" );
-	add_token( 0x044, "between" );
-	add_token( 0x045, "within" );
-	add_token( 0x046, "drop" );
-	add_token( 0x047, "dup" );
-	add_token( 0x048, "over" );
-	add_token( 0x049, "swap" );
-	add_token( 0x04A, "rot" );
-	add_token( 0x04b, "-rot" );
-	add_token( 0x04c, "tuck" );
-	add_token( 0x04d, "nip" );
-	add_token( 0x04e, "pick" );
-	add_token( 0x04f, "roll" );
-	add_token( 0x050, "?dup" );
-	add_token( 0x051, "depth" );
-	add_token( 0x052, "2drop" );
-	add_token( 0x053, "2dup" );
-	add_token( 0x054, "2over" );
-	add_token( 0x055, "2swap" );
-	add_token( 0x056, "2rot" );
-	add_token( 0x057, "2/" );
-	add_token( 0x058, "u2/" );
-	add_token( 0x059, "2*" );
-	add_token( 0x05a, "/c" );
-	add_token( 0x05b, "/w" );
-	add_token( 0x05c, "/l" );
-	add_token( 0x05d, "/n" );
-	add_token( 0x05e, "ca+" );
-	add_token( 0x05f, "wa+" );
-	add_token( 0x060, "la+" );
-	add_token( 0x061, "na+" );
-	add_token( 0x062, "char+" );
-	add_token( 0x063, "wa1+" );
-	add_token( 0x064, "la1+" );
-	add_token( 0x065, "cell+" );
-	add_token( 0x066, "chars" );
-	add_token( 0x067, "/w*" );
-	add_token( 0x068, "/l*" );
-	add_token( 0x069, "cells" );
-	add_token( 0x06a, "on" );
-	add_token( 0x06b, "off" );
-	add_token( 0x06c, "+!" );
-	add_token( 0x06d, "@" );
-	add_token( 0x06e, "l@" );
-	add_token( 0x06f, "w@" );
-	add_token( 0x070, "<w@" );
-	add_token( 0x071, "c@" );
-	add_token( 0x072, "!" );
-	add_token( 0x073, "l!" );
-	add_token( 0x074, "w!" );
-	add_token( 0x075, "c!" );
-	add_token( 0x076, "2@" );
-	add_token( 0x077, "2!" );
-	add_token( 0x078, "move" );
-	add_token( 0x079, "fill" );
-	add_token( 0x07a, "comp" );
-	add_token( 0x07b, "noop" );
-	add_token( 0x07c, "lwsplit" );
-	add_token( 0x07d, "wljoin" );
-	add_token( 0x07e, "lbsplit" );
-	add_token( 0x07f, "bljoin" );
-	add_token( 0x080, "wbflip" );
-	add_token( 0x081, "upc" );
-	add_token( 0x082, "lcc" );
-	add_token( 0x083, "pack" );
-	add_token( 0x084, "count" );
-	add_token( 0x085, "body>" );
-	add_token( 0x086, ">body" );
-	add_token( 0x087, "fcode-revision" );
-	add_token( 0x088, "span" );
-	add_token( 0x089, "unloop" );
-	add_token( 0x08a, "expect" );
-	add_token( 0x08b, "alloc-mem" );
-	add_token( 0x08c, "free-mem" );
-	add_token( 0x08d, "key?" );
-	add_token( 0x08e, "key" );
-	add_token( 0x08f, "emit" );
-	add_token( 0x090, "type" );
-	add_token( 0x091, "(cr" );
-	add_token( 0x092, "cr" );
-	add_token( 0x093, "#out" );
-	add_token( 0x094, "#line" );
-	add_token( 0x095, "hold" );
-	add_token( 0x096, "<#" );
-	add_token( 0x097, "u#>" );
-	add_token( 0x098, "sign" );
-	add_token( 0x099, "u#" );
-	add_token( 0x09a, "u#s" );
-	add_token( 0x09b, "u." );
-	add_token( 0x09c, "u.r" );
-	add_token( 0x09d, "." );
-	add_token( 0x09e, ".r" );
-	add_token( 0x09f, ".s" );
-	add_token( 0x0a0, "base" );
-	add_token( 0x0a1, "convert" );
-	add_token( 0x0a2, "$number" );
-	add_token( 0x0a3, "digit" );
-	add_token( 0x0a4, "-1" );
-	add_token( 0x0a5, "0" );
-	add_token( 0x0a6, "1" );
-	add_token( 0x0a7, "2" );
-	add_token( 0x0a8, "3" );
-	add_token( 0x0a9, "bl" );
-	add_token( 0x0aa, "bs" );
-	add_token( 0x0ab, "bell" );
-	add_token( 0x0ac, "bounds" );
-	add_token( 0x0ad, "here" );
-	add_token( 0x0ae, "aligned" );
-	add_token( 0x0af, "wbsplit" );
-	add_token( 0x0b0, "bwjoin" );
-	add_token( 0x0b1, "b(<mark)" );
-	add_token( 0x0b2, "b(>resolve)" );
-	add_token( 0x0b3, "set-token-table" );
-	add_token( 0x0b4, "set-table" );
-	add_token( 0x0b5, "new-token" );
-	add_token( 0x0b6, "named-token" );
-	add_token( 0x0b7, "b(:)" );
-	add_token( 0x0b8, "b(value)" );
-	add_token( 0x0b9, "b(variable)" );
-	add_token( 0x0ba, "b(constant)" );
-	add_token( 0x0bb, "b(create)" );
-	add_token( 0x0bc, "b(defer)" );
-	add_token( 0x0bd, "b(buffer:)" );
-	add_token( 0x0be, "b(field)" );
-	add_token( 0x0bf, "b(code)" );
-	add_token( 0x0c0, "instance" );
-	add_token( 0x0c2, "b(;)" );
-	add_token( 0x0c3, "b(to)" );
-	add_token( 0x0c4, "b(case)" );
-	add_token( 0x0c5, "b(endcase)" );
-	add_token( 0x0c6, "b(endof)" );
-	add_token( 0x0c7, "#" );
-	add_token( 0x0c8, "#s" );
-	add_token( 0x0c9, "#>" );
-	add_token( 0x0ca, "external-token" );
-	add_token( 0x0cb, "$find" );
-	add_token( 0x0cc, "offset16" );
-	add_token( 0x0cd, "evaluate" );
-	add_token( 0x0d0, "c," );
-	add_token( 0x0d1, "w," );
-	add_token( 0x0d2, "l," );
-	add_token( 0x0d3, "," );
-	add_token( 0x0d4, "um*" );
-	add_token( 0x0d5, "um/mod" );
-	add_token( 0x0d8, "d+" );
-	add_token( 0x0d9, "d-" );
-	add_token( 0x0da, "get-token" );
-	add_token( 0x0db, "set-token" );
-	add_token( 0x0dc, "state" );
-	add_token( 0x0dd, "compile" );
-	add_token( 0x0de, "behavior" );
-	add_token( 0x0f0, "start0" );
-	add_token( 0x0f1, "start1" );
-	add_token( 0x0f2, "start2" );
-	add_token( 0x0f3, "start4" );
-	add_token( 0x0fc, "ferror" );
-	add_token( 0x0fd, "version1" );
-	add_token( 0x0fe, "4-byte-id" );
-	add_token( 0x0ff, "end1" );
-	add_token( 0x101, "dma-alloc" );
-	add_token( 0x102, "my-address" );
-	add_token( 0x103, "my-space" );
-	add_token( 0x104, "memmap" );
-	add_token( 0x105, "free-virtual" );
-	add_token( 0x106, ">physical" );
-	add_token( 0x10f, "my-params" );
-	add_token( 0x110, "property" );
-	add_token( 0x111, "encode-int" );
-	add_token( 0x112, "encode+" );
-	add_token( 0x113, "encode-phys" );
-	add_token( 0x114, "encode-string" );
-	add_token( 0x115, "encode-bytes" );
-	add_token( 0x116, "reg" );
-	add_token( 0x117, "intr" );
-	add_token( 0x118, "driver" );
-	add_token( 0x119, "model" );
-	add_token( 0x11a, "device-type" );
-	add_token( 0x11b, "parse-2int" );
-	add_token( 0x11c, "is-install" );
-	add_token( 0x11d, "is-remove" );
-	add_token( 0x11e, "is-selftest" );
-	add_token( 0x11f, "new-device" );
-	add_token( 0x120, "diagnostic-mode?" );
-	add_token( 0x121, "display-status" );
-	add_token( 0x122, "memory-test-issue" );
-	add_token( 0x123, "group-code" );
-	add_token( 0x124, "mask" );
-	add_token( 0x125, "get-msecs" );
-	add_token( 0x126, "ms" );
-	add_token( 0x127, "finish-device" );
-	add_token( 0x128, "decode-phys" );
-	add_token( 0x12b, "interpose" );
-	add_token( 0x130, "map-low" );
-	add_token( 0x131, "sbus-intr>cpu" );
-	add_token( 0x150, "#lines" );
-	add_token( 0x151, "#columns" );
-	add_token( 0x152, "line#" );
-	add_token( 0x153, "column#" );
-	add_token( 0x154, "inverse?" );
-	add_token( 0x155, "inverse-screen?" );
-	add_token( 0x156, "frame-buffer-busy?" );
-	add_token( 0x157, "draw-character" );
-	add_token( 0x158, "reset-screen" );
-	add_token( 0x159, "toggle-cursor" );
-	add_token( 0x15a, "erase-screen" );
-	add_token( 0x15b, "blink-screen" );
-	add_token( 0x15c, "invert-screen" );
-	add_token( 0x15d, "insert-characters" );
-	add_token( 0x15e, "delete-characters" );
-	add_token( 0x15f, "insert-lines" );
-	add_token( 0x160, "delete-lines" );
-	add_token( 0x161, "draw-logo" );
-	add_token( 0x162, "frame-buffer-adr" );
-	add_token( 0x163, "screen-height" );
-	add_token( 0x164, "screen-width" );
-	add_token( 0x165, "window-top" );
-	add_token( 0x166, "window-left" );
-	add_token( 0x16a, "default-font" );
-	add_token( 0x16b, "set-font" );
-	add_token( 0x16c, "char-height" );
-	add_token( 0x16d, "char-width" );
-	add_token( 0x16e, ">font" );
-	add_token( 0x16f, "fontbytes" );
-	add_token( 0x170, "fb1-draw-character" );
-	add_token( 0x171, "fb1-reset-screen" );
-	add_token( 0x172, "fb1-toggle-cursor" );
-	add_token( 0x173, "fb1-erase-screen" );
-	add_token( 0x174, "fb1-blink-screen" );
-	add_token( 0x175, "fb1-invert-screen" );
-	add_token( 0x176, "fb1-insert-characters" );
-	add_token( 0x177, "fb1-delete-characters" );
-	add_token( 0x178, "fb1-insert-lines" );
-	add_token( 0x179, "fb1-delete-lines" );
-	add_token( 0x17a, "fb1-draw-logo" );
-	add_token( 0x17b, "fb1-install" );
-	add_token( 0x17c, "fb1-slide-up" );
-	add_token( 0x180, "fb8-draw-character" );
-	add_token( 0x181, "fb8-reset-screen" );
-	add_token( 0x182, "fb8-toggle-cursor" );
-	add_token( 0x183, "fb8-erase-screen" );
-	add_token( 0x184, "fb8-blink-screen" );
-	add_token( 0x185, "fb8-invert-screen" );
-	add_token( 0x186, "fb8-insert-characters" );
-	add_token( 0x187, "fb8-delete-characters" );
-	add_token( 0x188, "fb8-insert-lines" );
-	add_token( 0x189, "fb8-delete-lines" );
-	add_token( 0x18a, "fb8-draw-logo" );
-	add_token( 0x18b, "fb8-install" );
-	add_token( 0x1a0, "return-buffer" );
-	add_token( 0x1a1, "xmit-packet" );
-	add_token( 0x1a2, "poll-packet" );
-	add_token( 0x1a4, "mac-address" );
-	add_token( 0x201, "device-name" );
-	add_token( 0x202, "my-args" );
-	add_token( 0x203, "my-self" );
-	add_token( 0x204, "find-package" );
-	add_token( 0x205, "open-package" );
-	add_token( 0x206, "close-package" );
-	add_token( 0x207, "find-method" );
-	add_token( 0x208, "call-package" );
-	add_token( 0x209, "$call-parent" );
-	add_token( 0x20a, "my-package" );
-	add_token( 0x20b, "ihandle>phandle" );
-	add_token( 0x20d, "my-unit" );
-	add_token( 0x20e, "$call-method" );
-	add_token( 0x20f, "$open-package" );
-	add_token( 0x210, "processor-type" );
-	add_token( 0x211, "firmware-version" );
-	add_token( 0x212, "fcode-version" );
-	add_token( 0x213, "alarm" );
-	add_token( 0x214, "(is-user-word)" );
-	add_token( 0x215, "suspend-fcode" );
-	add_token( 0x216, "abort" );
-	add_token( 0x217, "catch" );
-	add_token( 0x218, "throw" );
-	add_token( 0x219, "user-abort" );
-	add_token( 0x21a, "get-my-property" );
-	add_token( 0x21b, "decode-int" );
-	add_token( 0x21c, "decode-string" );
-	add_token( 0x21d, "get-inherited-property" );
-	add_token( 0x21e, "delete-property" );
-	add_token( 0x21f, "get-package-property" );
-	add_token( 0x220, "cpeek" );
-	add_token( 0x221, "wpeek" );
-	add_token( 0x222, "lpeek" );
-	add_token( 0x223, "cpoke" );
-	add_token( 0x224, "wpoke" );
-	add_token( 0x225, "lpoke" );
-	add_token( 0x226, "lwflip" );
-	add_token( 0x227, "lbflip" );
-	add_token( 0x228, "lbflips" );
-	add_token( 0x229, "adr-mask" );
-	add_token( 0x230, "rb@" );
-	add_token( 0x231, "rb!" );
-	add_token( 0x232, "rw@" );
-	add_token( 0x233, "rw!" );
-	add_token( 0x234, "rl@" );
-	add_token( 0x235, "rl!" );
-	add_token( 0x236, "wbflips" );
-	add_token( 0x237, "lwflips" );
-	add_token( 0x238, "probe" );
-	add_token( 0x239, "probe-virtual" );
-	add_token( 0x23b, "child" );
-	add_token( 0x23c, "peer" );
-	add_token( 0x23d, "next-property" );
-	add_token( 0x23e, "byte-load" );
-	add_token( 0x23f, "set-args" );
-	add_token( 0x240, "left-parse-string" );
+    tic_hdr_t *temp_vocab;

+    /*  The  add_to_current()  function will have been called before this
+     *      one when a new colon-definition is created, so  save_current
+     *      will have been set to point to the entry that had been made
+     *      just before the newest one, which we are hiding here.
+     */
+
+    temp_vocab = save_current ;
+    save_current = *current_definitions;
+    *current_definitions = temp_vocab;
+
+}
+
+void reveal_last_colon ( void )
+{
+    /*  We call this function either when the colon-definition is
+     *      completed, or when "recursive"-ness is intentional.
+     */
+    *current_definitions = save_current ;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  create_current_alias
+ *      Synopsis:       Create an alias for OLD_NAME, called NEW_NAME, in
+ *                          the "current" scope of definitions, which may
+ *                          be either the Global ("core") Vocabulary or the
+ *                          current Device-Node Vocabulary.  Indicate if
+ *                          successful (i.e., OLD_NAME was valid).
+ *                      This is actually a little trickier than it may at
+ *                          first appear; read the Rules in the Process
+ *                          Explanation for full details...
+ *
+ *      Inputs:
+ *         Parameters:
+ *             new_name          The name for the new alias to be created
+ *             old_name          The name of the old function to be duplicated
+ *         Global Variables:
+ *             current_definitions        Device-node vocabulary currently
+ *                                            in effect.
+ *             scope_is_global            TRUE if "global" scope is in effect
+ *         Local Static Variables:
+ *             global_voc_dict_ptr        "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:                TRUE if OLD_NAME was found.
+ *         Global Variables:
+ *             *current_definitions      Updated with the new entry 
+ *         Memory Allocated
+ *             By support routine.
+ *         When Freed?
+ *             When RESET-SYMBOLS is issued (if "global" scope is in effect)
+ *                 or when the device-node is "finish"ed.
+ *         Printout:
+ *             Advisory message if Rule 3 (see below) is invoked.
+ *
+ *      Process Explanation:
+ *          Both the "old" and "new" names are presumed to already point to
+ *              stable, freshly allocated memory-spaces.
+ *          Rules:
+ *          (1)
+ *          If "global" scope is in effect, and the "old" name is found in
+ *              the Global Vocabulary, then the "new" name will be created
+ *              in the Global Vocabulary.
+ *          (2)
+ *          Similarly, if "device" scope is in effect, and the "old" name is
+ *              found in the current device-node's vocabulary, the "new" name
+ *              will be created in the current device-node's vocabulary.
+ *          (3)
+ *          BUT!:  If "device" scope is in effect, and the "old" name is found
+ *              in the Global Vocabulary, then the "new" name will be created
+ *              in the current device-node's vocabulary.  It will only be
+ *              recognized in the scope of that device-node, and will be
+ *              removed from memory when the device-node is "finish"ed.
+ *          And, yes, it *is* supposed to work that way...   ;-)
+ *
+ *          Again, because  current_definitions  points to the Global Vocab
+ *              pointer during "global" scope, the first two rules of this
+ *              routine are extremely straightforward; it's Rule 3 that you
+ *              have to watch out for...  ;-)
+ *
+ *          And one other thing:
+ *              We will always make the alias's  pfld_size  zero.  See the
+ *              prolog for  create_tic_alias()  in  ticvocab.c  for details...
+ *
+ *      Extraneous Remarks:
+ *          I will stretch the rules of well-structured code here, too.
+ *
+ **************************************************************************** */
+
+bool create_current_alias( char *new_name, char *old_name )
+{
+    bool retval = FALSE;
+
+    if ( create_tic_alias( new_name, old_name, current_definitions) )
+    {
+	 return ( TRUE );
+    }
+    
+    if ( INVERSE(scope_is_global) )
+    {
+	tic_hdr_t *found = lookup_core_word( old_name );
+	if ( found != NULL )
+	{
+	    add_tic_entry( new_name, found->funct,
+			       found->pfield.deflt_elem,
+				   found->fword_defr,
+				       0, found->ign_func,
+					   current_definitions );
+	    retval = TRUE;
+	    {
+		tokenization_error( INFO,
+		   "%s is a Global definition, but its alias, %s, "
+		       "will only be defined %s",
+			   strupr( old_name), new_name,
+			       in_what_node( current_device_node) );
+		show_node_start();
+	    }
+	}
+    }
+
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Support functions specific to the lists will be defined
+ *          after the lists are created.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *     Create the initial list (or "Table") of FCode-Tokens.
+ *
+ *     Most Standard FCode tokens are not specified as to their definition
+ *         type, but a few have a definer specified as either a VALUE, a
+ *         VARIABLE or a DEFER; we will enter them with the appropriate macro.
+ *
+ **************************************************************************** */
+
+static tic_hdr_t tokens_table[] =
+{
+	BUILTIN_FCODE( 0x000, "end0" ) ,
+	BUILTIN_FCODE( 0x010, "b(lit)" ) ,
+	BUILTIN_FCODE( 0x011, "b(')" ) ,
+	BUILTIN_FCODE( 0x012, "b(")" ) ,
+	BUILTIN_FCODE( 0x013, "bbranch" ) ,
+	BUILTIN_FCODE( 0x014, "b?branch" ) ,
+	BUILTIN_FCODE( 0x015, "b(loop)" ) ,
+	BUILTIN_FCODE( 0x016, "b(+loop)" ) ,
+	BUILTIN_FCODE( 0x017, "b(do)" ) ,
+	BUILTIN_FCODE( 0x018, "b(?do)" ) ,
+	BUILTIN_FCODE( 0x019, "i" ) ,
+	BUILTIN_FCODE( 0x01a, "j" ) ,
+	BUILTIN_FCODE( 0x01b, "b(leave)" ) ,
+	BUILTIN_FCODE( 0x01c, "b(of)" ) ,
+	BUILTIN_FCODE( 0x01d, "execute" ) ,
+	BUILTIN_FCODE( 0x01e, "+" ) ,
+	BUILTIN_FCODE( 0x01f, "-" ) ,
+	BUILTIN_FCODE( 0x020, "*" ) ,
+	BUILTIN_FCODE( 0x021, "/" ) ,
+	BUILTIN_FCODE( 0x022, "mod" ) ,
+	BUILTIN_FCODE( 0x023, "and" ) ,
+	BUILTIN_FCODE( 0x024, "or" ) ,
+	BUILTIN_FCODE( 0x025, "xor" ) ,
+	BUILTIN_FCODE( 0x026, "invert" ) ,
+	BUILTIN_FCODE( 0x026, "not" ) ,           /*  Synonym for "invert" */
+	BUILTIN_FCODE( 0x027, "lshift" ) ,
+	BUILTIN_FCODE( 0x027, "<<" ) ,            /*  Synonym for "lshift" */
+	BUILTIN_FCODE( 0x028, "rshift" ) ,
+	BUILTIN_FCODE( 0x028, ">>" ) ,            /*  Synonym for "rshift" */
+	BUILTIN_FCODE( 0x029, ">>a" ) ,
+	BUILTIN_FCODE( 0x02a, "/mod" ) ,
+	BUILTIN_FCODE( 0x02b, "u/mod" ) ,
+	BUILTIN_FCODE( 0x02c, "negate" ) ,
+	BUILTIN_FCODE( 0x02d, "abs" ) ,
+	BUILTIN_FCODE( 0x02e, "min" ) ,
+	BUILTIN_FCODE( 0x02f, "max" ) ,
+	BUILTIN_FCODE( 0x030, ">r" ) ,
+	BUILTIN_FCODE( 0x031, "r>" ) ,
+	BUILTIN_FCODE( 0x032, "r@" ) ,
+	BUILTIN_FCODE( 0x033, "exit" ) ,
+	BUILTIN_FCODE( 0x034, "0=" ) ,
+	BUILTIN_FCODE( 0x035, "0<>" ) ,
+	BUILTIN_FCODE( 0x036, "0<" ) ,
+	BUILTIN_FCODE( 0x037, "0<=" ) ,
+	BUILTIN_FCODE( 0x038, "0>" ) ,
+	BUILTIN_FCODE( 0x039, "0>=" ) ,
+	BUILTIN_FCODE( 0x03a, "<" ) ,
+	BUILTIN_FCODE( 0x03b, ">" ) ,
+	BUILTIN_FCODE( 0x03c, "=" ) ,
+	BUILTIN_FCODE( 0x03d, "<>" ) ,
+	BUILTIN_FCODE( 0x03e, "u>" ) ,
+	BUILTIN_FCODE( 0x03f, "u<=" ) ,
+	BUILTIN_FCODE( 0x040, "u<" ) ,
+	BUILTIN_FCODE( 0x041, "u>=" ) ,
+	BUILTIN_FCODE( 0x042, ">=" ) ,
+	BUILTIN_FCODE( 0x043, "<=" ) ,
+	BUILTIN_FCODE( 0x044, "between" ) ,
+	BUILTIN_FCODE( 0x045, "within" ) ,
+	BUILTIN_FCODE( 0x046, "drop" ) ,
+	BUILTIN_FCODE( 0x047, "dup" ) ,
+	BUILTIN_FCODE( 0x048, "over" ) ,
+	BUILTIN_FCODE( 0x049, "swap" ) ,
+	BUILTIN_FCODE( 0x04A, "rot" ) ,
+	BUILTIN_FCODE( 0x04b, "-rot" ) ,
+	BUILTIN_FCODE( 0x04c, "tuck" ) ,
+	BUILTIN_FCODE( 0x04d, "nip" ) ,
+	BUILTIN_FCODE( 0x04e, "pick" ) ,
+	BUILTIN_FCODE( 0x04f, "roll" ) ,
+	BUILTIN_FCODE( 0x050, "?dup" ) ,
+	BUILTIN_FCODE( 0x051, "depth" ) ,
+	BUILTIN_FCODE( 0x052, "2drop" ) ,
+	BUILTIN_FCODE( 0x053, "2dup" ) ,
+	BUILTIN_FCODE( 0x054, "2over" ) ,
+	BUILTIN_FCODE( 0x055, "2swap" ) ,
+	BUILTIN_FCODE( 0x056, "2rot" ) ,
+	BUILTIN_FCODE( 0x057, "2/" ) ,
+	BUILTIN_FCODE( 0x058, "u2/" ) ,
+	BUILTIN_FCODE( 0x059, "2*" ) ,
+	BUILTIN_FCODE( 0x05a, "/c" ) ,
+	BUILTIN_FCODE( 0x05b, "/w" ) ,
+	BUILTIN_FCODE( 0x05c, "/l" ) ,
+	BUILTIN_FCODE( 0x05d, "/n" ) ,
+	BUILTIN_FCODE( 0x05e, "ca+" ) ,
+	BUILTIN_FCODE( 0x05f, "wa+" ) ,
+	BUILTIN_FCODE( 0x060, "la+" ) ,
+	BUILTIN_FCODE( 0x061, "na+" ) ,
+	BUILTIN_FCODE( 0x062, "char+" ) ,
+	BUILTIN_FCODE( 0x062, "ca1+" ) ,          /*  Synonym for char+" */
+	BUILTIN_FCODE( 0x063, "wa1+" ) ,
+	BUILTIN_FCODE( 0x064, "la1+" ) ,
+	BUILTIN_FCODE( 0x065, "cell+" ) ,
+	BUILTIN_FCODE( 0x065, "na1+" ) ,          /*  Synonym for "cell+" */
+	BUILTIN_FCODE( 0x066, "chars" ) ,
+	BUILTIN_FCODE( 0x066, "/c*" ) ,           /*  Synonym for "chars" */
+	BUILTIN_FCODE( 0x067, "/w*" ) ,
+	BUILTIN_FCODE( 0x068, "/l*" ) ,
+	BUILTIN_FCODE( 0x069, "cells" ) ,
+	BUILTIN_FCODE( 0x069, "/n*" ) ,           /*  Synonym for "cells" */
+	BUILTIN_FCODE( 0x06a, "on" ) ,
+	BUILTIN_FCODE( 0x06b, "off" ) ,
+	BUILTIN_FCODE( 0x06c, "+!" ) ,
+	BUILTIN_FCODE( 0x06d, "@" ) ,
+	BUILTIN_FCODE( 0x06e, "l@" ) ,
+	BUILTIN_FCODE( 0x06f, "w@" ) ,
+	BUILTIN_FCODE( 0x070, "<w@" ) ,
+	BUILTIN_FCODE( 0x071, "c@" ) ,
+	BUILTIN_FCODE( 0x072, "!" ) ,
+	BUILTIN_FCODE( 0x073, "l!" ) ,
+	BUILTIN_FCODE( 0x074, "w!" ) ,
+	BUILTIN_FCODE( 0x075, "c!" ) ,
+	BUILTIN_FCODE( 0x076, "2@" ) ,
+	BUILTIN_FCODE( 0x077, "2!" ) ,
+	BUILTIN_FCODE( 0x078, "move" ) ,
+	BUILTIN_FCODE( 0x079, "fill" ) ,
+	BUILTIN_FCODE( 0x07a, "comp" ) ,
+	BUILTIN_FCODE( 0x07b, "noop" ) ,
+	BUILTIN_FCODE( 0x07c, "lwsplit" ) ,
+	BUILTIN_FCODE( 0x07d, "wljoin" ) ,
+	BUILTIN_FCODE( 0x07e, "lbsplit" ) ,
+	BUILTIN_FCODE( 0x07f, "bljoin" ) ,
+	BUILTIN_FCODE( 0x080, "wbflip" ) ,
+	BUILTIN_FCODE( 0x080, "flip" ) ,   /*  Synonym for "wbflip"  */
+	BUILTIN_FCODE( 0x081, "upc" ) ,
+	BUILTIN_FCODE( 0x082, "lcc" ) ,
+	BUILTIN_FCODE( 0x083, "pack" ) ,
+	BUILTIN_FCODE( 0x084, "count" ) ,
+	BUILTIN_FCODE( 0x085, "body>" ) ,
+	BUILTIN_FCODE( 0x086, ">body" ) ,
+	BUILTIN_FCODE( 0x087, "fcode-revision" ) ,
+	BUILTIN_FCODE( 0x087, "version" ) , /*  Synonym for "fcode-revision" */
+	BI_FCODE_VRBLE( 0x088, "span" ) ,
+	BUILTIN_FCODE( 0x089, "unloop" ) ,
+	BUILTIN_FCODE( 0x08a, "expect" ) ,
+	BUILTIN_FCODE( 0x08b, "alloc-mem" ) ,
+	BUILTIN_FCODE( 0x08c, "free-mem" ) ,
+	BUILTIN_FCODE( 0x08d, "key?" ) ,
+	BUILTIN_FCODE( 0x08e, "key" ) ,
+	BUILTIN_FCODE( 0x08f, "emit" ) ,
+	BUILTIN_FCODE( 0x090, "type" ) ,
+	BUILTIN_FCODE( 0x091, "(cr" ) ,
+	BUILTIN_FCODE( 0x092, "cr" ) ,
+	BI_FCODE_VRBLE( 0x093, "#out" ) ,
+	BI_FCODE_VRBLE( 0x094, "#line" ) ,
+	BUILTIN_FCODE( 0x095, "hold" ) ,
+	BUILTIN_FCODE( 0x096, "<#" ) ,
+	BUILTIN_FCODE( 0x097, "u#>" ) ,
+	BUILTIN_FCODE( 0x098, "sign" ) ,
+	BUILTIN_FCODE( 0x099, "u#" ) ,
+	BUILTIN_FCODE( 0x09a, "u#s" ) ,
+	BUILTIN_FCODE( 0x09b, "u." ) ,
+	BUILTIN_FCODE( 0x09c, "u.r" ) ,
+	BUILTIN_FCODE( 0x09d, "." ) ,
+	BUILTIN_FCODE( 0x09e, ".r" ) ,
+	BUILTIN_FCODE( 0x09f, ".s" ) ,
+	BI_FCODE_VRBLE( 0x0a0, "base" ) ,
+	OBSOLETE_FCODE( 0x0a1, "convert" ) ,
+	BUILTIN_FCODE( 0x0a2, "$number" ) ,
+	BUILTIN_FCODE( 0x0a3, "digit" ) ,
+	BI_FCODE_CONST( 0x0a4, "-1" ) ,
+	BI_FCODE_CONST( 0x0a4, "true" ) ,          /*  Synonym for "-1" */
+	BI_FCODE_CONST( 0x0a5, "0" ) ,
+	BI_FCODE_CONST( 0x0a5, "false" ) ,         /*  Synonym for "0" */
+	BI_FCODE_CONST( 0x0a5, "struct" ) ,        /*  Synonym for "0" */
+	BI_FCODE_CONST( 0x0a6, "1" ) ,
+	BI_FCODE_CONST( 0x0a7, "2" ) ,
+	BI_FCODE_CONST( 0x0a8, "3" ) ,
+	BI_FCODE_CONST( 0x0a9, "bl" ) ,
+	BI_FCODE_CONST( 0x0aa, "bs" ) ,
+	BI_FCODE_CONST( 0x0ab, "bell" ) ,
+	BUILTIN_FCODE( 0x0ac, "bounds" ) ,
+	BUILTIN_FCODE( 0x0ad, "here" ) ,
+	BUILTIN_FCODE( 0x0ae, "aligned" ) ,
+	BUILTIN_FCODE( 0x0af, "wbsplit" ) ,
+	BUILTIN_FCODE( 0x0b0, "bwjoin" ) ,
+	BUILTIN_FCODE( 0x0b1, "b(<mark)" ) ,
+	BUILTIN_FCODE( 0x0b2, "b(>resolve)" ) ,
+	OBSOLETE_FCODE( 0x0b3, "set-token-table" ) ,
+	OBSOLETE_FCODE( 0x0b4, "set-table" ) ,
+	BUILTIN_FCODE( 0x0b5, "new-token" ) ,
+	BUILTIN_FCODE( 0x0b6, "named-token" ) ,
+	BUILTIN_FCODE( 0x0b7, "b(:)" ) ,
+	BUILTIN_FCODE( 0x0b8, "b(value)" ) ,
+	BUILTIN_FCODE( 0x0b9, "b(variable)" ) ,
+	BUILTIN_FCODE( 0x0ba, "b(constant)" ) ,
+	BUILTIN_FCODE( 0x0bb, "b(create)" ) ,
+	BUILTIN_FCODE( 0x0bc, "b(defer)" ) ,
+	BUILTIN_FCODE( 0x0bd, "b(buffer:)" ) ,
+	BUILTIN_FCODE( 0x0be, "b(field)" ) ,
+	OBSOLETE_FCODE( 0x0bf, "b(code)" ) ,
+	BUILTIN_FCODE( 0x0c0, "instance" ) ,
+	BUILTIN_FCODE( 0x0c2, "b(;)" ) ,
+	BUILTIN_FCODE( 0x0c3, "b(to)" ) ,
+	BUILTIN_FCODE( 0x0c4, "b(case)" ) ,
+	BUILTIN_FCODE( 0x0c5, "b(endcase)" ) ,
+	BUILTIN_FCODE( 0x0c6, "b(endof)" ) ,
+	BUILTIN_FCODE( 0x0c7, "#" ) ,
+	BUILTIN_FCODE( 0x0c8, "#s" ) ,
+	BUILTIN_FCODE( 0x0c9, "#>" ) ,
+	BUILTIN_FCODE( 0x0ca, "external-token" ) ,
+	BUILTIN_FCODE( 0x0cb, "$find" ) ,
+	BUILTIN_FCODE( 0x0cc, "offset16" ) ,
+	BUILTIN_FCODE( 0x0cd, "evaluate" ) ,
+	BUILTIN_FCODE( 0x0cd, "eval" ) ,   /*  Synonym for "evaluate"  */
+	BUILTIN_FCODE( 0x0d0, "c," ) ,
+	BUILTIN_FCODE( 0x0d1, "w," ) ,
+	BUILTIN_FCODE( 0x0d2, "l," ) ,
+	BUILTIN_FCODE( 0x0d3, "," ) ,
+	BUILTIN_FCODE( 0x0d4, "um*" ) ,
+	BUILTIN_FCODE( 0x0d4, "u*x" ) ,        /*  Synonym for "um*" */
+	BUILTIN_FCODE( 0x0d5, "um/mod" ) ,
+	BUILTIN_FCODE( 0x0d5, "xu/mod" ) ,   /*  Synonym for "um/mod"  */
+	BUILTIN_FCODE( 0x0d8, "d+" ) ,
+	BUILTIN_FCODE( 0x0d8, "x+" ) ,   /*  Synonym for "d+"  */
+	BUILTIN_FCODE( 0x0d9, "d-" ) ,
+	BUILTIN_FCODE( 0x0d9, "x-" ) ,   /*  Synonym for "d-"  */
+	BUILTIN_FCODE( 0x0da, "get-token" ) ,
+	BUILTIN_FCODE( 0x0db, "set-token" ) ,
+	BI_FCODE_VRBLE( 0x0dc, "state" ) ,
+	BUILTIN_FCODE( 0x0dd, "compile" ) ,
+	BUILTIN_FCODE( 0x0de, "behavior" ) ,
+	BUILTIN_FCODE( 0x0f0, "start0" ) ,
+	BUILTIN_FCODE( 0x0f1, "start1" ) ,
+	BUILTIN_FCODE( 0x0f2, "start2" ) ,
+	BUILTIN_FCODE( 0x0f3, "start4" ) ,
+	BUILTIN_FCODE( 0x0fc, "ferror" ) ,
+	BUILTIN_FCODE( 0x0fd, "version1" ) ,
+	OBSOLETE_FCODE( 0x0fe, "4-byte-id" ) ,
+	BUILTIN_FCODE( 0x0ff, "end1" ) ,
+	OBSOLETE_FCODE( 0x101, "dma-alloc" ) ,
+	BUILTIN_FCODE( 0x102, "my-address" ) ,
+	BUILTIN_FCODE( 0x103, "my-space" ) ,
+	OBSOLETE_FCODE( 0x104, "memmap" ) ,
+	BUILTIN_FCODE( 0x105, "free-virtual" ) ,
+	OBSOLETE_FCODE( 0x106, ">physical" ) ,
+	OBSOLETE_FCODE( 0x10f, "my-params" ) ,
+	BUILTIN_FCODE( 0x110, "property" ) ,
+	BUILTIN_FCODE( 0x110, "attribute" ) ,    /*  Synonym for "property"  */
+	BUILTIN_FCODE( 0x111, "encode-int" ) ,
+	BUILTIN_FCODE( 0x111, "xdrint" ) ,     /*  Synonym for "encode-int"  */
+	BUILTIN_FCODE( 0x112, "encode+" ) ,
+	BUILTIN_FCODE( 0x112, "xdr+" ) ,          /*  Synonym for "encode+"  */
+	BUILTIN_FCODE( 0x113, "encode-phys" ) ,
+	BUILTIN_FCODE( 0x113, "xdrphys" ) ,   /*  Synonym for "encode-phys"  */
+	BUILTIN_FCODE( 0x114, "encode-string" ) ,
+	BUILTIN_FCODE( 0x114, "xdrstring" ) , /* Synonym for "encode-string" */
+	BUILTIN_FCODE( 0x115, "encode-bytes" ) ,
+	BUILTIN_FCODE( 0x115, "xdrbytes" ) ,  /*  Synonym for "encode-bytes" */
+	BUILTIN_FCODE( 0x116, "reg" ) ,
+	OBSOLETE_FCODE( 0x117, "intr" ) ,
+	OBSOLETE_FCODE( 0x118, "driver" ) ,
+	BUILTIN_FCODE( 0x119, "model" ) ,
+	BUILTIN_FCODE( 0x11a, "device-type" ) ,
+	BUILTIN_FCODE( 0x11b, "parse-2int" ) ,
+	BUILTIN_FCODE( 0x11b, "decode-2int" ) , /*  Synonym for "parse-2int" */
+	BUILTIN_FCODE( 0x11c, "is-install" ) ,
+	BUILTIN_FCODE( 0x11d, "is-remove" ) ,
+	BUILTIN_FCODE( 0x11e, "is-selftest" ) ,
+	BUILTIN_FCODE( 0x11f, "new-device" ) ,
+	BUILTIN_FCODE( 0x120, "diagnostic-mode?" ) ,
+	BUILTIN_FCODE( 0x121, "display-status" ) ,
+	BUILTIN_FCODE( 0x122, "memory-test-issue" ) ,
+	OBSOLETE_FCODE( 0x123, "group-code" ) ,
+	BI_FCODE_VRBLE( 0x124, "mask" ) ,
+	BUILTIN_FCODE( 0x125, "get-msecs" ) ,
+	BUILTIN_FCODE( 0x126, "ms" ) ,
+	BUILTIN_FCODE( 0x127, "finish-device" ) ,
+	BUILTIN_FCODE( 0x128, "decode-phys" ) ,
+	BUILTIN_FCODE( 0x12b, "interpose" ) ,
+	BUILTIN_FCODE( 0x130, "map-low" ) ,
+	BUILTIN_FCODE( 0x130, "map-sbus" ) ,   /*  Synonym for "map-low"  */
+	BUILTIN_FCODE( 0x131, "sbus-intr>cpu" ) ,
+	BI_FCODE_VALUE( 0x150, "#lines" ) ,
+	BI_FCODE_VALUE( 0x151, "#columns" ) ,
+	BI_FCODE_VALUE( 0x152, "line#" ) ,
+	BI_FCODE_VALUE( 0x153, "column#" ) ,
+	BI_FCODE_VALUE( 0x154, "inverse?" ) ,
+	BI_FCODE_VALUE( 0x155, "inverse-screen?" ) ,
+	OBSOLETE_VALUE( 0x156, "frame-buffer-busy?" ) ,
+	BI_FCODE_DEFER( 0x157, "draw-character" ) ,
+	BI_FCODE_DEFER( 0x158, "reset-screen" ) ,
+	BI_FCODE_DEFER( 0x159, "toggle-cursor" ) ,
+	BI_FCODE_DEFER( 0x15a, "erase-screen" ) ,
+	BI_FCODE_DEFER( 0x15b, "blink-screen" ) ,
+	BI_FCODE_DEFER( 0x15c, "invert-screen" ) ,
+	BI_FCODE_DEFER( 0x15d, "insert-characters" ) ,
+	BI_FCODE_DEFER( 0x15e, "delete-characters" ) ,
+	BI_FCODE_DEFER( 0x15f, "insert-lines" ) ,
+	BI_FCODE_DEFER( 0x160, "delete-lines" ) ,
+	BI_FCODE_DEFER( 0x161, "draw-logo" ) ,
+	BI_FCODE_VALUE( 0x162, "frame-buffer-adr" ) ,
+	BI_FCODE_VALUE( 0x163, "screen-height" ) ,
+	BI_FCODE_VALUE( 0x164, "screen-width" ) ,
+	BI_FCODE_VALUE( 0x165, "window-top" ) ,
+	BI_FCODE_VALUE( 0x166, "window-left" ) ,
+	BUILTIN_FCODE( 0x16a, "default-font" ) ,
+	BUILTIN_FCODE( 0x16b, "set-font" ) ,
+	BI_FCODE_VALUE( 0x16c, "char-height" ) ,
+	BI_FCODE_VALUE( 0x16d, "char-width" ) ,
+	BUILTIN_FCODE( 0x16e, ">font" ) ,
+	BI_FCODE_VALUE( 0x16f, "fontbytes" ) ,
+	OBSOLETE_FCODE( 0x170, "fb1-draw-character" ) ,
+	OBSOLETE_FCODE( 0x171, "fb1-reset-screen" ) ,
+	OBSOLETE_FCODE( 0x172, "fb1-toggle-cursor" ) ,
+	OBSOLETE_FCODE( 0x173, "fb1-erase-screen" ) ,
+	OBSOLETE_FCODE( 0x174, "fb1-blink-screen" ) ,
+	OBSOLETE_FCODE( 0x175, "fb1-invert-screen" ) ,
+	OBSOLETE_FCODE( 0x176, "fb1-insert-characters" ) ,
+	OBSOLETE_FCODE( 0x177, "fb1-delete-characters" ) ,
+	OBSOLETE_FCODE( 0x178, "fb1-insert-lines" ) ,
+	OBSOLETE_FCODE( 0x179, "fb1-delete-lines" ) ,
+	OBSOLETE_FCODE( 0x17a, "fb1-draw-logo" ) ,
+	OBSOLETE_FCODE( 0x17b, "fb1-install" ) ,
+	OBSOLETE_FCODE( 0x17c, "fb1-slide-up" ) ,
+	BUILTIN_FCODE( 0x180, "fb8-draw-character" ) ,
+	BUILTIN_FCODE( 0x181, "fb8-reset-screen" ) ,
+	BUILTIN_FCODE( 0x182, "fb8-toggle-cursor" ) ,
+	BUILTIN_FCODE( 0x183, "fb8-erase-screen" ) ,
+	BUILTIN_FCODE( 0x184, "fb8-blink-screen" ) ,
+	BUILTIN_FCODE( 0x185, "fb8-invert-screen" ) ,
+	BUILTIN_FCODE( 0x186, "fb8-insert-characters" ) ,
+	BUILTIN_FCODE( 0x187, "fb8-delete-characters" ) ,
+	BUILTIN_FCODE( 0x188, "fb8-insert-lines" ) ,
+	BUILTIN_FCODE( 0x189, "fb8-delete-lines" ) ,
+	BUILTIN_FCODE( 0x18a, "fb8-draw-logo" ) ,
+	BUILTIN_FCODE( 0x18b, "fb8-install" ) ,
+	OBSOLETE_FCODE( 0x1a0, "return-buffer" ) ,
+	OBSOLETE_FCODE( 0x1a1, "xmit-packet" ) ,
+	OBSOLETE_FCODE( 0x1a2, "poll-packet" ) ,
+	BUILTIN_FCODE( 0x1a4, "mac-address" ) ,
+	BUILTIN_FCODE( 0x201, "device-name" ) ,
+	BUILTIN_FCODE( 0x201, "name" ) ,   /*  Synonym for "device-name"  */
+	BUILTIN_FCODE( 0x202, "my-args" ) ,
+	BI_FCODE_VALUE( 0x203, "my-self" ) ,
+	BUILTIN_FCODE( 0x204, "find-package" ) ,
+	BUILTIN_FCODE( 0x205, "open-package" ) ,
+	BUILTIN_FCODE( 0x206, "close-package" ) ,
+	BUILTIN_FCODE( 0x207, "find-method" ) ,
+	BUILTIN_FCODE( 0x208, "call-package" ) ,
+	BUILTIN_FCODE( 0x209, "$call-parent" ) ,
+	BUILTIN_FCODE( 0x20a, "my-parent" ) ,
+	BUILTIN_FCODE( 0x20b, "ihandle>phandle" ) ,
+	BUILTIN_FCODE( 0x20d, "my-unit" ) ,
+	BUILTIN_FCODE( 0x20e, "$call-method" ) ,
+	BUILTIN_FCODE( 0x20f, "$open-package" ) ,
+	OBSOLETE_FCODE( 0x210, "processor-type" ) ,
+	OBSOLETE_FCODE( 0x211, "firmware-version" ) ,
+	OBSOLETE_FCODE( 0x212, "fcode-version" ) ,
+	BUILTIN_FCODE( 0x213, "alarm" ) ,
+	BUILTIN_FCODE( 0x214, "(is-user-word)" ) ,
+	BUILTIN_FCODE( 0x215, "suspend-fcode" ) ,
+	BUILTIN_FCODE( 0x216, "abort" ) ,
+	BUILTIN_FCODE( 0x217, "catch" ) ,
+	BUILTIN_FCODE( 0x218, "throw" ) ,
+	BUILTIN_FCODE( 0x219, "user-abort" ) ,
+	BUILTIN_FCODE( 0x21a, "get-my-property" ) ,
+	BUILTIN_FCODE( 0x21a, "get-my-attribute" ) ,   /*  Synonym for "get-my-property"  */
+	BUILTIN_FCODE( 0x21b, "decode-int" ) ,
+	BUILTIN_FCODE( 0x21b, "xdrtoint" ) ,   /*  Synonym for "decode-int"  */
+	BUILTIN_FCODE( 0x21c, "decode-string" ) ,
+	BUILTIN_FCODE( 0x21c, "xdrtostring" ), /* Synonym for "decode-string" */
+	BUILTIN_FCODE( 0x21d, "get-inherited-property" ) ,
+	BUILTIN_FCODE( 0x21d, "get-inherited-attribute" ) ,   /*  Synonym for "get-inherited-property"  */
+	BUILTIN_FCODE( 0x21e, "delete-property" ) ,
+	BUILTIN_FCODE( 0x21e, "delete-attribute" ) ,   /*  Synonym for "delete-property"  */
+	BUILTIN_FCODE( 0x21f, "get-package-property" ) ,
+	BUILTIN_FCODE( 0x21f, "get-package-attribute" ) ,   /*  Synonym for "get-package-property"  */
+	BUILTIN_FCODE( 0x220, "cpeek" ) ,
+	BUILTIN_FCODE( 0x221, "wpeek" ) ,
+	BUILTIN_FCODE( 0x222, "lpeek" ) ,
+	BUILTIN_FCODE( 0x223, "cpoke" ) ,
+	BUILTIN_FCODE( 0x224, "wpoke" ) ,
+	BUILTIN_FCODE( 0x225, "lpoke" ) ,
+	BUILTIN_FCODE( 0x226, "lwflip" ) ,
+	BUILTIN_FCODE( 0x227, "lbflip" ) ,
+	BUILTIN_FCODE( 0x228, "lbflips" ) ,
+	OBSOLETE_FCODE( 0x229, "adr-mask" ) ,
+	BUILTIN_FCODE( 0x230, "rb@" ) ,
+	BUILTIN_FCODE( 0x231, "rb!" ) ,
+	BUILTIN_FCODE( 0x232, "rw@" ) ,
+	BUILTIN_FCODE( 0x233, "rw!" ) ,
+	BUILTIN_FCODE( 0x234, "rl@" ) ,
+	BUILTIN_FCODE( 0x235, "rl!" ) ,
+	BUILTIN_FCODE( 0x236, "wbflips" ) ,
+	BUILTIN_FCODE( 0x236, "wflips" ) ,   /*  Synonym for "wbflips"  */
+	BUILTIN_FCODE( 0x237, "lwflips" ) ,
+	BUILTIN_FCODE( 0x237, "lflips" ) ,   /*  Synonym for "lwflips"  */
+	OBSOLETE_FCODE( 0x238, "probe" ) ,
+	OBSOLETE_FCODE( 0x239, "probe-virtual" ) ,
+	BUILTIN_FCODE( 0x23b, "child" ) ,
+	BUILTIN_FCODE( 0x23c, "peer" ) ,
+	BUILTIN_FCODE( 0x23d, "next-property" ) ,
+	BUILTIN_FCODE( 0x23e, "byte-load" ) ,
+	BUILTIN_FCODE( 0x23f, "set-args" ) ,
+	BUILTIN_FCODE( 0x240, "left-parse-string" ) ,
+
    /* FCodes from 64bit extension addendum */
-	add_token( 0x22e, "rx@" );
-	add_token( 0x22f, "rx!" );
-	add_token( 0x241, "bxjoin" );
-	add_token( 0x242, "<l@" );
-	add_token( 0x243, "lxjoin" );
-	add_token( 0x244, "wxjoin" );
-	add_token( 0x245, "x," );
-	add_token( 0x246, "x@" );
-	add_token( 0x247, "x!" );
-	add_token( 0x248, "/x" );
-	add_token( 0x249, "/x*" );
-	add_token( 0x24a, "xa+" );
-	add_token( 0x24b, "xa1+" );
-	add_token( 0x24c, "xbflip" );
-	add_token( 0x24d, "xbflips" );
-	add_token( 0x24e, "xbsplit" );
-	add_token( 0x24f, "xlflip" );
-	add_token( 0x250, "xlflips" );
-	add_token( 0x251, "xlsplit" );
-	add_token( 0x252, "xwflip" );
-	add_token( 0x253, "xwflips" );
-	add_token( 0x254, "xwsplit" );
+	BUILTIN_FCODE( 0x22e, "rx@" ) ,
+	BUILTIN_FCODE( 0x22f, "rx!" ) ,
+	BUILTIN_FCODE( 0x241, "bxjoin" ) ,
+	BUILTIN_FCODE( 0x242, "<l@" ) ,
+	BUILTIN_FCODE( 0x243, "lxjoin" ) ,
+	BUILTIN_FCODE( 0x244, "wxjoin" ) ,
+	BUILTIN_FCODE( 0x245, "x," ) ,
+	BUILTIN_FCODE( 0x246, "x@" ) ,
+	BUILTIN_FCODE( 0x247, "x!" ) ,
+	BUILTIN_FCODE( 0x248, "/x" ) ,
+	BUILTIN_FCODE( 0x249, "/x*" ) ,
+	BUILTIN_FCODE( 0x24a, "xa+" ) ,
+	BUILTIN_FCODE( 0x24b, "xa1+" ) ,
+	BUILTIN_FCODE( 0x24c, "xbflip" ) ,
+	BUILTIN_FCODE( 0x24d, "xbflips" ) ,
+	BUILTIN_FCODE( 0x24e, "xbsplit" ) ,
+	BUILTIN_FCODE( 0x24f, "xlflip" ) ,
+	BUILTIN_FCODE( 0x250, "xlflips" ) ,
+	BUILTIN_FCODE( 0x251, "xlsplit" ) ,
+	BUILTIN_FCODE( 0x252, "xwflip" ) ,
+	BUILTIN_FCODE( 0x253, "xwflips" ) ,
+	BUILTIN_FCODE( 0x254, "xwsplit" )
+};

-	add_special(COLON, 	":");
-	add_special(SEMICOLON, 	";");
-	add_special(TOKENIZE, 	"'");
-	add_special(AGAIN, 	"again");
-	add_special(ALIAS, 	"alias");
-	add_special(GETTOKEN, 	"[']");
-	add_special(ASCII, 	"ascii");
-	add_special(BEGIN, 	"begin");
-	add_special(BUFFER, 	"buffer:");
-	add_special(CASE, 	"case");
-	add_special(CONST, 	"constant");
-	add_special(CONTROL, 	"control");
-	add_special(CREATE, 	"create");
-	add_special(DECIMAL, 	"decimal");
-	add_special(DEFER, 	"defer");
-	add_special(CDO, 	"?do");
-	add_special(DO, 	"do");
-	add_special(ELSE, 	"else");
-	add_special(ENDCASE, 	"endcase");
-	add_special(ENDOF, 	"endof");
-	add_special(EXTERNAL, 	"external");
-	add_special(FIELD, 	"field");
-	add_special(HEADERLESS, "headerless");
-	add_special(HEADERS, 	"headers");
-	add_special(HEX, 	"hex");
-	add_special(IF, 	"if");
-	add_special(CLEAVE, 	"?leave");
-	add_special(LEAVE, 	"leave");
-	add_special(CLOOP, 	"+loop");
-	add_special(LOOP, 	"loop");
-	add_special(OCTAL,	"octal");
-	add_special(OF,		"of");
-	add_special(REPEAT,	"repeat");
-	add_special(THEN,	"then");
-	add_special(TO,		"to");
-	add_special(UNTIL,	"until");
-	add_special(VALUE,	"value");
-	add_special(VARIABLE,	"variable");
-	add_special(WHILE,	"while");
-	add_special(OFFSET16,	"offset16");
-	add_special(BEGINTOK,	"tokenizer[");
-	add_special(EMITBYTE,	"emit-byte");
-	add_special(ENDTOK,	"]tokenizer");
-	add_special(FLOAD,	"fload");
-	add_special(STRING,	""");
-	add_special(PSTRING,	"."");
-	add_special(PBSTRING,	".(");
-	add_special(SSTRING,	"s"");
-	add_special(RECURSIVE,	"recursive");
-	add_special(NEXTFCODE,	"next-fcode");
+static const int number_of_builtin_tokens =
+	 sizeof(tokens_table)/sizeof(tic_hdr_t);
+
+/* **************************************************************************
+ *
+ *      Support functions specific to the FCode-Tokens list.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_token
+ *      Synopsis:       Emit the FCode token for the given FCode name.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             fc_name                     The name of the FCode
+ *         Local Static Variables:
+ *             fc_tokens_list_start        "Tail" of the "FC-Tokens" list
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *
+ *      Error Detection:
+ *          This routine should only be called with hard-coded names from
+ *              within the program.  If the given name is not found in
+ *              the Built-in Tokens Table, that is a FATAL error.
+ *
+ *      Process Explanation:
+ *          Because the "FCode-Tokens" table was linked first, and the
+ *              pointer kept for this purpose, the "FC-Tokens" list can
+ *              be a subset of the "core" list, yet, when necessary, can
+ *              be searched with the same routines.
+ *
+ *      Extraneous Remarks:
+ *          I will bend the strict rules of well-structured code;
+ *              the exception case should never occur.
+ *
+ **************************************************************************** */
+
+void emit_token( const char *fc_name)
+{
+
+    if ( handle_tic_vocab( (char *)fc_name, fc_tokens_list_start) )
+    {
+        return;
+    }
+
+    tokenization_error( FATAL, "Did not recognize FCode name %s", fc_name);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_token
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                      word in the "FC-Tokens" list
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                       The name to look up
+ *         Local Static Variables:
+ *             fc_tokens_list_start        "Tail" of the "FC-Tokens" list
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to the data-structure, or
+ *                                            NULL if not found.
+ *
+ **************************************************************************** */
+
+tic_hdr_t *lookup_token( char *tname)
+{
+    tic_hdr_t *found ;
+
+    found = lookup_tic_entry( tname, fc_tokens_list_start);
+    return ( found ) ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  entry_is_token
+ *      Synopsis:       Indicate whether the supplied pointer to a tic_hdr_t
+ *                      data-structure is one for which a single-token FCode
+ *                      number is assigned.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             test_entry                The entry to test; may be NULL
+ *         Local macros:
+ *             FC_TOKEN_FUNC             The function associated with
+ *                                           most single-token entries.
+ *             OBSO_FC_FUNC              The function associated with
+ *                                           "obsolete" FCode tokens.
+ *
+ *      Outputs:
+ *         Returned Value:               TRUE if the data-structure is
+ *                                           a single-token entry.
+ *
+ *      Process Explanation:
+ *          We cannot rely on the "definer" field to indicate whether
+ *              it is a single-token entry; instead, we will look at
+ *              the associated function.
+ *          Keep this routine here to avoid needing to export the names
+ *              of the permitted functions or their synonymous macros.
+ *              If we ever need to change it, we can do so at a single
+ *              point of maintenance.
+ *          Because the entry might have been found in the initial list
+ *              of entries to the "FCode-Tokens" list, we need to check
+ *              whether the associated function is either the general
+ *              single-token emitting function,  FC_TOKEN_FUNC , or the
+ *              function  OBSO_FC_FUNC , which presents a message before
+ *              emitting, but is still a valid single-token function.
+ *
+ **************************************************************************** */
+
+bool entry_is_token( tic_hdr_t *test_entry )
+{
+    bool retval = FALSE;
+    if ( test_entry != NULL )
+    {
+	if ( ( test_entry->funct == FC_TOKEN_FUNC )  ||
+	     ( test_entry->funct ==  OBSO_FC_FUNC )  )
+	{
+	    retval = TRUE;
+	}
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  token_entry_warning
+ *      Synopsis:       Issue whatever warnings the given token_entry
+ *                      requires.  F['] needs this.
+ *      Inputs:
+ *         Parameters:
+ *             test_entry                The entry to test; may be NULL
+ *         Local macro:
+ *             OBSO_FC_FUNC              The function associated with
+ *                                           "obsolete" entries.
+ *      Outputs:
+ *         Returned Value:               NONE
+ *
+ *      Error Detection:
+ *          Warnings required by the given token_entry.
+ *
+ *      Extraneous Remarks:
+ *          At present, it's only the "Obsolete" warning.
+ *          But this is the place to add others, 
+ *              should they become necessary.
+ *
+ **************************************************************************** */
+
+void token_entry_warning( tic_hdr_t *t_entry)
+{
+    if ( t_entry->funct == OBSO_FC_FUNC )
+    {
+	obsolete_warning();
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Create the initial "FWords" list.
+ *
+ **************************************************************************** */
+
+static tic_fwt_hdr_t fwords_list[] = {
+
+	BI_FWD_SKP_OW(COLON,	 	":") ,
+	BUILTIN_FWORD(SEMICOLON, 	";") ,
+	BI_FWD_SKP_OW(TICK, 		"'") ,
+	BUILTIN_FWORD(AGAIN, 		"again") ,
+	BI_FWD_SKP_OW(BRACK_TICK,  	 "[']") ,
+	BI_FWD_SKP_OW(ASCII,		"ascii") ,
+	BUILTIN_FWORD(BEGIN,		"begin") ,
+	BI_FWD_SKP_OW(BUFFER,		"buffer:") ,
+	BUILTIN_FWORD(CASE,		"case") ,
+	BI_FWD_SKP_OW(CONST,		"constant") ,
+	BI_FWD_SKP_OW(CONTROL,		"control") ,
+	BI_FWD_SKP_OW(CREATE,		"create") ,
+
+	BI_FWD_SKP_OW(DEFER,		"defer") ,
+	BUILTIN_FWORD(CDO,		"?do") ,
+	BUILTIN_FWORD(DO,		"do") ,
+	BUILTIN_FWORD(ELSE,		"else") ,
+	BUILTIN_FWORD(ENDCASE,		"endcase") ,
+	BUILTIN_FWORD(ENDOF,		"endof") ,
+	BUILTIN_FWORD(EXTERNAL, 	"external") ,
+	BI_FWD_SKP_OW(FIELD,		"field") ,
+	BUILTIN_FWORD(FINISH_DEVICE,	"finish-device" ) ,
+	BUILTIN_FWORD(HEADERLESS,	"headerless") ,
+	BUILTIN_FWORD(HEADERS,		"headers") ,
+
+	BUILTIN_FWORD(INSTANCE ,	"instance") ,
+
+	BUILTIN_FWORD(IF,		"if") ,
+	BUILTIN_FWORD(UNLOOP,		"unloop") ,
+	BUILTIN_FWORD(LEAVE,		"leave") ,
+	BUILTIN_FWORD(PLUS_LOOP, 	"+loop") ,
+	BUILTIN_FWORD(LOOP,		"loop") ,
+
+	BUILTIN_FWORD(OF,		"of") ,
+	BUILTIN_FWORD(REPEAT,		"repeat") ,
+	BUILTIN_FWORD(THEN,		"then") ,
+	BI_FWD_SKP_OW(TO,		"to") ,
+	BI_FWD_SKP_OW(IS,		"is") , /*  Deprecated synonym to TO  */
+	BUILTIN_FWORD(UNTIL,		"until") ,
+	BI_FWD_SKP_OW(VALUE,		"value") ,
+	BI_FWD_SKP_OW(VARIABLE,		"variable") ,
+	BUILTIN_FWORD(WHILE,		"while") ,
+	BUILTIN_FWORD(OFFSET16,		"offset16") ,
+
+	BI_FWD_STRING(STRING,		""") ,     /*  XXXXX  */
+	BI_FWD_STRING(PSTRING,		"."") ,    /*  XXXXX  */
+	BI_FWD_STRING(PBSTRING,		".(") ,     /*  XXXXX  */
+	BI_FWD_STRING(SSTRING,		"s"") ,    /*  XXXXX  */
+	BUILTIN_FWORD(IFILE_NAME,	"[input-file-name]"),
+	BUILTIN_FWORD(ILINE_NUM,	"[line-number]"),
+	BUILTIN_FWORD(RECURSE,		"recurse") ,
+	BUILTIN_FWORD(RECURSIVE,	"recursive") ,
+	BUILTIN_FWORD(RET_STK_FETCH,	"r@") ,
+	BUILTIN_FWORD(RET_STK_FROM,	"r>") ,
+	BUILTIN_FWORD(RET_STK_TO,	">r") ,
+	BUILTIN_FWORD(THEN,		"endif" ) ,  /*  Synonym for "then"  */
+	BUILTIN_FWORD(NEW_DEVICE,	"new-device" ) ,
+	BUILTIN_FWORD(LOOP_I,		"i") ,
+	BUILTIN_FWORD(LOOP_J,		"j") ,
    /* version1 is also an fcode word, but it 
     * needs to trigger some tokenizer internals */
-	add_special(VERSION1,	"version1");
-	add_special(START0,	"start0");
-	add_special(START1,	"start1");
-	add_special(START2,	"start2");
-	add_special(START4,	"start4");
-	add_special(END0,	"end0");
-	add_special(END1,	"end1");
-	add_special(FCODE_V1,	"fcode-version1");
-	add_special(FCODE_V2,	"fcode-version2");
-	add_special(FCODE_V3,	"fcode-version3");
-	add_special(FCODE_END,	"fcode-end");
-	add_special(FCODE_DATE,	"fcode-date");
-	add_special(FCODE_TIME,	"fcode-time");
-	
-	add_special(HEXVAL,	"h#");
-	add_special(DECVAL,	"d#");
-	add_special(OCTVAL,	"o#");
-	add_special(CHAR,	"char");
-	add_special(CCHAR,	"[char]");
-	add_special(ABORTTXT,   "abort"");
+	BUILTIN_FWORD(VERSION1,		"version1") ,
+	BUILTIN_FWORD(START0,		"start0") ,
+	BUILTIN_FWORD(START1,		"start1") ,
+	BUILTIN_FWORD(START2,		"start2") ,
+	BUILTIN_FWORD(START4,		"start4") ,
+	BUILTIN_FWORD(END0,		"end0") ,
+	BUILTIN_FWORD(END1,		"end1") ,
+	BUILTIN_FWORD(FCODE_V1,		"fcode-version1") ,
+	BUILTIN_FWORD(FCODE_V2,		"fcode-version2") ,
+	BUILTIN_FWORD(FCODE_V3,		"fcode-version3") ,
+	BUILTIN_FWORD(FCODE_END,	"fcode-end") ,

-	add_special(ENCODEFILE,	"encode-file");
+        /*  Support for IBM-style Locals  */
+	BI_FWD_STRING(CURLY_BRACE,	"{") ,
+	BI_FWD_STRING(DASH_ARROW,	"->") ,
+	BUILTIN_FWORD(EXIT,		"exit") ,

+
+	BUILTIN_FWORD(CHAR,		"char") ,
+	BUILTIN_FWORD(CCHAR,		"[char]") ,
+	BI_FWD_STRING(ABORTTXT,		"abort"") ,
+
+	BUILTIN_FWORD(ENCODEFILE,	"encode-file") ,
+
+	BI_IG_FW_HDLR(ESCAPETOK,	"tokenizer[") ,
+	BI_IG_FW_HDLR(ESCAPETOK,	"f[") ,       /*  An IBM-ish synonym  */
+};
+
+static const int number_of_builtin_fwords =
+	 sizeof(fwords_list)/sizeof(tic_hdr_t);
+
+/* **************************************************************************
+ *
+ *      Create the initial list of "Shared_Words" (words that can
+ *          be executed similarly both during normal tokenization,
+ *          and also within "Tokenizer Escape Mode").
+ *
+ **************************************************************************** */
+
+static tic_fwt_hdr_t shared_words_list[] = {
+	SHARED_FWORD(FLOAD,		"fload") ,
+	/*  As does the "Allow Multi-Line" directive   */
+	SHR_SAMIG_FWRD(ALLOW_MULTI_LINE, "multi-line") ,
+
+	SHR_FWD_SKOW( F_BRACK_TICK,	 "f[']") ,
+
+	SH_FW_SK2WIL(ALIAS, 		"alias") ,
+	SHARED_FWORD(DECIMAL,		"decimal") ,
+	SHARED_FWORD(HEX,		"hex") ,
+	SHARED_FWORD(OCTAL,		"octal") ,
+	SH_FW_SK_WIL(HEXVAL,		"h#") ,
+	SH_FW_SK_WIL(DECVAL,		"d#") ,
+	SH_FW_SK_WIL(OCTVAL,		"o#") ,
+
+	SH_FW_SK_WIL(ASC_NUM,		"a#") ,
+	SH_FW_SK_WIL(ASC_LEFT_NUM,	"al#") ,
+
+	/* IBM-style extension.  Might be generalizable...  */
+	SHARED_FWORD(FLITERAL, 	"fliteral") ,
+
+	/*  Directives to extract the value of a Command-Line symbol */
+	SH_FW_SK_WIL(DEFINED,		"[defined]") ,
+	SH_FW_SK_WIL(DEFINED,		"#defined") ,
+	SH_FW_SK_WIL(DEFINED,		"[#defined]") ,
+
+	/*  Present the current date or time, either as an  */
+	/*  in-line string or as a user-generated message.  */
+	SHARED_FWORD(FCODE_DATE,	"[fcode-date]") ,
+	SHARED_FWORD(FCODE_TIME,	"[fcode-time]") ,
+
+	/*  Current definition under construction, similarly  */
+	SHARED_FWORD(FUNC_NAME,	"[function-name]"),
+
+	/*  Synonymous forms of the #ELSE and #THEN operators,
+	 *      associated with Conditional-Compilation,
+	 *      allowing for various syntax-styles, and for
+	 *      expansion by alias.
+	 */
+
+	/*  #ELSE  operators */
+	SHARED_FWORD(CONDL_ELSE,	"#else") ,
+	SHARED_FWORD(CONDL_ELSE,	"[else]") ,
+	SHARED_FWORD(CONDL_ELSE,	"[#else]") ,
+
+	/*  #THEN  operators */
+	SHARED_FWORD(CONDL_ENDER,	"#then") ,
+	SHARED_FWORD(CONDL_ENDER,	"[then]") ,
+	SHARED_FWORD(CONDL_ENDER,	"[#then]") ,
+	/*   #ENDIF variants for users who favor C-style notation   */
+	SHARED_FWORD(CONDL_ENDER,	"#endif") ,
+	SHARED_FWORD(CONDL_ENDER,	"[endif]") ,
+	SHARED_FWORD(CONDL_ENDER,	"[#endif]") ,
+
+
+	SHARED_FWORD(OVERLOAD,  "overload" ) ,
+
+	SHARED_FWORD(GLOB_SCOPE , "global-definitions" ) ,
+	SHARED_FWORD(DEV_SCOPE , "device-definitions" ) ,
+
+	/*  Directives to change a command-line flag value from source   */
+	SH_FW_SK_WIL(CL_FLAG,	"[FLAG]") ,
+	SH_FW_SK_WIL(CL_FLAG,	"#FLAG") ,
+	SH_FW_SK_WIL(CL_FLAG,	"[#FLAG]") ,
+
+	/*  Directives to force display of a command-line flags' values   */
+	SHARED_FWORD(SHOW_CL_FLAGS,	"[FLAGS]") ,
+	SHARED_FWORD(SHOW_CL_FLAGS,	"#FLAGS") ,
+	SHARED_FWORD(SHOW_CL_FLAGS,	"[#FLAGS]") ,
+	SHARED_FWORD(SHOW_CL_FLAGS,	"SHOW-FLAGS") ,
+
+	/*  Directives to save and retrieve the FCode Assignment number  */
+	SHARED_FWORD(PUSH_FCODE,	"FCODE-PUSH") ,
+	SHARED_FWORD(POP_FCODE, 	"FCODE-POP") ,
+
+	/*  Directive to reset the FCode Assignment number and
+	 *      re-initialize FCode Range overlap checking.
+	 */
+	SHARED_FWORD(RESET_FCODE,	"FCODE-RESET") ,
+
    /* pci header generation is done differently 
     * across the available tokenizers. We try to
     * be compatible to all of them
     */
-	add_special(PCIHDR,	"pci-header");
-	add_special(PCIEND,	"pci-end");            /* SUN syntax */
-	add_special(PCIEND,	"pci-header-end");     /* Firmworks syntax */
-	add_special(PCIREV,	"pci-revision");       /* SUN syntax */
-	add_special(PCIREV,	"pci-code-revision");  /* SUN syntax */
-	add_special(PCIREV,	"set-rev-level");      /* Firmworks syntax */
-	add_special(NOTLAST,	"not-last-image");
+	SHARED_FWORD(PCIHDR,	  "pci-header") ,
+	SHARED_FWORD(PCIEND,	  "pci-end") ,           /* SUN syntax */
+	SHARED_FWORD(PCIEND,	  "pci-header-end") ,    /* Firmworks syntax */
+	SHARED_FWORD(PCIREV,	  "pci-revision") ,      /* SUN syntax */
+	SHARED_FWORD(PCIREV,	  "pci-code-revision") , /* SUN syntax */
+	SHARED_FWORD(PCIREV,	  "set-rev-level") ,     /* Firmworks syntax */
+	SHARED_FWORD(NOTLAST,	  "not-last-image") ,
+	SHARED_FWORD(NOTLAST,	  "not-last-img") ,      /* Shorthand form  */
+	SHARED_FWORD(ISLAST,	  "last-image") ,
+	SHARED_FWORD(ISLAST,	  "last-img") ,          /* Shorthand form  */
+	SHARED_FWORD(SETLAST,	  "set-last-image") ,
+	SHARED_FWORD(SETLAST,	  "set-last-img") ,      /* Shorthand form  */
+
+	SH_FW_SK_WIL(SAVEIMG,	  "save-image") ,
+	SH_FW_SK_WIL(SAVEIMG,	  "save-img") ,          /* Shorthand form  */
+
+	SHARED_FWORD(RESETSYMBS,  "reset-symbols") ,
+
+	/*  User-Macro definers    */
+	SHARED_IG_HDLR("[MACRO]", add_user_macro,  0 ,  skip_user_macro) ,
+
+	/*  Comments and Remarks   */
+	SHARED_IG_HDLR("\",      process_remark, '\n', process_remark) ,
+	SHARED_IG_HDLR("(",       process_remark, ')',  process_remark) ,
+
+	/*  Directives to print or discard a user-generated message */
+	SHARED_IG_HDLR("[MESSAGE]",  user_message, '\n', skip_user_message) ,
+	SHARED_IG_HDLR("#MESSAGE",   user_message, '\n', skip_user_message) ,
+	SHARED_IG_HDLR("[#MESSAGE]", user_message, '\n', skip_user_message) ,
+	SHARED_IG_HDLR("#MESSAGE"", user_message, '"' , skip_user_message) ,
+};
+
+static const int number_of_shared_words =
+	sizeof(shared_words_list)/sizeof(tic_hdr_t);
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_shared_word
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                      word, only if it is a "Shared Word"
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to look for
+ *         Local Static Variables:
+ *             global_voc_dict_ptr       "Tail" of Global Vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to the data-structure, or
+ *                                            NULL if not found.
+ *
+ *      Process Explanation:
+ *          The "Shared Words" are scattered among the Global Vocabulary;
+ *              the user is allowed to create aliases, which may be in the
+ *              Current-Device.  We will search through the "current" scope
+ *              and decide whether the name we found is a "Shared Word" by
+ *              looking for  COMMON_FWORD  in the "definer" field.
+ *
+ *      Extraneous Remarks:
+ *          This is the only place where an additional check of the
+ *              "definer" field is required to identify a desired entry.
+ *              Should another "definer"-type be required, I recommend 
+ *              defining a general-purpose function in  ticvocab.c  and
+ *              applying it here and in the other place(s).
+ *
+ **************************************************************************** */
+ 
+tic_hdr_t *lookup_shared_word( char *tname)
+{
+    tic_hdr_t *found ;
+    tic_hdr_t *retval = NULL ;
+
+    found = lookup_current( tname );
+    if ( found != NULL )
+    {
+	if ( found->fword_defr == COMMON_FWORD )
+	{
+	    retval = found ;
+	}
+    }
+
+    return ( retval );
+
 }
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_shared_word
+ *      Synopsis:       Perform the function associated with the given name
+ *                      only if it is a "Shared Word".  Indicate if it was.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                The "target" name for which to look
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if the name is a valid "Shared Word"
+ *
+ *      Extraneous Remarks:
+ *          This is very similar to a call to  handle_tic_vocab() except
+ *              for the additional filtering for a "Shared Word" definer.
+ *
+ **************************************************************************** */
+
+bool handle_shared_word( char *tname )
+{
+    tic_hdr_t *found ;
+    bool retval = FALSE;
+    
+    found = lookup_shared_word( tname );
+    if ( found != NULL )
+    {
+        found->funct(found->pfield);
+	retval = TRUE;
+    }
+
+    return ( retval ) ;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_shared_f_exec_word
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                      word, only if it is a "Shared F-Exec Word"
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                     The name to look for
+ *         Local Static Variables:
+ *             global_voc_dict_ptr       "Tail" of Global Vocabulary
+ *         Macro Definitions:
+ *             FWORD_EXEC_FUNC           The "Active" function of the
+ *                                           sub-class of "Shared Word"s
+ *                                           that is the object of this
+ *                                           routine's search
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to the data-structure, or
+ *                                            NULL if not found.
+ *
+ *      Process Explanation:
+ *          The "Shared F-Exec Words" are the subset of "Shared Words" that
+ *              have the  FWORD_EXEC_FUNC  as their "Active" function.
+ *
+ *      Extraneous Remarks:
+ *          This is the only routine that requires a check of two fields;
+ *              it seems unlikely that there will be any need to generalize
+ *              the core of this routine...
+ *
+ **************************************************************************** */
+ 
+tic_hdr_t *lookup_shared_f_exec_word( char *tname)
+{
+    tic_hdr_t *found ;
+    tic_hdr_t *retval = NULL ;
+
+    found = lookup_shared_word( tname );
+    if ( found != NULL )
+    {
+	if ( found->funct == FWORD_EXEC_FUNC )
+	{
+	    retval = found ;
+	}
+    }
+
+    return ( retval );
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  init_dictionary
+ *      Synopsis:       Initialize all the vocabularies.  For the Global
+ *                      Vocabulary, fill in the link fields in each of the
+ *                      otherwise pre-initialized built-in lists, link the
+ *                      lists together, and set the relevant pointers.   For
+ *                      other lists, call their initialization routines.
+ *
+ *      Inputs:
+ *         Parameters:                         NONE
+ *         Global Variables:
+ *         Local Static Variables:
+ *             tokens_table                    Base of the "FC-Tokens" list
+ *             number_of_builtin_tokens        Number of "FC-Token" entries
+ *             fwords_list                     Base of the "FWords" list
+ *             number_of_builtin_fwords        Number of "FWord" entries
+ *             shared_words_list               Base of the "Shared Words" list
+ *             number_of_shared_words          Number of "Shared Words" entries
+ *
+ *      Outputs:
+ *         Returned Value:                      NONE
+ *         Local Static Variables:
+ *             global_voc_dict_ptr              "Tail" of Global Vocabulary
+ *             fc_tokens_list_start             "Tail" of "FC-Tokens" list
+ *             fc_tokens_list_ender             End of "FC-Tokens" search
+ *             shared_fwords_ender              End of Shared Words" search
+ *             global_voc_reset_ptr             Reset-point for Global Vocab
+ *
+ *      Process Explanation:
+ *          The first linked will be the last searched.
+ *              Link the "FC-Tokens" first, and mark their limits
+ *              Link the "FWords" next,
+ *              Mark the end-limit of the "Shared Words", and link them
+ *              The "Conditionals", defined in another file, are also "Shared";
+ *                  link them next.
+ *              Then link the Built-In Macros, also defined in another file.
+ *          These constitute the Global Vocabulary.
+ *              Mark the reset-point for the Global Vocabulary.
+ *          The "Tokenizer Escape" mode vocabulary is not linked to the Global
+ *              Vocabulary; call its initialization routine.
+ *
+ *      Extraneous Remarks:
+ *          I only wish I had done this sooner...
+ *
+ **************************************************************************** */
+
+void init_dictionary( void )
+{
+
+    global_voc_dict_ptr  = NULL;   /*  Belt-and-suspenders...  */
+
+    /*  The "FC-Tokens" list must be linked first.  */
+    fc_tokens_list_ender = global_voc_dict_ptr;
+    init_tic_vocab( tokens_table,
+                	number_of_builtin_tokens,
+                            &global_voc_dict_ptr ) ;
+    fc_tokens_list_start = global_voc_dict_ptr;
+
+    /*  Link the "FWords" next */
+   init_tic_vocab( (tic_hdr_t *)fwords_list,
+                        number_of_builtin_fwords,
+                            &global_voc_dict_ptr ) ;
+
+    /*  Mark the end-limit of the "Shared Words", and link them. */
+    shared_fwords_ender = global_voc_dict_ptr;
+    init_tic_vocab( (tic_hdr_t *)shared_words_list,
+                	number_of_shared_words,
+                            &global_voc_dict_ptr ) ;
+
+    /*  Link the "Conditionals" to the Global Vocabulary.  */
+    init_conditionals_vocab( &global_voc_dict_ptr ) ;
+
+    /*  Link the Built-In Macros */
+    init_macros( &global_voc_dict_ptr ) ;
+
+    /*  Mark the reset-point for the Global Vocabulary.  */
+    global_voc_reset_ptr = global_voc_dict_ptr;
+
+    /*   Initialize the "Tokenizer Escape" mode vocabulary  */
+    init_tokz_esc_vocab();
+
+    /*   Locals and Device-Node vocabularies are initially empty  */
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_normal_vocabs
+ *      Synopsis:       Reset the vocabularies that were created in "Normal"
+ *                          (as distinguished from "Tokenizer Escape") mode
+ *                          to the proper state for a fresh tokenization.
+ *
+ *      Associated FORTH words:              END0  END1
+ *      Associated Tokenizer directives:     RESET-SYMBOLS  (in "Normal" mode)
+ *                                           FCODE-END
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             global_voc_reset_ptr       Position to which to reset
+ *                                            the "Global" Vocabulary
+ *             current_device_node        Vocab struct of current dev-node
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Global Variables:    
+ *             global_voc_dict_ptr       Reset to "Built-In" position 
+ *             current_device_node       Reset to point at "root" dev-node
+ *         Memory Freed
+ *             All memory allocated by user-definitions in "normal" mode
+ *             -- Macros, Conditionals and Device-node Vocabularies -- are
+ *             reset via function call.
+ *
+ *      Error Detection:
+ *          Presence of extra device-node data structure(s) indicates that
+ *              there were more "new-device" calls than "finish-device";
+ *              report each as an ERROR 
+ *
+ *      Process Explanation:
+ *          Vocabularies created in other files, that have different
+ *              data-structures, will have "reset" routines of their
+ *              own associated with them in the files in which they
+ *              are created.  Those routines will be called from here.
+ *          Definitions in the "Tokenizer Escape Vocabulary", i.e.,
+ *              the one in effect in "Tokenizer Escape" mode, are
+ *              specifically not touched by this routine.
+ *
+ **************************************************************************** */
+
+void reset_normal_vocabs( void )
+{
+    reset_tic_vocab( &global_voc_dict_ptr, global_voc_reset_ptr );
+
+    /*  Delete the top-level device-node vocab.
+     *      If there are extra device-nodes,
+     *      delete their data structures and show errors
+     */
+    do
+    {
+        if ( current_device_node->parent_node != NULL )
+	{
+	    tokenization_error( TKERROR,
+		 "Missing FINISH-DEVICE for new device");
+	    started_at( current_device_node->ifile_name,
+		 current_device_node->line_no );
+	}
+	    delete_device_vocab();
+    }  while ( current_device_node->parent_node != NULL );
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_vocabs
+ *      Synopsis:       Reset all the vocabularies to the proper state
+ *                      for beginning a fresh tokenization, particularly
+ *                      when multiple files are named on the command-line
+ *      
+ *      Inputs:
+ *         Parameters:                NONE
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Memory Freed
+ *             All memory allocated by user-definitions will be freed
+ *
+ *      Process Explanation:
+ *          Call the  reset_normal_vocabs()  routine to get the vocabularies
+ *              that apply to "Normal" mode, then call the "reset" routine
+ *              for the "Tokenizer Escape Vocabulary", which is supposed
+ *              to persist across device-nodes but not across input-files
+ *              named on the command-line.
+ *
+ **************************************************************************** */
+
+void reset_vocabs( void )
+{
+    reset_normal_vocabs();
+    reset_tokz_esc();
+}

Modified: fcode-utils/toke/dictionary.h
===================================================================
--- fcode-utils/toke/dictionary.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/dictionary.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _TOKE_DICTIONARY_H
+#define _TOKE_DICTIONARY_H
+
 /*
  *                     OpenBIOS - free your system! 
  *                         ( FCode tokenizer )
@@ -24,81 +27,152 @@
  *
  */

-#define COLON		0x01
-#define SEMICOLON	0x02
-#define TOKENIZE	0x03
-#define AGAIN		0x04
-#define ALIAS		0x05
-#define GETTOKEN 	0x06
-#define ASCII		0x07
-#define BEGIN		0x08
-#define BUFFER		0x09
-#define CASE		0x0a
-#define CONST		0x0b
-#define CONTROL		0x0c
-#define CREATE		0x0d
-#define DECIMAL		0x0e
-#define DEFER		0x0f
-#define CDO		0x10
-#define DO		0x11
-#define ELSE		0x12
-#define ENDCASE		0x13
-#define ENDOF		0x14
-#define EXTERNAL	0x15
-#define FIELD		0x16
-#define HEADERLESS	0x17
-#define HEADERS		0x18
-#define HEX		0x19
-#define IF		0x1a
-#define CLEAVE		0x1b
-#define LEAVE		0x1c
-#define CLOOP		0x1d
-#define LOOP		0x1e
-#define OCTAL		0x1f
-#define OF		0x20
-#define REPEAT		0x21
-#define THEN		0x22
-#define TO		0x23
-#define UNTIL		0x24
-#define VALUE		0x25
-#define VARIABLE	0x26	
-#define WHILE		0x27
-#define OFFSET16	0x28	
-#define BEGINTOK	0x29	
-#define EMITBYTE	0x2a	
-#define ENDTOK		0x2b
-#define FLOAD		0x2c
-#define STRING		0x2d
-#define PSTRING		0x2e
-#define PBSTRING	0x2f
-#define SSTRING		0x30
-#define RECURSIVE	0x31
-#define HEXVAL		0x32
-#define DECVAL		0x33
-#define OCTVAL		0x34
+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */

-#define END0		0xdb
-#define END1		0xdc
-#define CHAR		0xdd
-#define CCHAR		0xde
-#define ABORTTXT	0xdf
+/* **************************************************************************
+ *
+ *      Numeric values for FWord-type vocabulary entries.  Used by the
+ *          big "switch" statement in handle_internal(); a subset are 
+ *          also used as "definer-type" values associated with various
+ *          types of definitions.
+ *
+ **************************************************************************** */

-#define NEXTFCODE	0xef
+typedef enum fword_token {
+      UNSPECIFIED  = 0xBAADD00D ,  /*  Default (absence-of) "definer"        */
+      COMMON_FWORD = 0xC0EDC0DE ,  /*  Definer indicating a "shared" FWord   */
+      BI_FWRD_DEFN = 0xB1F4409D ,  /*  Definer indicating a "built-in FWord" */
+      COLON = 1 ,
+      SEMICOLON ,
+      TICK ,
+      AGAIN ,
+      ALIAS ,
+      BRACK_TICK ,
+      F_BRACK_TICK ,
+      ASCII ,
+      BEGIN ,
+      BUFFER ,
+      CASE ,
+      CONST ,
+      CONTROL ,
+      CREATE ,
+      DECIMAL ,
+      DEFER ,
+      DEFINED ,
+      CDO ,
+      DO ,
+      ELSE ,
+      ENDCASE ,
+      ENDOF ,
+      EXTERNAL ,
+      INSTANCE ,
+      FIELD ,
+      NEW_DEVICE ,
+      FINISH_DEVICE ,
+      FLITERAL ,
+      HEADERLESS ,
+      HEADERS ,
+      HEX ,
+      IF ,
+      UNLOOP ,
+      LEAVE ,
+      LOOP_I ,
+      LOOP_J ,
+      LOOP ,
+      PLUS_LOOP ,
+      OCTAL ,
+      OF ,
+      REPEAT ,
+      THEN ,
+      TO ,
+      IS ,
+      UNTIL ,
+      VALUE ,
+      VARIABLE ,
+      WHILE ,
+      OFFSET16 ,
+      ESCAPETOK ,
+      EMITBYTE ,
+      FLOAD ,
+      STRING ,
+      PSTRING ,
+      PBSTRING ,
+      SSTRING ,
+      RECURSIVE ,
+      RECURSE ,
+      RET_STK_FETCH ,
+      RET_STK_FROM ,
+      RET_STK_TO ,
+      HEXVAL ,
+      DECVAL ,
+      OCTVAL ,

-#define ENCODEFILE	0xf0
+       ret_stk_from ,
+     ASC_NUM ,          /*  Convert char seq to number  */
+      ASC_LEFT_NUM ,     /*  same, only left-justified.  */

-#define FCODE_V1	0xf1
-#define FCODE_V3	0xf2
-#define NOTLAST		0xf3
-#define PCIREV		0xf4
-#define PCIHDR		0xf5
-#define PCIEND		0xf6
-#define START0		0xf7
-#define START1		0xf8
-#define START2		0xf9
-#define START4		0xfa
-#define VERSION1	0xfb
-#define FCODE_TIME	0xfc
-#define FCODE_DATE	0xfd
-#define FCODE_V2	0xfe
-#define FCODE_END	0xff
+      CONDL_ENDER ,      /*  Conditional "[THEN] / [ENDIF]" variants  */
+      CONDL_ELSE ,       /*  Conditional "[ELSE]" directive variants  */
+
+      PUSH_FCODE ,	/*  Save the FCode Assignment number  */
+      POP_FCODE ,	/*  Retrieve the FCode Assignment number  */
+      RESET_FCODE ,	/*  Reset FCode Ass't nr and overlap checking  */
+
+      CURLY_BRACE ,      /*  Support for IBM-style Locals  */
+      DASH_ARROW ,
+      LOCAL_VAL ,
+      EXIT ,
+
+      FUNC_NAME ,
+      IFILE_NAME ,
+      ILINE_NUM ,
+
+      CL_FLAG ,
+      SHOW_CL_FLAGS ,
+
+      OVERLOAD ,
+      ALLOW_MULTI_LINE ,
+      MACRO_DEF ,
+      GLOB_SCOPE ,
+      DEV_SCOPE ,
+
+      /*  This value has to be adjusted
+       *      so that  FCODE_END  comes
+       *      out to be  0xff
+       */
+      END0 = 0xd7 ,      /*   0xd7   */
+      END1 ,             /*   0xd8   */
+      CHAR ,             /*   0xd9   */
+      CCHAR ,            /*   0xda   */
+      ABORTTXT ,         /*   0xdb   */
+
+      NEXTFCODE ,        /*   0xdc   */
+
+      ENCODEFILE ,       /*   0xdd   */
+
+      FCODE_V1 ,         /*   0xde   */
+      FCODE_V3 ,         /*   0xdf   */
+      NOTLAST ,          /*   0xef   */
+      ISLAST ,           /*   0xf0   */
+      SETLAST ,          /*   0xf1   */
+      PCIREV ,           /*   0xf2   */
+      PCIHDR ,           /*   0xf3   */
+      PCIEND ,           /*   0xf4   */
+      RESETSYMBS ,       /*   0xf5   */
+      SAVEIMG ,          /*   0xf6   */
+      START0 ,           /*   0xf7   */
+      START1 ,           /*   0xf8   */
+      START2 ,           /*   0xf9   */
+      START4 ,           /*   0xfa   */
+      VERSION1 ,         /*   0xfb   */
+      FCODE_TIME ,       /*   0xfc   */
+      FCODE_DATE ,       /*   0xfd   */
+      FCODE_V2 ,         /*   0xfe   */
+      FCODE_END = 0xff   /*   0xff   */
+}  fwtoken ;
+
+#endif   /*  _TOKE_DICTIONARY_H    */

Modified: fcode-utils/toke/emit.c
===================================================================
--- fcode-utils/toke/emit.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/emit.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,283 +24,548 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+
+#include "pcihdr.h"
+
 #include "toke.h"
+#include "vocabfuncts.h"
 #include "stack.h"
+#include "scanner.h"
 #include "emit.h"
+#include "clflags.h"
+#include "errhandler.h"
+#include "stream.h"
+#include "nextfcode.h"

-extern bool offs16;
-extern int verbose;
-extern u8 *ostart, *opc, *oend;
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              verbose         Enable optional messages, set by "-v" switch
+ *              noerrors        "Ignore Errors" flag, set by "-i" switch
+ *
+ **************************************************************************** */

-/* PCI data */
-extern u16 pci_vpd, pci_revision;
-extern bool pci_is_last_image;

-/* header pointers */
-extern u8  *fcode_hdr;
-extern u8  *pci_hdr;
+/* **************************************************************************
+ *
+ *      Macro to zero-fill a field of the size of the given structure
+ *          into the Output Buffer using the  emit_byte()  routine.
+ *
+ **************************************************************************** */

-extern bool haveend;
+#define EMIT_STRUCT(x)  {int j; for (j=0; j < sizeof(x) ; j++ ) emit_byte(0); }

-u16 lookup_token(char *name);

-static bool little_endian(void)
+/* **************************************************************************
+ *
+ *      Local Static Variables, offsets into Output Buffer of ...:
+ *          fcode_start_ob_off      First FCode-Start byte
+ *          fcode_hdr_ob_off        FCode Header (Format byte)
+ *          fcode_body_ob_off       First functional FCode after FCode Header.
+ *          pci_hdr_ob_off          Start of PCI ROM Header Block structure
+ *          pci_data_blk_ob_off     Start of PCI Data Header Block structure
+ *
+ *     For all of these, -1 means "Not initialized"
+ *
+ *************************************************************************** */
+
+static  int fcode_start_ob_off = -1;
+static  int fcode_hdr_ob_off = -1;
+static  int fcode_body_ob_off = -1;
+static  int pci_hdr_ob_off = -1;
+static  int pci_data_blk_ob_off = -1;
+
+/* **************************************************************************
+ *
+ *          These are to detect a particular error:  If FCode has
+ *              been written, before an Fcode-start<n> or before
+ *               a PCI-header
+ *
+ **************************************************************************** */
+
+static bool fcode_written = FALSE;
+
+/* **************************************************************************
+ *
+ *          Variables and Functions Imported, with
+ *          Exposure as Limited as possible:
+ *              ostart
+ *              olen
+ *              increase_output_buffer
+ *
+ **************************************************************************** */
+
+extern u8 *ostart;
+extern int olen;
+extern void increase_output_buffer( void);
+
+
+/* **************************************************************************
+ *
+ *      Function name:  init_emit
+ *      Synopsis:       Initialize Local Static Variables before starting
+ *                          Output.
+ *                      Exposure as Limited as possible.
+ *
+ **************************************************************************** */
+void init_emit( void);       /*    Prototype.  Limit Exposure.   */
+void init_emit( void)
 {
-	static short one=1;
-	return *(char *)&one==1;
+    fcode_start_ob_off  = -1;
+    fcode_hdr_ob_off    = -1;
+    fcode_body_ob_off   = -1;
+    pci_hdr_ob_off      = -1;
+    pci_data_blk_ob_off = -1;
+    fcode_written       = FALSE;
+    haveend             = FALSE;   /*  Get this one too...  */
 }

-int emit_byte(u8 data)
-{
-	u8 *newout;
-	int newsize;

-	if(opc==oend) {
-		/* need more output space */
-		newsize = (oend - ostart) * 2;
-		printf("Increasing output buffer to %d bytes.\n", newsize);
-		if ((newout=realloc(ostart, newsize)) == NULL) {
-			printf("toke: could not allocate %d bytes for output buffer\n", newsize);
-			exit(-1);
-		}
+/* **************************************************************************
+ *
+ *      Function name:  emit_byte
+ *      Synopsis:       Fundamental routine for placing a byte
+ *                          into the Output Buffer.  Also, check
+ *                          whether the buffer needs to be expanded.
+ *                      For internal use only.
+ *
+ **************************************************************************** */

-		/* move pointers */
-		opc=newout+(opc-ostart);
-		ostart=newout;
-		oend=ostart+newsize;
+static void emit_byte(u8 data)
+{
+	if ( opc == olen)
+	{
+	    increase_output_buffer();
    }
    
-	*opc=data;
+
+	*(ostart+opc) = data ;
    opc++;
-
-	return 0;
 }

-int emit_fcode(u16 tok)
+void emit_fcode(u16 tok)
 {
    if ((tok>>8))
    	emit_byte(tok>>8);

emit_byte(tok&0xff);
-	
-	return 0;
+	fcode_written = TRUE;
 }

-int emit_token(const char *name)
+static void emit_num32(u32 num)
 {
-	return emit_fcode(lookup_token((char *)name));
-}
-
-int emit_num32(u32 num)
-{
    emit_byte(num>>24);
    emit_byte((num>>16)&0xff);
    emit_byte((num>>8)&0xff);
    emit_byte(num&0xff);

-	return 0;
 }

-int emit_num16(u16 num)
+/* **************************************************************************
+ *
+ *      Function name:  user_emit_byte
+ *      Synopsis:       Action of user-mandated call to emit-byte.
+ *                          Covers the corner-case where this is the
+ *                          first command issued in the source input.
+ *
+ **************************************************************************** */
+
+void user_emit_byte(u8 data)
 {
-	emit_byte(num>>8);
-	emit_byte(num&0xff);
-
-	return 0;
+	emit_byte( data);
+	fcode_written = TRUE;
 }

-int emit_offset(s16 offs)
+void emit_offset(s16 offs)
 {
+    /*  Calling routine will test for out-of-range FCode-Offset  */
    if (offs16)
-		emit_num16(offs);
-	else
-		emit_byte(offs);
-	
-	return 0;
+		emit_byte(offs>>8);
+	emit_byte(offs&0xff);
 }

-s16 receive_offset(void)
+void emit_literal(u32 num)
 {
-	s16 offs=0;
-	
-	if (offs16) {
-		offs=((*opc)<<8)|(*(opc+1));
-	} else
-		offs=(*opc);
-
-	return offs;
+    emit_token("b(lit)");
+    emit_num32(num);
 }

-int emit_string(u8 *string, unsigned int cnt)
+void emit_string(u8 *string, signed int cnt)
 {
-	unsigned int i=0;
+	signed int i=0;
+	signed int cnt_cpy = cnt;
    
-	if (cnt>255) {
-		printf("string too long.");
-		exit(1);
+	if ( cnt_cpy > STRING_LEN_MAX )
+	{
+	    tokenization_error( TKERROR,
+		"String too long:  %d characters.  Truncating.\n",cnt);
+	    cnt_cpy = STRING_LEN_MAX ;
    }
-	emit_byte(cnt);
-	for (i=0; i<cnt; i++)
+	emit_byte(cnt_cpy);
+	for (i=0; i<cnt_cpy; i++)
    	emit_byte(string[i]);
-
-	return 0;
 }

-int emit_fcodehdr(void)
+void emit_fcodehdr(const char *starter_name)
 {
-	/* We comply with IEEE 1275-1994 */
-	emit_byte(0x08);
    
-	/*  checksum */
-	emit_num16(0);
+   /*  Check for error conditions   */
+    if ( fcode_written )
+    {
+        tokenization_error( TKERROR ,
+	    "Cannot create FCode header after FCode output has begun.\n");
+        if ( ! noerrors ) return ;
+    }

-	/* len */
-	emit_num32(0);
+	fcode_header_t *fcode_hdr;

-	return 0;
+	fcode_start_ob_off = opc;
+	emit_token( starter_name );
+
+	fcode_hdr_ob_off = opc;
+	fcode_hdr = (fcode_header_t *)(ostart+fcode_hdr_ob_off);
+
+	EMIT_STRUCT(fcode_header_t);
+
+	fcode_body_ob_off = opc;
+
+	/* Format = 8 means we comply with IEEE 1275-1994 */
+	fcode_hdr->format = 0x08;
+
 }

-int finish_fcodehdr(void)
+/* **************************************************************************
+ *
+ *      Function name:    finish_fcodehdr
+ *          Fill in the FCode header's checksum and length fields, and
+ *              reset the FCode-header pointer for the next image.
+ *
+ *          If  haveend  is true then the end0 has already been written
+ *              and  fcode_ender()  has been called.
+ *
+ *          Print a WARNING message if the end-of-file was encountered
+ *              without an end0 or an fcode-end
+ *
+ *          Print an informative message to standard-output giving the
+ *              checksum.  Call  list_fcode_ranges()  to print the
+ *              value of the last FCode-token that was assigned or
+ *              the Ranges of assigned FCode-token values if so be...
+ *              The final FCode-binary file-length will be printed
+ *              when the binary file is being closed.
+ *
+ **************************************************************************** */
+
+void finish_fcodehdr(void)
 {
-	u16 checksum=0;
-	u32 len,i;

-	if(!fcode_hdr)
+	if( fcode_hdr_ob_off == -1 )
    {
-		printf("warning: trying to fix up unknown fcode header\n");
-		return -1;
+		tokenization_error( TKERROR,
+		    "Missing FCode header.\n");
+		return ;
    }

-	/* If the program does not do this, we do */
+	/* If the program did not end cleanly, we'll handle it */
    if (!haveend)
+	{
+	    tokenization_error ( WARNING,
+	    "End-of-file encountered without END0 or FCODE-END.  "
+		"Supplying END0\n");
    	emit_token("end0");
+	    fcode_ender();
+	}

-	len=(unsigned long)opc-(unsigned long)(fcode_hdr-1);
+	/*  Calculate and place checksum and length, if haven't already  */
+	if ( fcode_start_ob_off != -1 )
+	{
+	    u16 checksum;
+	    int length;
+
+	    u8 *fcode_body = ostart+fcode_body_ob_off;
+	    u8 *ob_end = ostart+opc;
+	    fcode_header_t *fcode_hdr =
+	         (fcode_header_t *)(ostart+fcode_hdr_ob_off);
    
-	for (i=8;i<len;i++)
-		checksum+=fcode_hdr[i-1];
+	    length = opc - fcode_start_ob_off;

-	dpush((unsigned long)opc);
-	opc=fcode_hdr+1;
-	emit_num16(checksum);
-	emit_num32(len);
-	opc=(u8 *)dpop();
+	    for ( checksum = 0;
+	              fcode_body < ob_end ;
+		          checksum += *(fcode_body++) ) ;
+
+	    BIG_ENDIAN_WORD_STORE(fcode_hdr->checksum , checksum);
+	    BIG_ENDIAN_LONG_STORE(fcode_hdr->length , length);
+
    if (verbose)
-		printf("toke: checksum is 0x%04x (%d bytes)\n", 
-							checksum, len);
+	    {
+		printf( "toke: checksum is 0x%04x (%d bytes).  ",
+		    checksum, length);
+		list_fcode_ranges( TRUE);
+	    }
+	}

-	fcode_hdr=NULL;
+	/*  Reset things for the next image...   */
+	fcode_start_ob_off = -1;
+	fcode_hdr_ob_off   = -1;
+	fcode_body_ob_off  = -1;
+	fcode_written      = FALSE;
    haveend=FALSE;
-	return 0;
 }

-int emit_pcihdr(u16 vid, u16 did, u32 classid)
+/* **************************************************************************
+ *
+ *      Function name:  emit_pci_rom_hdr
+ *      Synopsis:       Write the PCI ROM Header Block into the Output Buffer
+ *
+ *      Inputs:
+ *         Parameters:                   NONE
+ *         Global Variables:        
+ *             opc                       Output Buffer Position Counter
+ *             ostart                    Start of Output Buffer
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:    
+ *             pci_hdr_ob_off            PCI ROM Header Block Position
+ *                                           (Offset) in Output Buffer
+ *         FCode Output buffer:
+ *             Write the part of the PCI ROM Header Block we know:
+ *                 Fill in the signature and the field reserved for
+ *                 "processor architecture unique data".
+ *             Fill in the "Pointer to PCI Data Structure" with the
+ *                 size of the data structure, because the first PCI
+ *                 Data Structure will follow immediately.
+ *
+ *      Error Detection:   (Handled by calling routine)
+ *
+ **************************************************************************** */
+
+static void emit_pci_rom_hdr(void)
 {
-	int i;
+    rom_header_t *pci_hdr;
+    pci_hdr_ob_off = opc;
+    pci_hdr = (rom_header_t *)(ostart + pci_hdr_ob_off);
+
+    EMIT_STRUCT(rom_header_t);
    
    /* PCI start signature */
-	emit_byte(0x55); emit_byte(0xaa);
+    LITTLE_ENDIAN_WORD_STORE(pci_hdr->signature,0xaa55);
    
    /* Processor architecture */
-	emit_byte(0x34); emit_byte(0x00);
+	/*  Note:
+	 *  The legacy code used to read:
+	 *
+	 *        pci_hdr->reserved[0] = 0x34;
+	 *
+	 *  I don't know what significance the literal  34  had, but
+	 *      by what might just be an odd coincidence, it is equal
+	 *      to the combined lengths of the  PCI-ROM-  and  PCI-Data-
+	 *      headers.
+	 *
+	 *  I suspect that it is more than merely an odd coincidence,
+	 *      and that the following would be preferable:
+	 */

-	/* 20 bytes of padding */
-	for (i=0; i<20; i++) emit_byte(0x00);
+    LITTLE_ENDIAN_WORD_STORE( pci_hdr->reserved ,
+	(sizeof(rom_header_t) + sizeof(pci_data_t)) ) ;

+	/* already handled padding */
+
    /* pointer to start of PCI data structure */
-	emit_byte(0x1c); emit_byte(0x00);
+    LITTLE_ENDIAN_WORD_STORE(pci_hdr->data_ptr, sizeof(rom_header_t) );

-	/* 2 bytes of zero padding */
-	emit_byte(0x00); emit_byte(0x00);
+}
    
-	/* PCI Data structure */
-	emit_byte('P'); emit_byte('C'); emit_byte('I'); emit_byte('R');
-	/* vendor id */
-	emit_byte(vid&0xff); emit_byte(vid>>8);
-	/* device id */
-	emit_byte(did&0xff); emit_byte(did>>8);
-	/* vital product data */
-	emit_byte(0x00); emit_byte(0x00);
-	/* length of pci data structure */
-	emit_byte(0x18); emit_byte(0x00);
-	/* PCI data structure revision */
-	emit_byte(0x00);
-	
-	if (little_endian()) {
-		/* reg level programming */
-		emit_byte(classid & 0xff);
-		/* class code */
-		emit_byte((classid>>8)&0xff);
-		emit_byte((classid>>16)&0xff);
-	} else {
-		/* class code */
-		emit_byte((classid>>8)&0xff);
-		emit_byte((classid>>16)&0xff);
-		/* reg level programming */
-		emit_byte(classid & 0xff);
+/* **************************************************************************
+ *
+ *      Function name:  emit_pci_data_block
+ *      Synopsis:       Write the PCI Data Block into the Output Buffer
+ *
+ *      Inputs:
+ *         Parameters:                   NONE
+ *         Global Variables:        
+ *             opc                       Output Buffer Position Counter
+ *         Data-Stack Items:
+ *             Top:                      Class Code
+ *             Next:                     Device ID
+ *             3rd:                      Vendor ID
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:    
+ *             pci_data_blk_ob_off       PCI Data Header Block Position
+ *                                           (Offset) in Output Buffer
+ *         Data-Stack, # of Items Popped:  3
+ *         FCode Output buffer:
+ *             Write the PCI Data Header Block:  Fill in the signature,
+ *                 Vendor ID, Device ID and Class Code, and everything
+ *                 else whose value we know.  (Size and Checksum will
+ *                 have to wait until we finish the image...)
+ *         Printout:
+ *             Advisory of Vendor, Device and Class 
+ *
+ *      Error Detection:   (Handled by calling routine)
+ *
+ **************************************************************************** */
+
+static void emit_pci_data_block(void)
+{
+    pci_data_t *pci_data_blk;
+    u32 class_id = dpop();
+    u16 dev_id   = dpop();
+    u16 vend_id  = dpop();
+
+    pci_data_blk_ob_off = opc;
+    pci_data_blk = (pci_data_t *)(ostart + pci_data_blk_ob_off);
+
+    EMIT_STRUCT(pci_data_t);
+
+    BIG_ENDIAN_LONG_STORE(pci_data_blk->signature , PCI_DATA_HDR );
+
+    LITTLE_ENDIAN_WORD_STORE(pci_data_blk->vendor , vend_id );
+    LITTLE_ENDIAN_WORD_STORE(pci_data_blk->device , dev_id );
+    LITTLE_ENDIAN_TRIPLET_STORE(pci_data_blk->class_code , class_id );
+
+    LITTLE_ENDIAN_WORD_STORE(pci_data_blk->dlen ,  sizeof(pci_data_t) );
+
+    pci_data_blk->drevision = PCI_DATA_STRUCT_REV ;
+
+	/* code type = open firmware = 1 */
+    pci_data_blk->code_type = 1;
+
+	/* last image flag */
+    pci_data_blk->last_image_flag = pci_is_last_image ? 0x80 : 0 ;
+
+    tokenization_error(INFO ,
+	"PCI header vendor id=0x%04x, "
+	    "device id=0x%04x, class=%06x\n",
+		vend_id, dev_id, class_id );
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_pcihdr
+ *      Synopsis:       Supervise the writing of PCI Header information
+ *                          into the Output Buffer, when the  PCI-HEADER
+ *                          tokenizer directive has been encountered.
+ *
+ *      Inputs:
+ *         Parameters:                   NONE
+ *         Global Variables:        
+ *             fcode_start_ob_off        Initted if FCode output has begun
+ *             noerrors                  The "Ignore Errors" flag
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:    
+ *         FCode Output buffer:
+ *
+ *      Error Detection:
+ *          An attempt to write a PCI Header after FCode output -- either an
+ *              Fcode-start<n> or ordinary FCode -- has begun is an ERROR.
+ *              Report it; do nothing further, unless "Ignore Errors" is set.
+ *
+ **************************************************************************** */
+
+void emit_pcihdr(void)
+{
+
+    /*  Check for error conditions   */
+    if (
+    /*  FCODE-START<n>  has already been issued  */
+              ( fcode_start_ob_off != -1 )
+    /*  Other FCode has been written             */
+	      ||  fcode_written
+	 )
+    {
+        tokenization_error( TKERROR ,
+	    "Cannot create PCI header after FCode output has begun.\n");
+        if ( ! noerrors ) return ;
    }

-	/* size of image - to be filled later */
-	emit_byte(0x00); emit_byte(0x00);
-	/* revision level */
-	emit_byte(0x00); emit_byte(0x00);
-	/* code type = open firmware */
-	emit_byte(0x01);
-	emit_byte(0x80);
-	/* 2 bytes of padding */
-	emit_byte(0x00); emit_byte(0x00);
+	emit_pci_rom_hdr();

-	return 0;
+	emit_pci_data_block();
 }

+/* **************************************************************************
+ *
+ *      Function name:  finish_pcihdr
+ *      Synopsis:       Fill-in the fields of the PCI Header that could
+ *                      not be determined until the end of the PCI-block.
+ *
+ *************************************************************************** */

-int finish_pcihdr(void)
+void finish_pcihdr(void)
 {
-	u8 *tpc;
-	u32 imgsize=opc-ostart, imgblocks;
+
+	u32 imagesize ;
+	u32 imageblocks;
    int padding;
    
-	if(!pci_hdr)
+	rom_header_t *pci_hdr;
+	pci_data_t   *pci_data_blk;
+
+	if( pci_data_blk_ob_off == -1 )
    {
-		printf("error: trying to fix up unknown pci header\n");
-		return -1;
+	    tokenization_error( TKERROR,
+		"%s without PCI-HEADER\n", strupr(statbuf) );
+	    return ;
    }

+	pci_hdr = (rom_header_t *)(ostart + pci_hdr_ob_off);
+	pci_data_blk = (pci_data_t *)(ostart + pci_data_blk_ob_off);
+
    /* fix up vpd */
-	tpc=opc;
-	opc=pci_hdr+36;
-	emit_byte(pci_vpd&0xff); emit_byte(pci_vpd>>8);
+	LITTLE_ENDIAN_WORD_STORE(pci_data_blk->vpd, pci_vpd);

-	/* fix up image size */
-	opc=pci_hdr+44;
-	imgblocks=(imgsize+511)>>9; /* size is in 512byte blocks */
-	emit_byte(imgblocks&0xff); emit_byte(imgblocks>>8);
+	/*   Calculate image size and padding */
+	imagesize = opc - pci_hdr_ob_off;     /*  Padding includes PCI hdr  */
+	imageblocks = (imagesize + 511) >> 9; /*  Size in 512-byte blocks   */
+	padding = (imageblocks << 9) - imagesize;
+
+	/* fix up image size. */
+	LITTLE_ENDIAN_WORD_STORE(pci_data_blk->ilen, imageblocks);
    
    /* fix up revision */
-	emit_byte(pci_revision&0xff); emit_byte(pci_revision>>8);
+	if ( big_end_pci_image_rev )
+	{
+	    BIG_ENDIAN_WORD_STORE(pci_data_blk->irevision, pci_image_rev);
+	}else{
+	    LITTLE_ENDIAN_WORD_STORE(pci_data_blk->irevision, pci_image_rev);
+	}
    
    /* fix up last image flag */
-	opc++;
-	emit_byte(pci_is_last_image?0x80:0x00);
-	opc=tpc;
+	pci_data_blk->last_image_flag = pci_is_last_image ? 0x80 : 0 ;
    
    /* align to 512bytes */
-	padding=((imgsize+511)&0xfffffe00)-imgsize;
+	
    printf("Adding %d bytes of zero padding to PCI image.\n",padding);
    while (padding--)
    	emit_byte(0);
+	if ( ! pci_is_last_image )
+	{
+	    printf("Note:  PCI header is not last image.\n");
+	}
+	printf("\n");
    
-	pci_hdr=NULL;
-	return 0;
+	pci_hdr_ob_off      = -1;
+	pci_data_blk_ob_off = -1;
 }

void finish_headers(void)
 {
-	if (fcode_hdr) finish_fcodehdr();
-	if (pci_hdr) finish_pcihdr();
+	if (fcode_hdr_ob_off != -1) finish_fcodehdr();
+	if (pci_hdr_ob_off != -1) finish_pcihdr();
 }

Modified: fcode-utils/toke/emit.h
===================================================================
--- fcode-utils/toke/emit.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/emit.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _TOKE_EMIT_H
+#define _TOKE_EMIT_H
+
 /*
  *                     OpenBIOS - free your system! 
  *                         ( FCode tokenizer )
@@ -24,21 +27,90 @@
  *
  */

-#ifndef _H_EMIT
-#define _H_EMIT
+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+ 
+#include "types.h"

-int emit_byte(u8 data);
-int emit_fcode(u16 tok);
-int emit_token(const char *name);
-int emit_num32(u32 num);
-int emit_num16(u16 num);
-int emit_offset(s16 offs);
-s16 receive_offset(void);
-int emit_string(u8 *string, unsigned int cnt);
-int emit_fcodehdr(void);
-int finish_fcodehdr(void);
-int emit_pcihdr(u16 vid, u16 did, u32 classid);
-int finish_pcihdr(void);
+/* **************************************************************************
+ *          Structure Name:    fcode_header_t
+ *          Synopsis:          FCode Header within the Output Buffer
+ *                            
+ *   Fields:
+ *       format, checksum and length    All as prescribed in P1275, Sec 5.2.2.5
+ *       
+ *       Note that the Checksum and Length fields are stated as byte-arrays
+ *           rather than as integers.  This is intended to guarantee that
+ *           their endian-ness will remain independent of the endian-ness
+ *           of the host-platform on which this Tokenizer is running.
+ *       Note also that, as integers, both are BIG-Endian.      
+ *
+ **************************************************************************** */
+
+typedef struct {
+    u8  format;
+    u8  checksum[2];  /* [0] = Hi byte, [1] = Lo */
+    u8  length[4];    /* [0] = Hi, [1] = Hi-mid, [2] = Lo-mid, [3] = Lo  */
+} fcode_header_t;
+
+
+/* **************************************************************************
+ *          Macro Name:   STRING_LEN_MAX
+ *                        Max number of bytes allowable in an output string
+ *                            
+ *   Arguments:           NONE
+ *
+ *       This value must remail hard-coded and immutable.  It represents the
+ *           maximum number allowed in a FORTH counted-string's count-byte
+ *           (which is, of course, the maximum number that can be expressed
+ *           in an unsigned byte).
+ *       It should not be used for anything else (e.g., buffer sizes), and
+ *           most especially not for anything that might be changed!
+ *
+ **************************************************************************** */
+
+#define STRING_LEN_MAX    255
+
+
+/* **************************************************************************
+ *          Macro Name:   GET_BUF_MAX
+ *                        Size alloted to input-string buffer
+ *                            
+ *   Arguments:           NONE
+ *
+ *       This is a generous allotment for the buffer into which
+ *           input strings are gathered.  Overflow calculations are
+ *           also based on it.  It may be changed safely.
+ *       We like to keep it a nice power-of-two to make the memory-
+ *           allocation routine run efficiently and happily (Okay, so
+ *           that's anthropormism:  It's efficient and *we*'re happy.
+ *           Better?  I thought so...  ;-)
+ *
+ **************************************************************************** */
+
+#define GET_BUF_MAX    1024
+
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+void  emit_fcode(u16 tok);
+void  user_emit_byte(u8 data);
+
+void  emit_offset(s16 offs);
+void  emit_string(u8 *string, signed int cnt);
+void  emit_fcodehdr(const char *starter_name);
+void  finish_fcodehdr(void);
+void  emit_pcihdr(void);
+void  finish_pcihdr(void);
 void finish_headers(void);
+void  emit_end0(void);
+void  emit_literal(u32 num);

-#endif   /* _H_EMIT */
+#endif   /* _TOKE_EMIT_H */

Added: fcode-utils/toke/errhandler.c
===================================================================
--- fcode-utils/toke/errhandler.c	                        (rev 0)
+++ fcode-utils/toke/errhandler.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,956 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Error-Handler for Tokenizer
+ *
+ *      Controls printing of various classes of errors
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          init_error_handler  Initialize the error-counts,
+ *                                  announce the file names.
+ *          tokenization_error  Handle an error of the given class,
+ *                                  print the given message in the 
+ *                                  standard format.
+ *          started_at          Supplemental message, giving a back-reference
+ *                                  to the "starting"  point of a compound
+ *                                  error, including last-colon identification.
+ *          just_started_at     Supplemental back-reference to "starting"  point
+ *                                  of compound error, but without last-colon
+ *                                  identification.
+ *          where_started       Supplemental message, giving a more terse back-
+ *                                  -reference to "start" of compound-error.
+ *          just_where_started Supplemental message, more terse back-reference,
+ *                                  without last-colon identification.
+ *          in_last_colon      Supplemental back-reference message,
+ *                                  identifying last Colon-definition.
+ *          safe_malloc         malloc with built-in failure test.
+ *          error_summary       Summarize final error-message status
+ *                                  before completing tokenization.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *          We will define a set of bit-valued error-types and a
+ *          global bit-mask.  Each error-message will be associated
+ *          with one of the bit-valued error-types.  The bit-mask,
+ *          which will be set by a combination of defaults and user
+ *          inputs (mainly command-line arguments), will control
+ *          whether an error-message of any given type is printed.
+ *          
+ *          Another bit-mask variable will accumulate the error-
+ *          types that occur within any given run; at the end of
+ *          the run, it will be examined to determine if the run
+ *          failed, i.e., if the output should be suppressed.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          Error-types fall into the following broad categories:
+ *              FATAL           Cause to immediately stop activity
+ *              TKERROR         Sufficient to make the run a failure,
+ *                                  but not to stop activity.
+ *              WARNING         Not necessarily an error, but something
+ *                                  to avoid.  E.g., it might rely on
+ *                                  assumptions that are not necessarily
+ *                                  what the user/programmer wants.  Or:
+ *                                  It's a deprecated feature, or one
+ *                                  that might be incompatible with
+ *                                  other standard tokenizers.
+ *              INFO            Nothing is changed in processing, but
+ *                                  an advisory is still in order.
+ *              MESSAGE         Message generated by the user.  (Complete;
+ *                                  new-line will be added by display routine.)
+ *              P_MESSAGE       Partial Message -- Instigated by user, but
+ *                                  pre-formatted and not complete.  New-line
+ *                                  will be added by follow-up routine.
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "types.h"
+#include "toke.h"
+#include "stream.h"
+#include "errhandler.h"
+#include "scanner.h"
+
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              iname           Name of file currently being processed
+ *              lineno          Current line-number being processed
+ *              noerrors        "Ignore Errors" flag, set by "-i" switch
+ *              opc             FCode Output Buffer Position Counter
+ *              verbose         If true, enable Advisory Messages
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *              Internal Static Variables
+ *          print_msg               Whether beginning of a message was printed;
+ *                                      therefore, whether to print the rest.
+ *          errs_to_print           Error Verbosity Mask.  Bits set correspond
+ *                                      to message-types that will be printed
+ *                                      May be altered by Command-Line switches.
+ *          err_types_found         Accumulated Error-types.  Bits
+ *                                      set correspond to error-types
+ *                                      that have occurred.
+ *          message_dest            Message Dest'n.  Usually ERRMSG_DESTINATION
+ *                                      (stdout) except when we need to switch.
+ *          err_count               Count of Error Messages
+ *          warn_count              Count of Warning Messages
+ *          info_count              Count of "Advisory" Messages
+ *          user_msg_count          Count of User-generated Messages
+ *          fatal_err_exit          Exit code to be used for "Fatal" error.
+ *                                       This is a special accommodation
+ *                                       for the  safe_malloc  routine.
+ *
+ **************************************************************************** */
+
+static bool  print_msg ;
+static int errs_to_print = ( FATAL | TKERROR | WARNING | 
+                             MESSAGE | P_MESSAGE | FORCE_MSG ) ;
+static int err_types_found =  0 ;
+static int err_count       =  0 ;
+static int warn_count      =  0 ;
+static int info_count      =  0 ;
+static int user_msg_count  =  0 ;
+static int fatal_err_exit  = -1 ;
+static FILE *message_dest;     /*  Would like to init to  ERRMSG_DESTINATION
+				*      here, but the compiler complains...
+				*/
+
+/* **************************************************************************
+ *
+ *              Internal Static Constant Structure
+ *          err_category            Correlate each error-type code with its
+ *                                      Counter-variable and the printable
+ *                                      form of its name.
+ *          num_categories          Number of entries in the err_category table
+ *
+ **************************************************************************** */
+
+typedef struct {
+    int  type_bit ;		/*  Error-type single-bit code        */
+    char *category_name ;	/*  Printable-name base               */
+    char *single ;		/*  Suffix to print singular of name  */
+    char *plural ;		/*  Suffix to print plural of name    */
+    int  *counter ;		/*  Associated Counter-variable       */
+    bool new_line ;		/*  Whether to print new-line at end  */
+} err_category ;
+
+static const err_category  error_categories[] = {
+    /*  FATAL  must be the first entry in the table.   */
+    /*  No plural is needed; only one is allowed....   */
+    { FATAL,    "Fatal Error", "", "",     &err_count      , TRUE  },
+
+    { TKERROR,    "Error"     , "", "s",    &err_count      , FALSE },
+    { WARNING,    "Warning"   , "", "s",    &warn_count     , FALSE },
+    { INFO,       "Advisor"   , "y", "ies", &info_count     , FALSE },
+    { MESSAGE ,   "Message"   , "", "s",    &user_msg_count , TRUE  },
+    { P_MESSAGE , "Message"   , "", "s",    &user_msg_count , FALSE }
+};
+
+static const int num_categories =
+    ( sizeof(error_categories) / sizeof(err_category) );
+
+
+#ifdef NEEDS_STRUPR
+
+/* **************************************************************************
+ *
+ *      Function name:  toup
+ *      Synopsis:       Support function for  strupper
+ *                      Converts one character
+ *
+ *      Inputs:
+ *         Parameters:
+ *             chr_ptr                 Pointer to the character
+ *
+ *      Outputs:
+ *         Returned Value:             None
+ *         Supplied Pointers:
+ *             The character pointed to is changed
+ *
+ *      Process Explanation:
+ *          Because this fills in a lack in the host system, we cannot
+ *              rely on the functions  islower  or  toupper , which are
+ *              usually built-in but might be similarly missing.
+ *
+ **************************************************************************** */
+
+static void toup( char *chr_ptr)
+{
+    const unsigned char upcas_diff = ( 'a' - 'A' );
+    if ( ( *chr_ptr >= 'a' ) && ( *chr_ptr <= 'z' ) )
+    {
+	*chr_ptr -= upcas_diff ;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  strupper
+ *      Synopsis:       Replacement for  strupr  on systems that don't 
+ *                      seem to have it.  A necessary hack.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             strung              Pointer to the string to be changed
+ *
+ *      Outputs:
+ *         Returned Value:         Same pointer that was passed in
+ *         Supplied Pointers:
+ *             The string pointed to will be converted to upper case
+ *
+ *      Process Explanation:
+ *          Because it fills in a lack in the host system, this routine
+ *              does not rely on the functions  islower  or  toupper
+ *              which are usually built-in but might be missing.
+ *
+ **************************************************************************** */
+
+char *strupper( char *strung)
+{
+    char *strindx;
+    for (strindx = strung; *strindx != 0; strindx++)
+    {
+        toup( strindx);
+    }
+    return strung;
+}
+
+/* **************************************************************************
+ *
+ *     If  strupr  is missing, it's a good bet that so is  strlwr 
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Function name:  tolow
+ *      Synopsis:       Support function for  strlower
+ *                      Converts one character
+ *
+ *      Inputs:
+ *         Parameters:
+ *             chr_ptr                 Pointer to the character
+ *
+ *      Outputs:
+ *         Returned Value:             None
+ *         Supplied Pointers:
+ *             The character pointed to is changed
+ *
+ *      Process Explanation:
+ *          Because this fills in a lack in the host system, we cannot
+ *              rely on the functions  isupper  or  tolower , which are
+ *              usually built-in but might be similarly missing.
+ *
+ **************************************************************************** */
+
+static void tolow( char *chr_ptr)
+{
+    const unsigned char lowcas_diff = ( 'A' - 'a' );
+    if ( ( *chr_ptr >= 'A' ) && ( *chr_ptr <= 'Z' ) )
+    {
+	*chr_ptr -= lowcas_diff ;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  strlower
+ *      Synopsis:       Replacement for  strlwr  on systems that don't 
+ *                      seem to have it.  A necessary hack.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             strung              Pointer to the string to be changed
+ *
+ *      Outputs:
+ *         Returned Value:         Same pointer that was passed in
+ *         Supplied Pointers:
+ *             The string pointed to will be converted to lower case
+ *
+ *      Process Explanation:
+ *          Because it fills in a lack in the host system, this routine
+ *              does not rely on the functions  isupper  or  tolower
+ *              which are usually built-in but might be missing.
+ *
+ **************************************************************************** */
+
+char *strlower( char *strung)
+{
+    char *strindx;
+    for (strindx = strung; *strindx != 0; strindx++)
+    {
+        tolow( strindx);
+    }
+    return strung;
+}
+
+
+#endif  /*   NEEDS_STRUPR   */
+ 
+/* **************************************************************************
+ *
+ *      Function name:  init_error_handler
+ *      Synopsis:       Initialize the error-handler before starting a
+ *                          new tokenization; both the aspects that will
+ *                          persist across the entire run and those that
+ *                          need to be reset, such as error-counts.
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Global Variables: 
+ *              verbose                Set by "-v" switch
+ *         Macro:
+ *             ERRMSG_DESTINATION      Error message destination;
+ *                                         (Set by development-time switch)
+ *             FFLUSH_STDOUT           Flush STDOUT if err-msg-dest is STDERR
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Global Variables:
+ *             errs_to_print           Add the INFO bit if verbose is set
+ *         Local Static Variables:
+ *             message_dest            Point it at ERRMSG_DESTINATION (stderr)
+ *           Reset the following to zero:
+ *             err_types_found         Accumulated Error-types.
+ *             err_count               Count of Error Messages
+ *             warn_count              Count of Warning Messages
+ *             info_count              Count of "Advisory" Messages
+ *             user_msg_count          Count of User-generated Messages
+ *         Other Exotic Effects:
+ *             Flush stdout if Error message destination is not stdout, to
+ *                 avoid collisions with stderr once Error Messaging begins.
+ *
+ *      Extraneous Remarks:
+ *          This needs to be done before attempting to read the input file,
+ *              so that any Messages that occur there can be properly counted.
+ *
+ **************************************************************************** */
+
+void init_error_handler( void)
+{
+    message_dest  =  ERRMSG_DESTINATION;
+    if ( verbose )  errs_to_print |= INFO ;
+    err_types_found = 0 ;
+    err_count = 0 ;
+    warn_count = 0 ;
+    info_count = 0 ;
+    user_msg_count = 0 ;
+    FFLUSH_STDOUT
+}
+
+/* **************************************************************************
+ *
+ *      Function name:    tokenization_error
+ *      Synopsis:         Handle an error of the given class,
+ *                            print the given message in the standard format.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             err_type       int        One of the bit-valued error-types
+ *             The remaining parameters are a format string and corresponding
+ *                 data objects such as would be sent to  printf() 
+ *         Global Variables:
+ *             errs_to_print        Error Verbosity Mask.
+ *             iname                Name of file currently being processed
+ *             lineno               Current line-number being processed
+ *             fatal_err_exit       Exit code for "Fatal" error, if applicable.
+ *         Macro:
+ *             ERRMSG_DESTINATION        Error message destination;
+ *                                           (Development-time switch)
+ *         Note:  Whether this routine will or will not supply a new-line
+ *             at the end of the printout depends on the category of the
+ *             message.  The new-line is included for a FATAL or a User-
+ *             Generated Message, and excluded for the rest.  For those,
+ *             the calling routine must be responsible for including a
+ *             new-line at the end of the format string or for otherwise
+ *             finishing the line, as by calling started_at()
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Local Static Variables:
+ *             err_types_found             Accumulated Error-types.
+ *             print_msg                   Whether this message was printed;
+ *                                             may be used by started_at()
+ *                    One of the following Category Counters
+ *                         will be incremented, as applicable:
+ *             err_count
+ *             warn_count
+ *             info_count 
+ *             user_msg_count
+ *         Printout:    Directed to  stdout or stderr 
+ *                          (see definition of ERRMSG_DESTINATION)
+ *
+ *      Error Detection:
+ *              Err_type not in list
+ *                      Print special message; treat cause as an Error.
+ *                      Force printout.
+ *
+ *      Process Explanation:
+ *          Accumulated the Error-type into  err_types_found 
+ *          Identify the Error-Category:
+ *              Check the Error-Type against the bit-code.
+ *                  The Error-type may have more than one bit set,
+ *                  but if it matches the Category bit-code, it's it.
+ *              If it doesn't match any Error-Category bit-code, print
+ *                  a special message and treat it as an ERROR code.
+ *          Check the Error-Type against the Error Verbosity Mask;
+ *          If it has a bit set, print the Error-Category, together
+ *                  with the source-file name and line number, and
+ *                  the rest of the message as supplied.
+ *              The table that translates the Error-type into a printable
+ *                  Error-Category string also identifies the applicable
+ *                  Category Counter; increment it.
+ *          Of course, there's no return from a FATAL error; it exits.
+ *
+ **************************************************************************** */
+
+void tokenization_error( int err_type, char* msg, ... )
+{
+    int indx ;
+
+    /*  Initial settings:  treat as an Error.  */
+    char *catgy_name = "Error";
+    char *catgy_suffx = "";
+    int *catgy_counter = &err_count;
+    bool print_new_line = FALSE;
+
+    /*  Accumulated the Error-type into  err_types_found  */
+    err_types_found |= err_type;
+
+    /*  Identify the Error-Category.  */
+    for ( indx = 0 ; indx < num_categories ; indx ++ )
+    {
+        if ( ( error_categories[indx].type_bit & err_type ) != 0 )
+        {
+            catgy_name = error_categories[indx].category_name;
+            catgy_suffx = error_categories[indx].single;
+            catgy_counter = error_categories[indx].counter;
+	    print_new_line = error_categories[indx].new_line;
+            break;
+        }
+    }
+
+    /*  Special message if  err_type  not in list; treat as an Error.  */
+    if ( catgy_name == NULL )
+    {
+         fprintf(ERRMSG_DESTINATION,
+	      "Program error: Unknown Error-Type, 0x%08x.  "
+              "  Will treat as Error.\n", err_type) ;
+         err_types_found |= TKERROR;
+         print_msg = TRUE ;
+    } else {
+         /*  Check the Error-Type against the Error Verbosity Mask  */
+         print_msg = BOOLVAL( ( errs_to_print & err_type ) != 0 );
+    }
+
+    if ( print_msg )
+    {
+        va_list argp;
+
+        if ( iname != NULL )
+	{
+	    fprintf(ERRMSG_DESTINATION, "%s%s:  File %s, Line %d.  ",
+        	 catgy_name, catgy_suffx, iname, lineno);
+	}else{
+	    /*  Don't print iname or lineno if no file opened.  */
+	    fprintf(ERRMSG_DESTINATION, "%s%s:  ",
+        	 catgy_name, catgy_suffx);
+	}
+
+        va_start(argp, msg);
+        vfprintf(ERRMSG_DESTINATION, msg, argp);
+        va_end(argp);
+	if ( print_new_line ) fprintf(ERRMSG_DESTINATION, "\n");
+
+	/*   Increment the category-counter.  */
+	*catgy_counter += 1;
+    }
+    if ( err_type == FATAL )
+    {
+        fprintf(ERRMSG_DESTINATION, "Tokenization terminating.\n");
+        error_summary();
+        exit ( fatal_err_exit );
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  print_where_started
+ *      Synopsis:       Supplemental message, following a tokenization_error,
+ *                          giving a back-reference to the "start" point of
+ *                          the compound-error being reported.
+ *                      This is a retro-fit; it does the heavy lifting for
+ *                          the routines  started_at() ,  just_started_at() , 
+ *                           where_started() ,  just_where_started() and
+ *                           in_last_colon() .
+ *
+ *      Inputs:
+ *         Parameters:
+ *             show_started         Whether to print a phrase about "started"
+ *             show_that_st         Whether to print "that started" as opposed
+ *                                      to " , which started"
+ *             saved_ifile          File-name saved for "back-reference"
+ *             saved_lineno         Line-number saved for "back-reference"
+ *             may_show_incolon     Whether to allow a call to  in_last_colon()
+ *                                      Needed to prevent infinite recursion...
+ *         Global Variables:        
+ *             iname                Name of file currently being processed
+ *             lineno               Current line-number being processed
+ *         Local Static Variables:
+ *             print_msg            Whether the beginning part of the message
+ *                                      was printed by tokenization_error()
+ *             message_dest         Message Destination. Is ERRMSG_DESTINATION
+ *                                      (stdout) usually, except sometimes...
+ *
+ *      Outputs:
+ *         Returned Value:          None
+ *         Printout:
+ *             The remainder of a message:  the location of a back-reference.
+ *                 The phrase "that started" is switchable.  This routine
+ *                 will supply the leading space and a new-line; the routines
+ *                 that call this can be used to finish the line.
+ *
+ *      Process Explanation:
+ *          This routine is called immediately after tokenization_error()
+ *              If tokenization_error() didn't print, neither will we.
+ *              The residual state of  print_msg  will tell us that.
+ *          If the preceding message ended with something general about a
+ *              "Colon Definition" or "Device-Node" or the like, we want
+ *              the message to read:  "that started on line ... [in file ...]"
+ *          If the end of the preceding message was something more specific,
+ *              we just want the message to read:  "on line ... [in file ...]"
+ *          If the saved input file name doesn't match our current input
+ *              file name, we will print it and the saved line-number.
+ *          If the file name hasn't changed, we will print only the saved
+ *              line-number.
+ *          If neither is changed, there's no point in printing any of the
+ *              above-mentioned text.    
+ *          If a Colon-definition is in progress, show its name and the
+ *              line on which it started.  Protect against infinite loop!
+ *          End the line.
+ *
+ *      Extraneous Remarks:
+ *          This is a retrofit.  Earlier, it was just  started_at() .  Later,
+ *              I generated more specific messages, and needed a way to leave
+ *              out the "that started".  I could, theoretically, have added
+ *              the extra parameter to  started_at() , but by now there are
+ *              so many of calls to it that I'd rather leave them as is, and
+ *              just change the name of the routine in the few places that
+ *              need the terser form of the message.
+ *
+ **************************************************************************** */
+
+static void print_where_started( bool show_started,
+                                   bool show_that_st,
+				   char * saved_ifile,
+				       unsigned int saved_lineno,
+                                           bool may_show_incolon)
+{
+    if ( print_msg )
+    {
+	bool fil_is_diff;
+	bool lin_is_diff;
+
+	/*  File names are case-sensitive  */
+	fil_is_diff = BOOLVAL(strcmp(saved_ifile, iname) != 0 );
+	lin_is_diff = BOOLVAL(saved_lineno != lineno );
+	if ( fil_is_diff || lin_is_diff )
+	{
+	    if ( show_started )
+	    {
+		if ( show_that_st )
+		{
+		    fprintf(message_dest, " that");
+		}else{
+		    fprintf(message_dest, " , which");
+		}
+		fprintf(message_dest, " started");
+	    }
+	    fprintf(message_dest, " on line %d", saved_lineno);
+	    if ( fil_is_diff )
+	    {
+	        fprintf(message_dest, " of file %s", saved_ifile);
+	    }
+	}
+
+	if ( may_show_incolon )
+	{
+	    in_last_colon();
+	}else{
+	    fprintf(message_dest, "\n");
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  started_at
+ *      Synopsis:       Supplemental back-reference message,
+ *                          with the "that started"  phrase,
+ *                          and with last-colon identification.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             saved_ifile          File-name saved for "back-reference"
+ *             saved_lineno         Line-number saved for "back-reference"
+ *
+ *      Outputs:
+ *         Returned Value:          None
+ *         Global Variables:
+ *         Printout:
+ *             The "...started at..." remainder of a message, giving a back-
+ *                 -reference to the  "start" point supplied in the params,
+ *                 and the start of the current Colon-definition if one is
+ *                 in effect.
+ *             Will supply a new-line and can be used to finish the line.
+ *
+ **************************************************************************** */
+
+void started_at( char * saved_ifile, unsigned int saved_lineno)
+{
+    print_where_started( TRUE, TRUE, saved_ifile, saved_lineno, TRUE);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  print_started_at
+ *      Synopsis:       Same as started_at() except output will be directed
+ *                          to  stdout  instead of to ERRMSG_DESTINATION
+ *
+ *      Extraneous Remarks:
+ *          A retrofit.  Can you tell?
+ *
+ **************************************************************************** */
+ 
+void print_started_at( char * saved_ifile, unsigned int saved_lineno)
+{
+    message_dest = stdout;
+	started_at( saved_ifile, saved_lineno);
+    message_dest = ERRMSG_DESTINATION;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  just_started_at
+ *      Synopsis:       Supplemental back-reference message,
+ *                          with the "that started"  phrase,
+ *                          but without last-colon identification.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             saved_ifile          File-name saved for "back-reference"
+ *             saved_lineno         Line-number saved for "back-reference"
+ *
+ *      Outputs:
+ *         Returned Value:          None
+ *         Global Variables:
+ *         Printout:
+ *             The "...started at..." remainder of a message, giving a back-
+ *                 -reference to the  "start" point supplied in the params,
+ *                 and no more.
+ *             Will supply a new-line and can be used to finish the line.
+ *
+ **************************************************************************** */
+
+void just_started_at( char * saved_ifile, unsigned int saved_lineno)
+{
+    print_where_started( TRUE, TRUE, saved_ifile, saved_lineno, FALSE);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  where_started
+ *      Synopsis:       Supplemental back-reference message,
+ *                          without the "that started"  phrase,
+ *                          but with last-colon identification.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             saved_ifile          File-name saved for "back-reference"
+ *             saved_lineno         Line-number saved for "back-reference"
+ *
+ *      Outputs:
+ *         Returned Value:          None
+ *         Global Variables:
+ *         Printout:
+ *             The remainder of a message, giving a back-reference to the
+ *                 "start" point supplied in the parameters, and the start
+ *                 of the current Colon-definition if one is in effect.
+ *             Will supply a new-line and can be used to finish the line.
+ *
+ **************************************************************************** */
+
+void where_started( char * saved_ifile, unsigned int saved_lineno)
+{
+    print_where_started( FALSE, FALSE, saved_ifile, saved_lineno, TRUE);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  just_where_started
+ *      Synopsis:       Supplemental back-reference message,
+ *                          without the "that started"  phrase,
+ *                          and without last-colon identification.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             saved_ifile          File-name saved for "back-reference"
+ *             saved_lineno         Line-number saved for "back-reference"
+ *
+ *      Outputs:
+ *         Returned Value:          None
+ *         Global Variables:
+ *         Printout:
+ *             The remainder of a message, giving a back-reference to the
+ *                 "start" point supplied in the parameters, and no more.
+ *             Will supply a new-line and can be used to finish the line.
+ *
+ **************************************************************************** */
+
+void just_where_started( char * saved_ifile, unsigned int saved_lineno)
+{
+    print_where_started( FALSE, FALSE, saved_ifile, saved_lineno, FALSE);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  in_last_colon
+ *      Synopsis:       Supplemental back-reference message, identifying
+ *                          last Colon-definition if one is in effect.
+ *                      Can be used to finish the line in either case.
+ *
+ *      Inputs:
+ *         Parameters:                   NONE
+ *         Global Variables:
+ *             incolon                   TRUE if Colon-definition is in progress
+ *             last_colon_defname        Name of last colon-definition
+ *             last_colon_filename       File where last colon-def'n made
+ *             last_colon_lineno         Line number of last colon-def'n
+ *         Local Static Variables:
+ *             print_msg            Whether the beginning part of the message
+ *                                      was printed by tokenization_error()
+ *             message_dest         Message Destination. Is ERRMSG_DESTINATION
+ *                                      (stdout) usually, except sometimes...
+ *
+ *      Outputs:
+ *         Returned Value:                  NONE
+ *         Printout:
+ *             Remainder of a message:
+ *                "in definition of  ... , which started ..."
+ *
+ *      Process Explanation:
+ *          Because this routine does some of its own printing, it needs
+ *              to check the residual state of  print_msg  first.
+ *          The calling routine does not need to test   incolon ; it can
+ *              call this to end the line in either case.
+ *
+ **************************************************************************** */
+
+void in_last_colon( void )
+{
+    if ( print_msg )
+    {
+	if ( incolon )
+	{
+	    fprintf( message_dest, " in definition of  %s ",
+		strupr( last_colon_defname) );
+	    print_where_started( TRUE, FALSE,
+		last_colon_filename, last_colon_lineno, FALSE);
+	}else{
+	    fprintf(message_dest, "\n");
+	}
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  safe_malloc
+ *      Synopsis:       malloc with built-in failure test.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             size       size_t     Size of memory-chunk to allocate
+ *             phrase     char *     Phrase to print after "... while "
+ *                                       in case of failure.
+ *
+ *      Outputs:
+ *         Returned Value:           Pointer to allocated memory
+ *         Global Variables:
+ *             fatal_err_exit       On memory allocation failure, change
+ *                                       to a special system-defined value
+ *
+ *      Error Detection:
+ *          On memory allocation failure, declare a FATAL error.  Set up
+ *              for a special system-defined EXIT value that indicates
+ *              insufficient memory.
+ *
+ *      Process Explanation:
+ *          It is the responsibility of the calling routine to be sure
+ *              the "phrase" is unique within the program.  It is intended
+ *              as a debugging aid, to help localize the point of failure.
+ *
+ **************************************************************************** */
+
+_PTR safe_malloc( size_t size, char *phrase)
+{
+    _PTR retval ;
+    retval = malloc (size);
+    if ( !retval )
+    {
+        fatal_err_exit = -ENOMEM ;
+        tokenization_error( FATAL, "Out of memory while %s.", phrase);
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:         error_summary
+ *      Synopsis:              Summarize final error-message status
+ *                                 before completing tokenization.
+ *                             Indicate if OK to produce output.
+ *      
+ *      Inputs:
+ *         Parameters:                   NONE
+ *         Global Variables:        
+ *             noerrors             "Ignore Errors" flag, set by "-i" switch
+ *             err_types_found      Accumulated Error-types.
+ *             error_categories     Table of Error-types, Message-Counters
+ *                                      and their printable names.
+ *             opc                  FCode Output Buffer Position Counter
+ *                                      (zero means there was no output).
+ *
+ *      Outputs:
+ *         Returned Value:          True = OK to produce output (But caller
+ *                                      must still verify non-zero opc)
+ *         Printout:
+ *             Various messages.
+ *
+ *      Process Explanation:
+ *          The first entry in the error_categories table is FATAL    
+ *              We won't need to print a tally of that...
+ *      
+ **************************************************************************** */
+
+bool error_summary( void )
+{
+    /*  Bit-mask of error-types that require suppressing output   */
+    static const int suppress_mask = ( FATAL | TKERROR );
+    bool retval = TRUE;
+    bool suppressing = FALSE;
+
+    /*  There's no escaping a FATAL error   */
+    if ( ( err_types_found & FATAL ) != 0 )
+    {
+	/*   FATAL error.  Don't even bother with the tally.   */
+	suppressing = TRUE;
+    } else {
+
+	if ( opc == 0 )
+	{
+	    printf ( "Nothing Tokenized");
+	}else{
+	    printf ( "Tokenization Completed");
+	}
+
+	if ( err_types_found != 0 )
+	{
+	    int indx;
+	    bool tally_started = FALSE ;
+	    printf (". ");
+	    /*
+	     *  Print a tally of the error-types;
+	     *  handle plurals and punctuation appropriately.
+	     */
+	    /*  Start at indx = 1 to skip examining FATALs   */
+	    for ( indx = 1; indx < num_categories ; indx ++ )
+	    {
+		if ( *(error_categories[indx].counter) > 0 )
+		{
+		    printf ("%s %d %s%s",
+	        	tally_started ? "," : "" ,
+			    *(error_categories[indx].counter),
+				error_categories[indx].category_name,
+				    *(error_categories[indx].counter) > 1 ?
+					 error_categories[indx].plural :
+					     error_categories[indx].single );
+		    /*  Zero out the counter, to prevent displaying the
+		     *      number of Messages twice, since it's shared
+		     *      by the "Messages" and "P_Messages" categories.
+		     */
+		    *(error_categories[indx].counter) = 0;
+		    tally_started = TRUE;
+		}
+	    }
+	}
+        printf (".\n");
+
+	if ( ( err_types_found & suppress_mask ) != 0 )
+	{    /*  Errors found.  Not  OK to produce output    */
+             /*  Unless "Ignore Errors" flag set...          */
+	    if ( INVERSE(noerrors) )
+            {
+		suppressing = TRUE;
+            }else{
+		if ( opc > 0 )
+		{
+		    printf ("Error-detection over-ridden; "
+				"producing binary output.\n");
+		}
+            }
+	}
+    }
+    if ( suppressing )
+    {
+	retval = FALSE ;
+	printf ("Suppressing binary output.\n");
+    }
+    return ( retval );
+}
+

Added: fcode-utils/toke/errhandler.h
===================================================================
--- fcode-utils/toke/errhandler.h	                        (rev 0)
+++ fcode-utils/toke/errhandler.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,119 @@
+#ifndef _TOKE_ERRHANDLER_H
+#define _TOKE_ERRHANDLER_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Function Prototypes for Tokenizer Error-Handler
+ *
+ *      Defines symbols for the various classes of errors
+ *          for the Error-Handler and for all its users
+ *
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "types.h"
+
+#define  FATAL       0x80000000
+#define  TKERROR     0x04000000
+#define  WARNING     0x00200000
+#define  INFO        0x00010000
+#define  MESSAGE     0x00000800
+#define  P_MESSAGE   0x00000040
+#define  FORCE_MSG   0x00000001
+
+void init_error_handler( void);
+void tokenization_error( int err_type, char* msg, ... );
+void started_at( char * saved_ifile, unsigned int saved_lineno);
+void print_started_at( char * saved_ifile, unsigned int saved_lineno);
+void just_started_at( char * saved_ifile, unsigned int saved_lineno);
+void where_started( char * saved_ifile, unsigned int saved_lineno);
+void just_where_started( char * saved_ifile, unsigned int saved_lineno);
+void in_last_colon( void );
+_PTR safe_malloc( size_t size, char *phrase);
+bool error_summary( void );   /*  Return TRUE if OK to produce output. */
+
+
+/* **************************************************************************
+ *
+ *      Macros:
+ *          ERRMSG_DESTINATION      During development, I used this to switch
+ *               error message destination between STDOUT and STDERR, until I
+ *               settled on which is preferable.  Recently, I have proven to
+ *               my satisfaction that STDERR is preferable:  error-messages
+ *               produced by a sub-shell will be correctly synchronized with
+ *               the error-messages we produce.  When I tested using STDOUT
+ *               for error-messages, that error-case looked garbled.
+ *          FFLUSH_STDOUT           fflush( stdout) if error message destination
+ *              is STDERR, No-op if it's STDOUT.  A few of these, judiciously
+ *              placed, kept our own regular and error messages nicely in sync.
+ *
+ **************************************************************************** */
+
+#define ERRMSG_DESTINATION stderr
+#define FFLUSH_STDOUT  fflush( stdout);
+
+/*  We're no longer switching the above.
+ *  The below is left here to show what had been done formerly.
+ */
+#if -1     /*  Switchable error-message destination  */ 
+#else      /*  Switchable error-message destination  */
+#define ERRMSG_DESTINATION stdout
+#define FFLUSH_STDOUT  /*  Don't need to do anything here  */
+#endif     /*  Switchable error-message destination  */
+
+/*  Some systems don't seem to have  strupr   */
+#ifdef SYS_IS_GNU_Linux
+#define NEEDS_STRUPR
+#endif  /*   SYS_IS_GNU_Linux   */
+#ifdef SYS_IS_AIX
+#define NEEDS_STRUPR
+#endif  /*   SYS_IS_AIX   */
+/*  ???  Should this instead be tuned to Proc'r is PPC ???  Why? */
+
+#ifdef NEEDS_STRUPR
+
+/* **************************************************************************
+ *
+ *     A necessary hack for systems that don't seem
+ *         to have  strupr  and  strlwr
+ *     Let's avoid a naming conflict, just in case... 
+ *
+ **************************************************************************** */
+
+extern char *strupper( char *strung);
+#define strupr strupper
+
+extern char *strlower( char *strung);
+#define strlwr strlower
+
+
+#endif  /*   NEEDS_STRUPR   */
+
+#endif  /*   _TOKE_ERRHANDLER_H   */

Added: fcode-utils/toke/flowcontrol.c
===================================================================
--- fcode-utils/toke/flowcontrol.c	                        (rev 0)
+++ fcode-utils/toke/flowcontrol.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,2080 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Support Functions for tokenizing FORTH Flow-Control structures.
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *               These first two do their work after the calling routine
+ *                        has written the token for the required variant:
+ *
+ *          mark_do                        Mark branches for "do" variants
+ *          resolve_loop                   Resolve "loop" variants' branches
+ *
+ *               The remaining routines' descriptions are all similar:
+ *                        Write the token(s), handle the outputs, mark
+ *                        or resolve the branches, and verify correct
+ *                        control-structure matching, for tokenizing
+ *                        the ........................ statement in FORTH
+ *          emit_if                          IF
+ *          emit_else                        ELSE
+ *          emit_then                        THEN
+ *          emit_begin                       BEGIN
+ *          emit_again                       AGAIN
+ *          emit_until                       UNTIL
+ *          emit_while                       WHILE
+ *          emit_repeat                      REPEAT
+ *          emit_case                        CASE
+ *          emit_of                          OF
+ *          emit_endof                       ENDOF
+ *          emit_endcase                     ENDCASE
+ *
+ *      Three additional routines deal with matters of overall balance
+ *      of the Control-Structures, and identify the start of any that
+ *      were not balanced.  The first just displays Messages:
+ *
+ *          announce_control_structs
+ *
+ *      The other two clear and re-balance them:
+ *
+ *          clear_control_structs_to_limit
+ *          clear_control_structs
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Correct analysis of Return-Stack usage around Flow-Control
+ *              constructs, including within Do-Loops or before Loop
+ *              Elements like I and J or UNLOOP or LEAVE.
+ *          Similarly, Return-Stack usage around IF ... ELSE ... THEN
+ *              statements needs analysis.  For instance, the following:
+ * 
+ *          blablabla >R  yadayada IF  R> gubble ELSE flubble R>  THEN
+ * 
+ *              is, in fact, correct, while something like:
+ * 
+ *          blablabla >R  yadayada IF  R> gubble THEN
+ * 
+ *              is an error.
+ * 
+ *          Implementing an analysis that would be sufficiently accurate
+ *              to justify reporting an ERROR with certainty (rather than
+ *              a mere WARNING speculatively) would probably require full
+ *              coordination with management of Flow-Control constructs,
+ *              and so is noted here.
+ *
+ **************************************************************************** */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "types.h"
+#include "toke.h"
+#include "emit.h"
+#include "vocabfuncts.h"
+#include "scanner.h"
+#include "stack.h"
+#include "errhandler.h"
+#include "flowcontrol.h"
+#include "stream.h"
+
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              opc              FCode Output Buffer Position Counter
+ *              noerrors         "Ignore Errors" flag, set by "-i" switch     
+ *              do_loop_depth    How deep we are inside DO ... LOOP variants   
+ *              incolon          State of tokenization; TRUE if inside COLON
+ *              statbuf          The word just read from the input stream
+ *              iname            Name of input file currently being processed
+ *              lineno           Current line-number being processed
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *              control_stack_depth        Number of items on "Control-Stack"
+ *
+ **************************************************************************** */
+
+int control_stack_depth = 0;
+
+
+/* **************************************************************************
+ *
+ *              Internal Static Functions:
+ *          push_cstag                     Push an item onto the Control-Stack
+ *          pop_cstag                      Pop one item from the Control-Stack
+ *          control_stack_size_test        Test C-S depth; report if error
+ *          control_structure_mismatch     Print error-message
+ *          offset_too_large               Print error-message
+ *          matchup_control_structure      Error-check Control-Stack
+ *          matchup_two_control_structures Error-check two Control-Stack entries
+ *          emit_fc_offset                 Error-check and output FCode-Offset
+ *          control_structure_swap         Swap control-struct branch-markers
+ *          mark_backward_target           Mark target of backward-branch
+ *          resolve_backward               Resolve backward-target for branch
+ *          mark_forward_branch            Mark forward-branch
+ *          resolve_forward                Resolve forward-branch at target
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *              Internal Named Constants
+ *   Note:  These control-structure identifier tags -- a.k.a. cstags --
+ *       are used to identify the matching components of particular
+ *       control-structures.  They are passed as parameters, and either
+ *       "Pushed" onto the "Control-Stack", or compared with what is on
+ *       "Top" of the "Control-Stack", as an error-check.
+ *
+ *            name                    used by forth words:
+ *         BEGIN_CSTAG             BEGIN AGAIN UNTIL REPEAT
+ *         IF_CSTAG                IF ELSE THEN
+ *         WHILE_CSTAG             WHILE REPEAT THEN
+ *         DO_CSTAG                DO ?DO LOOP +LOOP
+ *         CASE_CSTAG              CASE OF ENDCASE
+ *         OF_CSTAG                OF ENDOF
+ *         ENDOF_CSTAG             ENDOF ENDCASE
+ *
+ *   The numbers assigned are arbitrary; they were selected for a
+ *       high unlikelihood of being encountered in normal usage,
+ *       and constructed with a hint of mnemonic value in mind.
+ *
+ **************************************************************************** */
+                                 /*     Mnemonic:   */
+#define BEGIN_CSTAG  0xC57be916  /*  CST BEGIN      */
+#define IF_CSTAG     0xC57A901f  /*  CSTAG (0) IF   */
+#define WHILE_CSTAG  0xC573412e  /*  CST WHILE      */
+#define DO_CSTAG     0xC57A90d0  /*  CSTAG (0) DO   */
+#define CASE_CSTAG   0xC57Aca5e  /*  CSTA CASE      */
+#define OF_CSTAG     0xC57A90f0  /*  CSTAG OF (0)   */
+#define ENDOF_CSTAG  0xC57e6d0f  /*  CST ENDOF   */
+
+
+/* **************************************************************************
+ *
+ *     Control-Structure identification, matching, completion and error
+ *         messaging will be supported by a data structure, which we
+ *         will call a CSTAG-Group
+ *
+ *     It consists of one "Data-item" and several "Marker" items, thus:
+ *
+ *         The Data-item in most cases will be a value of OPC (the Output
+ *             Buffer Position Counter) which will be used in calculating 
+ *             an offset or placing an offset or both, as the case may be,
+ *             for the control structure in question.  The one exception
+ *             is for a CSTAG-Group generated by a CASE statement; its
+ *             Data-item will be an integer count of the number of "OF"s
+ *             to be resolved when the ENDCASE statement is reached.
+ *
+ *         The CSTAG for the FORTH word, as described above
+ *         The name of the input file in which the word was encountered
+ *             (actually, a pointer to a mem-alloc'ed copy of the filename)
+ *         The line number, within the input file, of the word's invocation
+ *         The Absolute Token Number in all Source Input of the word
+ *         The FORTH word that started the structure, (used in error messages)
+ *         A flag to indicate when two CSTAG-Groups are created together,
+ *             which will be used to prevent duplicate error messages when,
+ *             for instance, a  DO  is mismatched with a  REPEAT .
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *               "Control-Stack" Diagram Notation
+ *
+ *     The CSTAG-Groups will be kept in an order resembling a data-stack,
+ *         (even though it won't be the data-stack itself).  We will refer
+ *         to this list of structures as the "Control Stack", and in our
+ *         comments we will show their arrangement in a format resembling
+ *         stack-diagram remarks.
+ *
+ *     In these "Control-Stack Diagrams", we will use the notation:
+ *		   <Stmt>_{FOR|BACK}w_<TAGNAM>
+ *         to represent a CSTAG-Group generated by a <Stmt> -type of
+ *         statement, with a "FORw"ard or "BACKw"ard branch-marker and
+ *         a CSTAG of the <TAGNAM> type.
+ *
+ *     A CASE-CSTAG-Group will have a different notation:
+ *		   N_OFs...CASE_CSTAG
+ *
+ *     In all cases, a CSTAG-Group will be manipulated as a unit.
+ *
+ *     The notation for Control-Stack Diagram remarks will largely resemble
+ *         the classic form used in FORTH, i.e., enclosed in parentheses,
+ *         lowest item to the left, top item on the right, with a double-
+ *         hyphen to indicate "before" or "after".
+ *
+ *     Enclosure in {curly-braces} followed by a subscript-range indicates
+ *         that the Stack-item or Group is repeated.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      We are not keeping the "Control Stack" structures on the regular
+ *          data stack because a sneaky combination of user-inputs could
+ *          throw things into chaos were we to use that scheme.  Consider
+ *          what would happen if a number were put on the stack, say, in
+ *          tokenizer-escape mode, in between elements of a flow-control
+ *          structure...  Theoretically, there is no reason to prohibit
+ *          that, but it would be unexpectedly problematical for most
+ *          FORTH-based tokenizers.
+ *
+ *      Maintaining the "Control Stack" structures in a linked-list would
+ *          be a more nearly bullet-proof approach.  The theory of operation
+ *          would be the same, broadly speaking, and there would be no need
+ *          to check for  NOT_CSTAG  and no risk of getting the elements of
+ *          the control-structures out of sync.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          Structure Name:    cstag_group_t
+ *          Synopsis:          Control-Structure Tag Group
+ *                            
+ *   Fields:
+ *       cs_tag             Control-structure identifier tag
+ *       cs_inp_fil         Name of input file where C-S was started
+ *       cs_line_num        Line-number in Current Source when C-S was started
+ *       cs_abs_token_num  "Absolute" Token Number when C-S was started
+ *       cs_word            The FORTH word that started the C-S
+ *       cs_not_dup         FALSE if second "Control Stack" entry for same word
+ *       cs_datum           Data-Item of the Group
+ *       prev               Pointer to previous CSTAG-Group in linked-list
+ *
+ *       All data using this structure will remain private to this file,
+ *           so we declare it here rather than in the  .h  file
+ *
+ **************************************************************************** */
+
+typedef struct cstag_group {
+    unsigned long cs_tag;
+    char *cs_inp_fil;
+    unsigned int cs_line_num;
+    unsigned int cs_abs_token_num;
+    char *cs_word;
+    bool cs_not_dup;
+    unsigned long cs_datum;
+    struct cstag_group *prev;
+} cstag_group_t;
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *     control_stack          "Thread" Pointer to the linked-list of 
+ *                                 "Control Stack" structure entries
+ *     not_cs_underflow       Flag used to prevent duplicate messages
+ *     not_consuming_two      Flag used to prevent loss of messages
+ *     didnt_print_otl        Flag used to prevent duplicate messages
+ *
+ **************************************************************************** */
+
+static cstag_group_t *control_stack = NULL;   /*  "Top" of the "Stack"  */
+
+/* **************************************************************************
+ *
+ *     not_cs_underflow is used only by routines that make two calls to
+ *         resolve a marker.  It is set TRUE before the first call; if
+ *         that call had a control-stack underflow, the error-message
+ *         routine resets it to FALSE.  The calling routine can then
+ *         test it as the condition for the second call.
+ *     Routines that make only one call to resolve a marker can ignore it.
+ *
+ **************************************************************************** */
+
+static bool not_cs_underflow;  /*  No need to initialize.  */
+
+/* **************************************************************************
+ *
+ *     not_consuming_two is also used only by routines that make two calls
+ *         to resolve a marker, but for this case, those routines only need  
+ *         to reset it to FALSE and not to test it; that will be done by
+ *         the  control_structure_mismatch()  routine when it looks at
+ *         the  cs_not_dup  field.  If the mismatch occurred because of
+ *         a combination of control-structures that consume one each,
+ *         the message will be printed even for the second "Control Stack"
+ *         entry.  The routine that changed it will have to set it back to
+ *         TRUE when it's done with it.
+ *
+ *     didnt_print_otl is used similarly, but only for the offset-too-large
+ *        error in the   DO ... LOOP  type of control-structures.
+ *
+ **************************************************************************** */
+
+static bool not_consuming_two = TRUE;
+static bool didnt_print_otl = TRUE;
+
+
+/* **************************************************************************
+ *
+ *      Function name:  push_cstag
+ *      Synopsis:       Push a new CSTAG-Group onto the front ("Top")
+ *                      of the (notional) Control-Stack.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             cstag           ID Tag for Control-Structure to "Push"
+ *             datum           The Data-Item for the new CSTAG-Group
+ *         Global Variables:
+ *             iname           Name of input file currently being processed
+ *             lineno          Current-Source line-number being processed
+ *             abs_tokenno     "Absolute"Token Number of word being processed
+ *             statbuf         The word just read, which started the C-S
+ *         Local Static Variables:
+ *             control_stack   Will become the new entry's "prev"
+ *
+ *      Outputs:
+ *         Returned Value:     None
+ *         Global Variables:
+ *             control_stack_depth            Incremented
+ *         Local Static Variables:
+ *             control_stack   Will become the "previous" entry in the list
+ *         Items Pushed onto Control-Stack:
+ *             Top:            A new CSTAG-Group, params as given
+ *         Memory Allocated
+ *             New CSTAG-Group structure
+ *             Duplicate of name of current input file
+ *             Duplicate of word just read
+ *         When Freed?
+ *             When Removing a CSTAG-Group, in pop_cstag()
+ *
+ **************************************************************************** */
+
+static void push_cstag( unsigned long cstag, unsigned long datum)
+{
+    cstag_group_t *cs_temp;
+
+    cs_temp = control_stack;
+    control_stack = safe_malloc( sizeof(cstag_group_t), "pushing CSTag");
+
+    control_stack->cs_tag = cstag;
+    control_stack->cs_inp_fil = strdup(iname);
+    control_stack->cs_line_num = lineno;
+    control_stack->cs_abs_token_num = abs_token_no;
+    control_stack->cs_word = strdup(statbuf);
+    control_stack->cs_not_dup = TRUE;
+    control_stack->cs_datum = datum;
+    control_stack->prev = cs_temp;
+
+    control_stack_depth++;
+    
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  pop_cstag
+ *      Synopsis:       Remove a CSTAG-Group from the front ("Top") of the
+ *                      (notional) Control-Stack.
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Global Variables:
+ *         Local Static Variables:
+ *             control_stack              CSTAG-Group on "Top"
+ *
+ *      Outputs:
+ *         Returned Value:                NONE
+ *         Global Variables:
+ *             control_stack_depth        Decremented
+ *         Local Static Variables:
+ *             control_stack              "Previous" entry will become current
+ *         Memory Freed
+ *             mem-alloc'ed copy of input filename
+ *             mem-alloc'ed copy of Control-structure FORTH word
+ *             CSTAG-Group structure
+ *         Control-Stack, # of Items Popped:  1
+ *
+ *      Process Explanation:
+ *          The calling routine might not check for empty Control-Stack,
+ *              so we have to be sure and check it here.
+ *
+ **************************************************************************** */
+
+static void pop_cstag( void)
+{
+
+    if ( control_stack != NULL )
+    {
+	cstag_group_t *cs_temp;
+
+	cs_temp = control_stack->prev;
+	free( control_stack->cs_word );
+	free( control_stack->cs_inp_fil );
+	free( control_stack );
+	control_stack = cs_temp;
+
+	control_stack_depth--;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  control_stack_size_test
+ *      Synopsis:       Detect Control Stack underflow; report if an ERROR.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             min_depth                 Minimum depth needed
+ *         Global Variables:
+ *             control_stack_depth       Current depth of Control Stack
+ *             statbuf                   Word to name in error message
+ *
+ *      Outputs:
+ *         Returned Value:                TRUE if adequate depth
+ *         Local Static Variables:
+ *             not_cs_underflow           Reset to FALSE if underflow detected.
+ *         Printout:
+ *             Error message is printed.
+ *                 Identify the colon definition if inside one.
+ *
+ *      Process Explanation:
+ *          Some statements need more than one item on the Control Stack;
+ *             they will do their own  control_stack_depth  testing and
+ *             make a separate call to this routine.
+ *
+ **************************************************************************** */
+
+static bool control_stack_size_test( int min_depth )
+{
+    bool retval = TRUE;
+
+    if ( control_stack_depth < min_depth )
+    {
+	retval = FALSE;
+	tokenization_error ( TKERROR,
+		"Control-Stack underflow at %s", strupr(statbuf) );
+	in_last_colon();
+
+	not_cs_underflow = FALSE;   /*  See expl'n early on in this file  */
+    }
+
+    return( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  control_structure_mismatch
+ *      Synopsis:       Report an ERROR after a Control Structure mismatch
+ *                      was detected.
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Global Variables:
+ *             statbuf              Word encountered, to name in error message
+ *         Local Static Variables:
+ *             control_stack        "Pushed" Control-Structure Tag Group
+ *             not_consuming_two    See explanation early on in this file
+ *         Control-Stack Items:
+ *             Top:                 "Current" Control-Structure Tag Group
+ *                                      Some of its "Marker" information
+ *                                      will be used in the error message
+ *
+ *      Outputs:
+ *         Returned Value:                NONE
+ *         Printout:
+ *             Error message is printed
+ *
+ *      Process Explanation:
+ *          This routine is called after a mismatch is detected, and
+ *              before the CSTAG-Group is "Popped" from the notional
+ *              Control-Stack.
+ *          If the  control_stack  pointer is NULL, print a different
+ *              Error message
+ *          Don't print if the "Control Stack" entry is a duplicate and
+ *              we're processing a statement that consumes two entries.
+ *
+ **************************************************************************** */
+
+static void control_structure_mismatch( void )
+{
+    if ( control_stack->cs_not_dup || not_consuming_two )
+    {
+	tokenization_error ( TKERROR,
+	    "The %s is mismatched with the %s" ,
+		strupr(statbuf), strupr(control_stack->cs_word));
+	where_started( control_stack->cs_inp_fil, control_stack->cs_line_num );
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  offset_too_large
+ *      Synopsis:       Report an ERROR after a too-large fcode-offset
+ *                      was detected.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             too_large_for_16     TRUE if the offset is too large to be
+ *                                      expressed as a 16-bit signed number.
+ *         Global Variables:
+ *             statbuf              Word encountered, to name in error message
+ *             offs16               Whether we are using 16-bit offsets
+ *         Local Static Variables:
+ *             control_stack        "Pushed" Control-Structure Tag Group
+ *             didnt_print_otl      Switch to prevent duplicate message
+ *         Control-Stack Items:
+ *             Top:                 "Current" Control-Structure Tag Group
+ *                                      Some of its "Marker" information
+ *                                      will be used in the error message
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Local Static Variables:
+ *             didnt_print_otl      Will be reset to FALSE
+ *             
+ *         Printout:
+ *             Error message:
+ *                 Branch offset too large between <here> and <there>
+ *             Advisory message, if we are using 8-bit offsets, will
+ *                 indicate whether switching to 16-bit offsets would help
+ *
+ *      Process Explanation:
+ *          Two branches are involved in a DO ... LOOP  structure:  an "outer"
+ *              forward-branch and a slightly smaller "inner" backward-branch.
+ *              In the majority of cases, if one offset exceeds the limit,
+ *              both will.  There is, however, a very small but distinct
+ *              possibility that the offset for the smaller branch will not
+ *              exceed the limit while the larger one does.  To prevent two
+ *              messages from being printed in the routine instance, but still
+ *              assure that one will be printed in the rare eventuality, we
+ *              utilize the flag called  didnt_print_otl  in conjunction
+ *              with the  cs_not_dup  field.
+ *
+ **************************************************************************** */
+
+static void offset_too_large( bool too_large_for_16 )
+{
+    if ( control_stack->cs_not_dup || didnt_print_otl )
+    {
+	tokenization_error( TKERROR,
+	    "Branch offset is too large between %s and the %s" ,
+		strupr(statbuf), strupr(control_stack->cs_word));
+	where_started( control_stack->cs_inp_fil, control_stack->cs_line_num );
+	if ( INVERSE( offs16 ) )
+	{
+	    if ( too_large_for_16 )
+	    {
+		tokenization_error ( INFO,
+		    "Offset would be too large even if 16-bit offsets "
+			"were in effect.\n");
+	    }else{
+		tokenization_error ( INFO,
+		    "Offset might fit if 16-bit offsets "
+			"(e.g., fcode-version2) were used.\n" );
+	    }
+	}
+    }
+    didnt_print_otl = FALSE;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_fc_offset
+ *      Synopsis:       Test whether the given FCode-Offset is out-of-range;
+ *                      before placing it into the FCode Output Buffer.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             fc_offset               The given FCode-Offset
+ *         Global Variables:
+ *             offs16                  Whether we are using 16-bit offsets
+ *             noerrors                "Ignore Errors" flag
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *
+ *      Error Detection:
+ *          Error if the given FCode-Offset exceeds the range that can
+ *              be expressed by the size (i.e., 8- or 16- -bits) of the
+ *              offsets we are using.  Call  offset_too_large()  to print
+ *              the Error message; also, if  noerrors  is in effect, issue
+ *              a Warning showing the actual offset and how it will be coded.
+ *
+ *      Process Explanation:
+ *          For forward-branches, the OPC will have to be adjusted to
+ *              indicate the location that was reserved for the offset
+ *              to be written, rather than the current location.  That
+ *              will all be handled by the calling routine.
+ *          We will rely on "C"'s type-conversion (type-casting) facilities.
+ *          Look at the offset value both as an 8-bit and as a 16-bit offset,
+ *              then determine the relevant course of action.
+ *
+ **************************************************************************** */
+
+static void emit_fc_offset( int fc_offset)
+{
+    int fc_offs_s16 = (s16)fc_offset;
+    int fc_offs_s8  =  (s8)fc_offset;
+    bool too_large_for_8  = BOOLVAL( fc_offset != fc_offs_s8 );
+    bool too_large_for_16 = BOOLVAL( fc_offset != fc_offs_s16);
+
+    if ( too_large_for_16 || ( INVERSE(offs16) && too_large_for_8 ) )
+    {
+	offset_too_large( too_large_for_16 );
+	if ( noerrors )
+	{
+	    int coded_as = offs16 ? (int)fc_offs_s16 : (int)fc_offs_s8 ;
+	    tokenization_error( WARNING,
+		"Actual offset is 0x%x (=dec %d), "
+		    "but it will be coded as 0x%x (=dec %d).\n",
+			fc_offset, fc_offset, coded_as, coded_as );
+	}
+    }
+
+    emit_offset( fc_offs_s16 );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  matchup_control_structure
+ *      Synopsis:       Error-check. Compare the given control-structure
+ *                          identifier tag with the one in the CSTAG-Group
+ *                          on "Top" of the "Control Stack".
+ *                      If they don't match, report an error, and, if not
+ *                          "Ignoring Errors", return Error indication.
+ *                      If no error, pass the Data-item back to the caller.
+ *                      Do not consume the CSTAG-Group; that will be the
+ *                          responsibility of the calling routine.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             cstag          Control-struc ID Tag expected by calling function
+ *         Global Variables:
+ *             noerrors       "Ignore Errors" flag
+ *         Local Static Variables:
+ *             control_stack   "Pushed" (current) Control-Structure Tag Group
+ *         Control-Stack Items:
+ *             Top:            Current CSTAG-Group
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE = Successful match, no error.
+ *
+ *      Error Detection:
+ *           Control Stack underflow or cstag mismatch.  See below for details.
+ *
+ *      Process Explanation:
+ *           If the expected cstag does not match the cs_tag from the CSTAG
+ *               Group on "Top" of the "Control Stack", print an ERROR message,
+ *               and, unless the "Ignore Errors" flag is in effect, prepare
+ *               to return FALSE.
+ *          However, if we've "underflowed" the "Control Stack", we dare not
+ *              ignore errors; that could lead to things like attempting to
+ *              write a forward-branch FCode-offset to offset ZERO, over the
+ *              FCODE- or PCI- -header block.  We don't want that...
+ *          So, if the  control_stack  pointer is NULL, we will print an
+ *              ERROR message and immediately return FALSE.
+ *          Since we will not consume the CSTAG-Group, the calling routine
+ *              can access the Data-Item and any "Marker" information it may
+ *              still require via the local  control_stack  pointer. The caller
+ *              will be responsible for removing the CSTAG-Group.
+ *
+ *      Special Exception to "Ignore Errors":
+ *          At the last usage of the  CASE_CSTAG , for the ENDCASE statement,
+ *              this routine will be called to control freeing-up memory, etc.
+ *          For the OF statement, it will be called to control incrementing
+ *              the OF-count datum.
+ *          Processing an ENDCASE statement with the datum from any other
+ *              CSTAG-Group can lead to a huge loop.
+ *          Processing any other "resolver" with the datum from an ENDCASE
+ *              CSTAG-Group can lead to mistaking a very low number for an
+ *              offset into the Output Buffer and attempting to write to it.
+ *          Incrementing the datum from any other CSTAG-Group can lead to
+ *              a variety of unacceptable errors, too many to guess.
+ *          So, if either the given cstag or the cs_tag field of the "Top"
+ *              CSTAG-Group is a CASE_CSTAG , we will not ignore errors.
+ *
+ **************************************************************************** */
+
+static bool matchup_control_structure( unsigned long cstag )
+{
+    bool retval = FALSE;
+
+    if ( control_stack_size_test( 1) )
+    {
+	retval = TRUE;
+
+	if ( control_stack->cs_tag != cstag )
+	{
+	    control_structure_mismatch();
+
+	    if (    ( INVERSE(noerrors) )
+		 || ( cstag == CASE_CSTAG )
+		 || ( control_stack->cs_tag == CASE_CSTAG )
+	            )
+	    {
+		retval = FALSE;
+	    }
+	}
+
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  control_structure_swap
+ *      Synopsis:       Swap control-structure branch-marker Groups
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Local Static Variables:
+ *             control_stack          Pointer to "Control Stack" linked-list
+ *         Control-Stack Items:
+ *             Top:                   CSTAG-Group_0
+ *             Next:                  CSTAG-Group_1
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Local Static Variables:
+ *             control_stack          Points to former "previous" and vice-versa
+ *         Items on Control-Stack:
+ *             Top:                   CSTAG-Group_1
+ *             Next:                  CSTAG-Group_0
+ *
+ *      Error Detection:
+ *          If control-stack depth is not at least 2, CS underflow ERROR.
+ *              This might trigger other routines' error detections also...
+ *
+ *      Extraneous Remarks:
+ *          Before control-structure identification was implemented, offsets
+ *              were kept on the data-stack, and this was a single SWAP.
+ *          When CSTAGs were added, the "Group" was only a pair kept on the
+ *               data-stack -- the CSTAG and the Data-item -- and this
+ *               became a TWO_SWAP()
+ *          For a while, when I tried keeping the CSTAG-Group on the stack,
+ *               this became a FOUR_SWAP()
+ *          That turned out to be unacceptably brittle; this way is much
+ *               more robust. 
+ *          I am so glad I called this functionality out into a separate
+ *              routine, early on in the development process.
+ *
+ *          This is the function called  1 CSROLL  in section A.3.2.3.2
+ *              of the ANSI Forth spec, which likewise corresponds to the
+ *              modifier that Wil Baden, in his characteristically elegant
+ *              nomenclature, dubbed:  BUT 
+ *
+ **************************************************************************** */
+
+static void control_structure_swap( void )
+{
+    if ( control_stack_size_test( 2) )
+    {
+	cstag_group_t *cs_temp;
+
+	cs_temp = control_stack->prev;
+
+	control_stack->prev = cs_temp->prev;
+	cs_temp->prev = control_stack;
+	control_stack = cs_temp;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  matchup_two_control_structures
+ *      Synopsis:       For functions that resolve two CSTAG-Groups, both
+ *                          matchup both "Top of Control Stack"  entries
+ *                          before processing them...
+ *
+ *      Inputs:
+ *         Parameters:
+ *             top_cstag      Control-struc ID Tag expected on "Top" CS entry
+ *             next_cstag     Control-struc ID Tag expected on "Next" CS entry
+ *         Local Static Variables:
+ *             not_cs_underflow   Used for underflow detection.
+ *         Control-Stack Items:
+ *             Top:            Current CSTAG-Group
+ *             Next:           Next CSTAG-Group
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE = Successful matches, no error.
+ *         Global Variables:
+ *             noerrors       "Ignore Errors" flag; cleared, then restored
+ *         Local Static Variables:
+ *             not_consuming_two               Cleared, then restored
+ *         Control-Stack, # of Items Popped:   2 (if matches unsuccessful)
+ *
+ *      Error Detection:
+ *          Control Stack underflow detected by control_structure_swap()
+ *          Control Structure mismatch detected by  control_structure_mismatch()
+ *
+ *      Process Explanation:
+ *          We will use  matchup_control_structure()  to do the "heavy lifting".
+ *          We will not be ignoring errors in these cases.
+ *          Save the results of a match of  top_cstag
+ *          Swap the top two CS entries.
+ *          If an underflow was detected, there's no more matching to be done.
+ *          Otherwise:
+ *              Save the results of a match of  next_cstag
+ *              Swap the top two CS entries again, to their original order.
+ *          The result is TRUE if both matches were successful.
+ *          If the matches were not successful, consume the top two entries
+ *              (unless there's only one, in which case consume it).
+ *
+ **************************************************************************** */
+
+static bool matchup_two_control_structures( unsigned long top_cstag,
+                                                unsigned long next_cstag)
+{
+    bool retval;
+    bool topmatch;
+    bool nextmatch = FALSE;
+    bool sav_noerrors = noerrors;
+    noerrors = FALSE;
+    not_consuming_two = FALSE;
+
+    not_cs_underflow = TRUE;
+    topmatch = matchup_control_structure( top_cstag);
+    if ( not_cs_underflow )
+    {
+	control_structure_swap();
+	if ( not_cs_underflow )
+	{
+	   nextmatch = matchup_control_structure( next_cstag);
+	   control_structure_swap();
+	}
+    }
+
+    retval = BOOLVAL( topmatch && nextmatch);
+
+    if ( INVERSE( retval) )
+    {
+        pop_cstag();
+        pop_cstag();
+    }
+
+    not_consuming_two = TRUE;
+    noerrors = sav_noerrors;
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  mark_backward_target
+ *      Synopsis:       Mark the target of an expected backward-branch
+ *
+ *      Associated FORTH words:                 BEGIN  DO  ?DO
+ *
+ *      Inputs:
+ *         Parameters:
+ *             cstag              Control-structure ID tag for calling function
+ *         Global Variables:
+ *             opc                Output Buffer Position Counter
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Items Pushed onto Control-Stack:
+ *             Top:                 <Stmt>_BACKw_<TAGNAM>
+ *
+ *      Process Explanation:
+ *          Just before this function is called, the token that begins the
+ *              control-structure was written to the FCode Output buffer.
+ *          OPC, the FCode Output Buffer Position Counter, is at the
+ *              destination to which the backward-branch will be targeted.
+ *          Create a CSTAG-Group with the given C-S Tag, and OPC as its datum;
+ *              push it onto the Control-Stack.
+ *          Later, when the backward-branch is installed, the FCode-offset
+ *              will be calculated as the difference between the OPC at
+ *              that time and the target-OPC we saved here.
+ *      
+ **************************************************************************** */
+
+static void mark_backward_target(unsigned long cstag )
+{
+    push_cstag( cstag, (unsigned long)opc);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  mark_forward_branch
+ *      Synopsis:       Mark the location of, and reserve space for, the
+ *                          FCode-offset associated with a forward branch.
+ *
+ *      Associated FORTH words:                 IF  WHILE  ELSE
+ *
+ *      Inputs:
+ *         Parameters:
+ *             cstag              Control-structure ID tag for calling function
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Items Pushed onto Control-Stack:
+ *             Top:                 <Stmt>_FORw_<TAGNAM>
+ *         FCode Output buffer:
+ *             Place-holder FCode-offset of zero.
+ *
+ *      Process Explanation:
+ *          Just before this function is called, the forward-branch token
+ *              that begins the control-structure was written to the FCode
+ *              Output buffer.
+ *          It will need an FCode-offset to the destination to which it will
+ *              be targeted, once that destination is known.
+ *          Create a CSTAG-Group with the given C-S Tag, and OPC as its datum;
+ *              push it onto the Control-Stack.  (This is the same action as
+ *              for marking a backward-target.)
+ *          Then write a place-holder FCode-offset of zero to the FCode
+ *              Output buffer.
+ *          Later, when the destination is known, the FCode-offset will be
+ *              calculated as the difference between the OPC at that time
+ *              and the FCode-offset location we're saving now.  That offset
+ *              will be over-written onto the place-holder offset of zero at
+ *              the location in the Output buffer that we saved on the
+ *              Control-Stack in this routine.
+ *
+ **************************************************************************** */
+
+static void mark_forward_branch(unsigned long cstag )
+{
+    mark_backward_target(cstag );
+    emit_offset(0);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  resolve_backward
+ *      Synopsis:       Resolve backward-target when a backward branch
+ *                      is reached.  Write FCode-offset to reach saved
+ *                      target from current location.
+ *      
+ *      Associated FORTH words:                 AGAIN  UNTIL  REPEAT
+ *                                                LOOP  +LOOP
+ *
+ *      Inputs:
+ *         Parameters:
+ *             cstag              Control-structure ID tag for calling function
+ *         Global Variables:
+ *             opc                Output Buffer Position Counter
+ *         Control-Stack Items:
+ *             Top:              <Stmt>_BACKw_<TAGNAM>
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Global Variables:
+ *             opc               Incremented by size of an FCode-offset
+ *         Control-Stack, # of Items Popped:   1
+ *         FCode Output buffer:
+ *             FCode-offset to reach backward-target
+ *
+ *      Error Detection:
+ *          Test for Control-structure ID tag match.
+ *
+ *      Process Explanation:
+ *          Just before this function is called, the backward-branch token
+ *              that ends the control-structure was written to the FCode
+ *              Output buffer.
+ *          The current OPC is at the point from which the FCode-offset
+ *              is to be calculated, and at which it is to be written.
+ *          The top of the Control-Stack should have the CSTAG-Group from
+ *              the statement that prepared the backward-branch target that
+ *              we expect to resolve.  Its datum is the OPC of the target
+ *              of the backward branch.
+ *          If the supplied Control-structure ID tag does not match the one
+ *              on top of the Control-Stack, announce an error.  We will
+ *              still write an FCode-offset, but it will be a place-holder
+ *              of zero.
+ *          Otherwise, the FCode-offset we will write will be the difference
+ *              between the target-OPC and our current OPC.
+ *
+ **************************************************************************** */
+
+static void resolve_backward( unsigned long cstag)
+{
+    unsigned long targ_opc;
+    int fc_offset = 0;
+
+    if ( matchup_control_structure( cstag) )
+    {
+	targ_opc = control_stack->cs_datum;
+	fc_offset = targ_opc - opc;
+    }
+
+    emit_fc_offset( fc_offset );
+    pop_cstag();
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  resolve_forward
+ *      Synopsis:       Resolve a forward-branch when its target has been
+ *                      reached.  Write the FCode-offset into the space
+ *                      that was reserved.
+ *
+ *      Associated FORTH words:                 ELSE  THEN  REPEAT
+ *                                                LOOP  +LOOP
+ *
+ *      Inputs:
+ *         Parameters:
+ *             cstag              Control-structure ID tag for calling function
+ *         Global Variables:
+ *             opc                Output Buffer Position Counter
+ *         Control-Stack Items:
+ *             Top:               <Stmt>_FORw_<TAGNAM>
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Global Variables:
+ *             opc               Changed, then restored.
+ *         Control-Stack, # of Items Popped:   1
+ *         FCode Output buffer:
+ *             FCode-offset is written to location where space was reserved
+ *                 when the forward-branch was marked.
+ *
+ *      Error Detection:
+ *          Test for Control-structure ID tag match.
+ *
+ *      Process Explanation:
+ *          Just before this function is called, the last token -- and 
+ *              possibly, FCode-offset -- that is within the scope of
+ *              what the branch might skip was written to the FCode
+ *              Output buffer.
+ *          The current OPC is at the point from which the FCode-offset
+ *              is to be calculated, but not at which it is to be written.
+ *          The top of the Control-Stack should have the CSTAG-Group from
+ *              the statement that prepared the forward-branch we expect
+ *              to resolve, and for which our current OPC is the target.
+ *              Its datum is the OPC of the space that was reserved for
+ *              the forward-branch whose target we have just reached.
+ *          If the supplied Control-structure ID tag does not match the one
+ *              on top of the Control-Stack, announce an error and we're done.
+ *          Otherwise, the datum is used both as part of the calculation of
+ *              the FCode-offset we are about to write, and as the location
+ *              to which we will write it.
+ *          The FCode-offset is calculated as the difference between our
+ *              current OPC and the reserved OPC location.
+ *          We will not be ignoring errors in these cases, because we would
+ *              be over-writing something that might not be a place-holder
+ *              for a forward-branch at an earlier location in the FCode
+ *              Output buffer.
+ *
+ **************************************************************************** */
+
+static void resolve_forward( unsigned long cstag)
+{
+    unsigned long resvd_opc;
+    bool sav_noerrors = noerrors;
+    bool cs_match_result;
+    noerrors = FALSE;
+    /*  Restore the "ignore-errors" flag before we act on our match result
+     *      because we want it to remain in effect for  emit_fc_offset()
+     */
+    cs_match_result = matchup_control_structure( cstag);
+    noerrors = sav_noerrors;
+
+    if ( cs_match_result )
+    {
+	int saved_opc;
+	int fc_offset;
+
+	resvd_opc = control_stack->cs_datum;
+	fc_offset = opc - resvd_opc;
+
+	saved_opc = opc;
+	opc = resvd_opc;
+
+
+	emit_fc_offset( fc_offset );
+	opc = saved_opc;
+    }
+    pop_cstag();
+}
+	
+
+/* **************************************************************************
+ *
+ *      The functions that follow are the exported routines that
+ *          utilize the preceding support-routines to effect their
+ *          associated FORTH words.
+ *
+ *      The routines they call will take care of most of the Error
+ *          Detection via stack-depth checking and Control-structure
+ *          ID tag matching, so those will not be called-out in the
+ *          prologues.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_if
+ *      Synopsis:       All the actions when IF is encountered
+ *
+ *      Associated FORTH word:                 IF
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Items Pushed onto Control-Stack:
+ *             Top:                If_FORw_IF
+ *         FCode Output buffer:
+ *             Token for conditional branch -- b?branch -- followed by
+ *                 place-holder of zero for FCode-offset
+ *              
+ *
+ **************************************************************************** */
+
+void emit_if( void )
+{
+    emit_token("b?branch");
+    mark_forward_branch( IF_CSTAG );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_then
+ *      Synopsis:       All the actions when THEN is encountered; also
+ *                      part of another forward-branch resolver's action.
+ *
+ *      Associated FORTH words:                 THEN  ELSE
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Local Static Variables:
+ *             control_stack       Points to "Top" Control-Structure Tag Group
+ *         Control-Stack Items:
+ *             Top:                If_FORw_IF | While_FORw_WHILE
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Control-Stack, # of Items Popped:   1
+ *         FCode Output buffer:
+ *             Token for forward-resolve -- b(>resolve) -- then the space
+ *                 reserved for the forward-branch FCode-offset is filled
+ *                 in so that it reaches the token after the  b(>resolve) . 
+ *
+ *      Process Explanation:
+ *          The THEN statement or the ELSE statement must be able to resolve
+ *              a WHILE statement, in order to implement the extended flow-
+ *              -control structures as described in sec. A.3.2.3.2 of the
+ *              ANSI Forth Spec.
+ *          But we must prevent the sequence  IF ... BEGIN ...  REPEAT  from
+ *              compiling as though it were:  IF ... BEGIN ...  AGAIN THEN
+ *          We do this by having a separate CSTAG for WHILE and allowing
+ *              it here but not allowing the IF_CSTAG when processing REPEAT.
+ *
+ **************************************************************************** */
+
+void emit_then( void )
+{
+    emit_token("b(>resolve)");
+    if ( control_stack != NULL )
+    {
+	if ( control_stack->cs_tag == WHILE_CSTAG )
+	{
+	    control_stack->cs_tag = IF_CSTAG;
+	}
+    }
+    resolve_forward( IF_CSTAG );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_else
+ *      Synopsis:       All the actions when ELSE is encountered
+ *
+ *      Associated FORTH word:                 ELSE
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Global Variables:
+ *             control_stack_depth   Current depth of Control Stack
+ *         Local Static Variables:
+ *             not_cs_underflow      If this is FALSE after the c-s swap, it
+ *                                       means an underflow resulted; skip
+ *                                       the call to resolve the first marker.
+ *         Control-Stack Items:
+ *             Top:                {If_FORw_IF}1
+ *                 (Datum is OPC of earlier forward-branch; must be resolved.)
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Control-Stack, # of Items Popped:   1
+ *         Items Pushed onto Control-Stack:
+ *             Top:                {If_FORw_IF}2
+ *                 (Datum is current OPC, after forward-branch is placed.)
+ *         FCode Output buffer:
+ *             Token for unconditional branch -- bbranch-- followed by
+ *                 place-holder of zero for FCode-offset.  Then, token
+ *                  for forward-resolve -- b(>resolve) -- and the space
+ *                  reserved earlier for the conditional forward-branch
+ *                  FCode-offset is filled in to reach the token after
+ *                  the  b(>resolve) .
+ *
+ *      Error Detection:
+ *          If the "Control-Stack" is empty, bypass the forward branch
+ *              and let the call to  control_structure_swap()  report
+ *              the underflow error.  Then use  not_cs_underflow  to
+ *              control whether to resolve the forward-branch. 
+ *
+ *      Process Explanation:
+ *          The final item needed within the scope of what the earlier
+ *              conditional branch might skip is an unconditional branch
+ *              over the "else"-clause to follow.  After that, the earlier
+ *              conditional branch needs to be resolved.  This last step
+ *              is identical to the action of  THEN .
+ *
+ **************************************************************************** */
+
+void emit_else( void )
+{
+    if ( control_stack_depth > 0 )
+    {
+	emit_token("bbranch");
+	mark_forward_branch( IF_CSTAG );
+    }
+    not_cs_underflow = TRUE;
+    control_structure_swap();
+    if ( not_cs_underflow )
+    {
+        emit_then();
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_begin
+ *      Synopsis:       All the actions when BEGIN is encountered
+ *
+ *      Associated FORTH word:                 BEGIN
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Items Pushed onto Control-Stack:
+ *             Top:                Begin_BACKw_BEGIN
+ *                 (Datum is current OPC, target of future backward-branch)
+ *         FCode Output buffer:
+ *             Token for target of backward branch -- b(<mark)
+ *
+ **************************************************************************** */
+
+void emit_begin( void )
+{
+    emit_token("b(<mark)");
+    mark_backward_target( BEGIN_CSTAG );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_again
+ *      Synopsis:       All the actions when AGAIN is encountered
+ *
+ *      Associated FORTH words:               AGAIN  REPEAT
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Control-Stack Items:
+ *             Top:                Begin_BACKw_BEGIN
+ *                        (Datum is OPC of backward-branch target at BEGIN)
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Control-Stack, # of Items Popped:   1
+ *         FCode Output buffer:
+ *             Token for unconditional branch -- bbranch -- followed by
+ *                 FCode-offset that reaches just after the  b(<mark) 
+ *                 token at the corresponding  BEGIN  statement. 
+ *
+ *      Process Explanation:
+ *          The FCode-offset is calculated as the difference between our
+ *              current OPC and the target-OPC saved on the Control-Stack.
+ *              
+ **************************************************************************** */
+
+void emit_again( void )
+{
+    emit_token("bbranch");
+    resolve_backward( BEGIN_CSTAG );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_until
+ *      Synopsis:       All the actions when UNTIL is encountered
+ *
+ *      Associated FORTH word:                 UNTIL
+ *
+ *      Process Explanation:
+ *          Same as AGAIN except token is conditional branch -- b?branch --
+ *              instead of unconditional.
+ *
+ **************************************************************************** */
+
+void emit_until( void )
+{
+    emit_token("b?branch");
+    resolve_backward( BEGIN_CSTAG );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_while
+ *      Synopsis:       All the actions when WHILE is encountered
+ *
+ *      Associated FORTH word:                 WHILE
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Global Variables:
+ *             control_stack_depth       Number of items on "Control-Stack"
+ *         Control-Stack Items:
+ *             Top:                      Begin_BACKw_BEGIN
+ *                                 (Datum is OPC of backward-branch target)
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Control-Stack:        1 item added below top item.
+ *         Items on Control-Stack:
+ *             Top:                Begin_BACKw_BEGIN
+ *             Next:               While_FORw_WHILE
+ *         FCode Output buffer:
+ *             Token for conditional branch -- b?branch -- followed by
+ *                 place-holder of zero for FCode-offset
+ *
+ *      Error Detection:
+ *          If the "Control-Stack" is empty, bypass creating the branch
+ *              and let the call to  control_structure_swap()  report
+ *              the underflow error.
+ *
+ *      Process Explanation:
+ *          Output a conditional forward-branch sequence, similar to  IF 
+ *              (except with a WHILE CSTAG), but be sure to leave the
+ *              control-structure branch-marker that was created by the
+ *              preceding  BEGIN   on top of the one just generated:
+ *              the  BEGIN  needs to be resolved first in any case, and
+ *              doing this here is the key to implementing the extended
+ *              control-flow structures as described in sec. A.3.2.3.2
+ *              of the ANSI Forth Spec.
+ *
+ *      Extraneous Remarks:
+ *          It was for the use of this function that Wil Baden coined the
+ *              name BUT for the control-structure swap routine.  The idea
+ *              was that the implementation of WHILE could be boiled down
+ *              to:  IF BUT   (couldn't quite fit an AND in there...;-} )
+ *          Naturally, this implementation is a smidgeon more complicated...
+ *
+ **************************************************************************** */
+
+void emit_while( void )
+{
+    if ( control_stack_depth > 0 )
+    {
+	emit_token("b?branch");
+	mark_forward_branch( WHILE_CSTAG );
+    }
+    control_structure_swap();
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_repeat
+ *      Synopsis:       All the actions when REPEAT is encountered
+ *
+ *      Associated FORTH word:                 REPEAT
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Local Static Variables:
+ *             not_cs_underflow    If FALSE after first call to resolve marker,
+ *                                     an underflow resulted; skip second call.
+ *         Control-Stack Items:
+ *             Top:                Begin_BACKw_BEGIN
+ *                        (Datum is OPC of backward-branch target at BEGIN)
+ *             Next:               If_FORw_IF
+ *                        (Datum is OPC of FCode-offset place-holder)
+ *
+ *      Outputs:
+ *         Returned Value:                    NONE
+ *         Local Static Variables:
+ *             not_consuming_two              Cleared, then restored
+ *         Control-Stack, # of Items Popped:   2
+ *         FCode Output buffer:
+ *             Token for unconditional branch -- bbranch -- followed by
+ *                 FCode-offset that reaches just after the  b(<mark) 
+ *                 token at the corresponding  BEGIN  statement.  Then
+ *                 the token for forward-resolve -- b(>resolve) -- and
+ *                 the space reserved for the conditional forward-branch
+ *                 FCode-offset is filled in so that it reaches the token
+ *                 after the  b(>resolve) .
+ *
+ *      Process Explanation:
+ *          The action is identical to that taken for AGAIN followed
+ *               by the action for THEN.
+ *          The Local Static Variable  not_consuming_two  gets cleared
+ *               and restored by this routine.
+ *
+ **************************************************************************** */
+
+void emit_repeat( void )
+{
+    if ( matchup_two_control_structures( BEGIN_CSTAG, WHILE_CSTAG ) )
+    {
+	not_cs_underflow = TRUE;
+	not_consuming_two = FALSE;
+	emit_again();
+	if ( not_cs_underflow )
+	{
+            emit_token("b(>resolve)");
+	    resolve_forward( WHILE_CSTAG );
+	}
+	not_consuming_two = TRUE;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  mark_do
+ *      Synopsis:       Common routine for marking the branches for
+ *                      the "do" variants
+ *
+ *      Associated FORTH words:              DO  ?DO
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Global Variables:
+ *             do_loop_depth         Incremented
+ *         Items Pushed onto Control-Stack:
+ *             Top:              Do_FORw_DO
+ *             Next:             Do_BACKw_DO
+ *         FCode Output buffer:
+ *             Place-holder of zero for FCode-offset
+ *
+ *      Error Detection:
+ *          The  do_loop_depth  counter will be used by other routines
+ *              to detect misplaced "LEAVE", "UNLOOP", "I" and suchlike.
+ *              (Imbalanced "LOOP"  statements are detected by the CSTag
+ *              matching mechanism.)
+ *
+ *      Process Explanation:
+ *          Just before this function is called, the forward-branching token
+ *              for the "DO" variant that begins the control-structure was
+ *              written to the FCode Output buffer.
+ *          It needs an FCode-offset for a forward-branch to just after
+ *              its corresponding "LOOP" variant and the FCode-offset
+ *              associated therewith.
+ *          That "LOOP" variant's associated FCode-offset is targeted
+ *              to the token that follows the one for this "DO" variant
+ *              and its FCode-offset.
+ *          Mark the forward-branch with the C-S Tag for DO and write a
+ *              place-holder FCode-offset of zero to FCode Output.
+ *          Indicate that the mark that will be processed second (but which
+ *              was made first) is a duplicate of the one that will be
+ *              processed first.
+ *          Then mark the backward-branch target, also with the DO C-S Tag.
+ *          Finally, increment the  do_loop_depth  counter.
+ *
+ *      Extraneous Remarks:
+ *          This is more complicated to describe than to code...  ;-)
+ *
+ **************************************************************************** */
+
+void mark_do( void )
+{
+    mark_forward_branch( DO_CSTAG);
+    control_stack->cs_not_dup = FALSE;
+    mark_backward_target( DO_CSTAG);
+    do_loop_depth++;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  resolve_loop
+ *      Synopsis:       Common routine for resolving the branches for
+ *                      the "loop" variants.
+ *
+ *      Associated FORTH words:              LOOP  +LOOP
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Global Variables:
+ *             statbuf             Word read from input stream (either "loop"
+ *                                     or "+loop"), used for Error Message.
+ *         Local Static Variables:
+ *             not_cs_underflow    If FALSE after first call to resolve marker,
+ *                                     an underflow resulted; skip second call.
+ *         Control-Stack Items:
+ *             Top:                Do_FORw_DO
+ *             Next:               Do_BACKw_DO
+ *
+ *      Outputs:
+ *         Returned Value:                    NONE
+ *         Global Variables:
+ *             do_loop_depth                  Decremented
+ *         Local Static Variables:
+ *             not_consuming_two              Cleared, then restored
+ *             didnt_print_otl                Set, then set again at end.
+ *         Control-Stack, # of Items Popped:   2
+ *         FCode Output buffer:
+ *             FCode-offset that reaches just after the token of the
+ *                 corresponding "DO" variant.  Then the space reserved
+ *                 for the FCode-offset of the forward-branch associated
+ *                 with the "DO" variant is filled in so that it reaches
+ *                 the token just after the "DO" variant's FCode-offset.
+ *                 
+ *      Error Detection:
+ *          A value of zero in  do_loop_depth  before it's decremented
+ *              indicates a  DO ... LOOP  imbalance, which is an ERROR,
+ *              but our other error-reporting mechanisms will catch it,
+ *              so we don't check or report it here.
+ *
+ *      Process Explanation:
+ *          Just before this function is called, the backward-branching
+ *              token for the "LOOP" variant that ends the control-structure
+ *              was written to the FCode Output buffer.
+ *          It needs an FCode-offset for a backward-branch targeted just
+ *              after its corresponding "DO" variant and the FCode-offset
+ *              associated therewith.
+ *          That "DO" variant's associated FCode-offset is targeted to
+ *              the token that follows the one for this "LOOP" variant
+ *              and its FCode-offset.
+ *          Make sure there are two DO C-S Tag entries on the Control Stack.
+ *          Resolve the backward-branch, matching your target to the first
+ *              C-S Tag for DO
+ *          Then resolve the forward-branch, targeting to your new OPC
+ *              position, and also making sure you match the DO C-S Tag.
+ *          We keep track of  do_loop_depth  for other error-detection
+ *              by decrementing it; make sure it doesn't go below zero.
+ *          Don't bother resolving the forward-branch if we underflowed
+ *              the "Control Stack" trying to resolve the backward-branch.
+ *          If the two top C-S Tag entries are not for a DO statement, the
+ *              matchup_two_control_structures() routine will consume both
+ *              or up to two of them, and we will place a dummy offset of
+ *              zero to follow-up the backward-branching token that has
+ *              already been written.
+ *      
+ *      Extraneous Remarks:
+ *          This is only a little more complicated to describe
+ *              than to code...  ;-)
+ *
+ **************************************************************************** */
+
+void resolve_loop( void )
+{
+    if ( INVERSE( matchup_two_control_structures( DO_CSTAG, DO_CSTAG) ) )
+    {
+	emit_offset( 0 );
+    }else{
+	not_cs_underflow = TRUE;
+	didnt_print_otl = TRUE;
+	not_consuming_two = FALSE;
+	resolve_backward( DO_CSTAG);
+	if ( not_cs_underflow )
+	{
+	    resolve_forward( DO_CSTAG);
+	}
+	if ( do_loop_depth > 0 ) do_loop_depth--;
+	not_consuming_two = TRUE;
+	didnt_print_otl = TRUE;   /*  Might have gotten cleared   */
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_case
+ *      Synopsis:       All the actions when CASE is encountered
+ *
+ *      Associated FORTH word:                 CASE
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Items Pushed onto Control-Stack:
+ *             Top:              N_OFs=0...CASE_CSTAG
+ *                    (Datum is 0 , Initial count of  OF .. ENDOF  pairs)
+ *         FCode Output buffer:
+ *             Token for start of a CASE structure -- b(case)
+ *                 Does not require an FCode-offset.
+ *
+ **************************************************************************** */
+
+void emit_case( void )
+{
+    push_cstag( CASE_CSTAG, 0);
+    emit_token("b(case)");
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_of
+ *      Synopsis:       All the actions when OF is encountered
+ *
+ *      Associated FORTH word:                 OF
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Control-Stack Items:
+ *             Top:                N_OFs...CASE_CSTAG
+ *                    (Datum is OF-count, number of  OF .. ENDOF  pairs)
+ *            {Next and beyond}:   {Endof_FORw_ENDOF}1..n_ofs
+ *            { Repeat for OF-count number of times }
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Control-Stack, 1 Item Pushed, 1 modified:
+ *             Top:                Of_FORw_OF
+ *             Next:               N_OFs+1...CASE_CSTAG
+ *                    (Datum has been incremented)
+ *            {3rd and beyond}:    {Endof_FORw_ENDOF}1..n_ofs
+ *            { Repeat for 1 through the un-incremented OF-count }
+ *            (Same as Next etcetera at input-time.)
+ *         FCode Output buffer:
+ *             Token for OF statement -- b(of) -- followed by
+ *                 place-holder FCode-offset of zero
+ *
+ *      Error Detection:
+ *          Matchup CASE-cstag before incrementing OF-count
+ *
+ *      Process Explanation:
+ *          Main difference between this implementation and that outlined
+ *              in "the book" (see below) is that we do not directly use
+ *              the routine for the IF statement's flow-control; we will
+ *              use a different CSTAG for better mismatch detection.
+ *
+ *      Extraneous Remarks:
+ *          This is a "by the book" (ANSI Forth spec, section A.3.2.3.2)
+ *              implementation (mostly).  Incrementing the OF-count here,
+ *              after we've matched up the CSTAG, gives us (and the user)
+ *              just a little bit more protection...
+ *
+ **************************************************************************** */
+
+void emit_of( void )
+{
+
+    if ( matchup_control_structure( CASE_CSTAG ) )
+    {
+	emit_token("b(of)");
+
+	/*
+	 *  See comment-block about "Control-Stack" Diagram Notation
+	 *       early on in this file.
+	 *
+	 */
+
+	/* (    {Endof_FORw_ENDOF}1..n_ofs  N_OFs...CASE_CSTAG -- )          */
+
+	/*  Increment the OF-count .  */
+	(control_stack->cs_datum)++;
+
+	/* (    {Endof_FORw_ENDOF}1..n_ofs  N_OFs+1...CASE_CSTAG -- )        */
+
+	mark_forward_branch( OF_CSTAG );
+	/* ( -- {Endof_FORw_ENDOF}1..n_ofs  N_OFs+1...CASE_CSTAG Of_FORw_OF )
+	 */
+    }
+    /*  Leave the CSTAG-Group on the "Control-Stack" .  */
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_endof
+ *      Synopsis:       All the actions when ENDOF is encountered
+ *
+ *      Associated FORTH word:                 ENDOF
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Control-Stack Items:
+ *             Top:                Of_FORw_OF
+ *             Next:               N_OFs+1...CASE_CSTAG
+ *                    (Datum has been incremented)
+ *            {3rd and beyond}:    {Endof_FORw_ENDOF}1..n_ofs
+ *            { Repeat for 1 through the un-incremented OF-count )
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Control-Stack, 1 Item Popped, 1 new Item Pushed.
+ *             Top:                N_OFs...CASE_CSTAG
+ *                    (The count itself is unchanged from input-time, but
+ *                         the number of {Endof_FORw_ENDOF} CSTAG-Groups
+ *                         has caught up with this number, so it is
+ *                         no longer notated as " + 1 ").
+ *            {Next and beyond}:   {Endof_FORw_ENDOF}1..n_ofs
+ *            { Repeat for 1 through the updated OF-count )
+ *         FCode Output buffer:
+ *             Token for ENDOF statement -- b(endof) -- followed by
+ *                 place-holder FCode-offset of zero.  Then the space reserved
+ *                 for the FCode-offset of the forward-branch associated
+ *                 with the "OF" statement is filled in so that it reaches
+ *                 the token just after the "ENDOF" statement's FCode-offset.
+ *
+ *      Error Detection:
+ *          If control-stack depth  is not at least 2, CS underflow ERROR
+ *              and no further action.
+ *          Routine that resolves the forward-branch checks for matchup error.
+ *
+ **************************************************************************** */
+
+void emit_endof( void )
+{
+    if ( control_stack_size_test( 2) )
+    {
+	emit_token("b(endof)");
+
+	/*  See "Control-Stack" Diagram Notation comment-block  */
+
+	/*  Stack-diagrams might need to be split across lines.  */
+
+	/* (    {Endof_FORw_ENDOF}1..n_ofs  N_OFs+1...CASE_CSTAG  ...  
+	 *                       ...                          Of_FORw_OF -- )
+	 */
+	mark_forward_branch(ENDOF_CSTAG);
+	/* ( -- {Endof_FORw_ENDOF}1..n_ofs  N_OFs+1...CASE_CSTAG  ...  
+	 *                       ...  Of_FORw_OF  {Endof_FORw_ENDOF}n_ofs+1 )
+	 */
+
+	control_structure_swap();
+	/* ( -- {Endof_FORw_ENDOF}1..n_ofs  N_OFs+1...CASE_CSTAG  ...
+	 *                       ...  {Endof_FORw_ENDOF}n_ofs+1  Of_FORw_OF )
+	 */
+
+	resolve_forward( OF_CSTAG );
+	/* ( -- {Endof_FORw_ENDOF}1..n_ofs  N_OFs+1...CASE_CSTAG        ...
+	 *                       ...  {Endof_FORw_ENDOF}n_ofs+1  )
+	 */
+
+	control_structure_swap();
+	/* ( -- {Endof_FORw_ENDOF}1..n_ofs         ...
+	 *                       ...  {Endof_FORw_ENDOF}n_ofs+1   ...
+	 *                                        ...  N_OFs+1...CASE_CSTAG )
+	 */
+
+	/*  The number of ENDOF-tagged Forward-Marker pairs has now
+	 *     caught up with the incremented OF-count; therefore,
+	 *     we can notate the above as:
+	 *
+	 *  ( {Endof_FORw_ENDOF}1..n_ofs  N_OFs CASE_CSTAG )
+	 *
+	 *     and we are ready for another  OF ... ENDOF  pair,
+	 *     or for the ENDCASE statement.
+	 */
+    }
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  emit_endcase
+ *      Synopsis:       All the actions when ENDCASE is encountered
+ *
+ *      Associated FORTH word:                 ENDCASE
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Control-Stack Items:
+ *             Top:                N_OFs...CASE_CSTAG
+ *                    (Datum is OF-count, number of  OF .. ENDOF  pairs)
+ *            {Next and beyond}:   {Endof_FORw_ENDOF}1..n_ofs
+ *            { Repeat for OF-count number of times }
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Control-Stack, # of Items Popped:  OF-count + 1
+ *         FCode Output buffer:
+ *             Token for ENDCASE statement -- b(endcase)
+ *             Then the spaces reserved for the FCode-offsets of all the
+ *                 forward-branches associated with the OF-count number
+ *                 of ENDOF statements are filled in so that they reach
+ *                 the token just after this "ENDCASE" statement.
+ *
+ *      Error Detection:
+ *          Routine that resolves the forward-branch checks for matchup error
+ *              for each forward-branch filled in, plus the matchup routine
+ *              checks before the OF-count is retrieved.
+ *
+ *      Process Explanation:
+ *          Retrieve the OF-count and resolve that number of ENDOF statements
+ *      
+ *      Extraneous Remarks:
+ *          The setup makes coding this routine appear fairly simple...  ;-}
+ *
+ **************************************************************************** */
+
+void emit_endcase( void )
+{
+    unsigned long n_endofs ;
+    if ( matchup_control_structure( CASE_CSTAG) )
+    {
+	int indx;
+
+	emit_token("b(endcase)");
+	n_endofs = control_stack->cs_datum;
+	for ( indx = 0 ; indx < n_endofs ; indx++ )
+	{
+	    /*  Because  matchup_control_structure  doesn't pop the
+	     *      control-stack, we have the  N_OFs...CASE_CSTAG
+	     *      item on top of the  Endof_FORw_ENDOF  item we
+	     *      want to resolve.  We need to keep it there so
+	     *      the  POP  is valid for the other path as well
+	     *      as at the end of this one.
+	     *  So we  SWAP  to get at the  Endof_FORw_ENDOF  item.
+	     */
+	    control_structure_swap();
+	    resolve_forward( ENDOF_CSTAG);
+	}
+    }
+    pop_cstag();
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  control_struct_incomplete
+ *      Synopsis:       Print a Message of given severity with origin info for
+ *                          a control-structure that has not been completed.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             c_s_entry             Control-structure about which to display
+ *             severity              Severity of the messages to display.
+ *             call_cond             String identifying Calling Condition;
+ *                                       used in the message.
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *             
+ *         Printout:
+ *             Message of given severity...
+ *
+ *      Process Explanation:
+ *          The calling routine will be responsible for all filtering of
+ *               duplicate structures and the like.  This routine will
+ *               simply display a message.
+ *
+ **************************************************************************** */
+
+static void control_struct_incomplete(
+			    int severity,
+				char *call_cond,
+				    cstag_group_t *c_s_entry)
+{
+    tokenization_error ( severity,
+	"%s before completion of %s" ,
+	    call_cond, strupr(c_s_entry->cs_word));
+    where_started( c_s_entry->cs_inp_fil, c_s_entry->cs_line_num );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  announce_control_structs
+ *      Synopsis:       Print a series of Messages (of severity as specified)
+ *                          announcing that the calling event is occurring
+ *                          in the context of Control-Flow structure(s),
+ *                          back to the given limit.  Leave the control
+ *                          structures in effect.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             severity              Severity of the messages to display.
+ *             call_cond             String identifying Calling Condition;
+ *                                       used in the message.
+ *             abs_token_limit       Limit, in terms of abs_token_no
+ *         Local Static Variables:
+ *             control_stack         Pointer to "Top" of "Control-Stack"
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *         Printout:
+ *             A Message for each unresolved Control-Flow structure.
+ *
+ **************************************************************************** */
+
+void announce_control_structs( int severity, char *call_cond,
+				          unsigned int abs_token_limit)
+{
+    cstag_group_t *cs_temp = control_stack;
+    while ( cs_temp != NULL )
+    {
+	if ( cs_temp->cs_abs_token_num < abs_token_limit )
+	{
+	    break;
+	}
+	if ( cs_temp->cs_not_dup )
+	{
+	    control_struct_incomplete( severity, call_cond, cs_temp );
+	}
+	cs_temp = cs_temp->prev;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  clear_control_structs_to_limit
+ *      Synopsis:       Clear items from the "Control-Stack" back to the given
+ *                          limit.  Print error-messages with origin info for
+ *                          control-structures that have not been completed.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             call_cond                 String identifying Calling Condition;
+ *                                           used in the Error message.
+ *             abs_token_limit           Limit, in terms of abs_token_no
+ *         Global Variables:
+ *             control_stack_depth       Number of items on "Control-Stack"
+ *             control_stack             Pointer to "Top" of "Control-Stack"
+ *         Control-Stack Items:
+ *             The  cs_inp_fil  and  cs_line_num  tags of any item cleared
+ *                 from the "Control-Stack" are used in error-messages.
+ *
+ *      Outputs:
+ *         Returned Value: 
+ *         Global Variables:
+ *             do_loop_depth             Decremented when "DO" item cleared.
+ *             control_stack_depth       Decremented by called routine.
+ *         Control-Stack, # of Items Popped:  As many as go back to given limit
+ *         Memory Freed
+ *             By called routine.
+ *
+ *      Error Detection:
+ *          Any item on the "Control-Stack" represents a Control-Structure
+ *              that was not completed when the Calling Condition was
+ *              encountered.  Error; identify the origin of the structure.
+ *          No special actions if  noerrors  is set.
+ *
+ *      Process Explanation:
+ *          The given limit corresponds to the value of  abs_token_no  at
+ *              the time the colon-definition (or whatever...) was created.
+ *              Any kind of Control-Structure imbalance at the end of the
+ *              colon-definition is an error and the entries must be cleared,
+ *              but the colon-definition may have been created inside nested
+ *              interpretation-time Control-Structures, and those must be
+ *              preserved. 
+ *             
+ *          Of course, if this routine is called with a given limit of zero,
+ *              that would mean all the entries are to be cleared.  That will
+ *              be the way  clear_control_structs()  is implemented.
+ *          We control the loop by the  cs_abs_token_num  field, but also
+ *              make sure we haven't underflowed  control_stack_depth
+ *          We skip messages and other processing for items that are duplicates
+ *                    of others, based on the  cs_not_dup  field.
+ *               If the cs_tag field is  DO_CSTAG  we decrement  do_loop_depth
+ *          The  pop_cstag()  routine takes care of the rest.
+ *               
+ *      Extraneous Remarks:
+ *          This is a retrofit; necessary because we now  permit definitions
+ *              to occur inside interpretation-time Control-Structures.  Calls
+ *              to  clear_control_structs()  are already scattered around...
+ *
+ **************************************************************************** */
+
+void clear_control_structs_to_limit( char *call_cond,
+				          unsigned int abs_token_limit)
+{
+    while ( control_stack_depth > 0 )
+    {
+	if ( control_stack->cs_abs_token_num < abs_token_limit )
+	{
+	    break;
+	}
+	if ( control_stack->cs_not_dup )
+	{
+	    control_struct_incomplete( TKERROR, call_cond, control_stack );
+	    if ( control_stack->cs_tag == DO_CSTAG) do_loop_depth--;
+	}
+	pop_cstag();
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  clear_control_structs
+ *      Synopsis:       Make sure the "Control-Stack" is cleared, and print
+ *                          error-messages (giving origin information) for
+ *                          control-structures that have not been completed.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             call_cond                 String identifying Calling Condition;
+ *                                           used in the Error message.
+ *         Global Variables:
+ *             control_stack_depth       Number of items on "Control-Stack"
+ *             control_stack             Pointer to "Top" of "Control-Stack"
+ *         Control-Stack Items:
+ *             The  cs_inp_fil  and  cs_line_num  tags of any item found on
+ *                 the "Control-Stack" are used in error-messages.
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:
+ *             control_stack_depth       Reset to zero.
+ *             do_loop_depth             Reset to zero.
+ *         Control-Stack, # of Items Popped:    All of them
+ *             
+ *      Error Detection:
+ *          Any item on the "Control-Stack" represents a Control-Structure
+ *              that was not completed when the Calling Condition was
+ *              encountered.  Error; identify the origin of the structure.
+ *          No special actions if  noerrors  is set.
+ *
+ *      Process Explanation:
+ *          Filter the duplicate messages caused by structures (e.g., DO)
+ *              that place two entries on the "Control-Stack" by testing
+ *              the  cs_not_dup  field of the "Top" "Control-Stack" item,
+ *              which would indicate double-entry...
+ *
+ *      Extraneous Remarks:
+ *          This is called before a definition of any kind, and after a 
+ *              colon-definition.  Flow-control constructs should *never*
+ *              be allowed to cross over between immediate-execution mode
+ *              and compilation mode.  Likewise, not between device-nodes.
+ *          Also, at the end of tokenization, there should not be any
+ *              unresolved flow-control constructs.
+ *
+ **************************************************************************** */
+
+void clear_control_structs( char *call_cond)
+{
+    clear_control_structs_to_limit( call_cond, 0);
+}

Added: fcode-utils/toke/flowcontrol.h
===================================================================
--- fcode-utils/toke/flowcontrol.h	                        (rev 0)
+++ fcode-utils/toke/flowcontrol.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,74 @@
+#ifndef _TOKE_FLOWCONTROL_H
+#define _TOKE_FLOWCONTROL_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      External Variables and Function Prototypes for Support Functions
+ *          used in tokenizing FORTH Flow-Control structures.
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */
+
+extern int control_stack_depth;
+ 
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+void emit_if( void );
+void emit_then( void );
+void emit_else( void );
+void emit_begin( void );
+void emit_again( void );
+void emit_until( void );
+void emit_while( void );
+void emit_repeat( void );
+void mark_do( void );
+void resolve_loop( void );
+void emit_case( void );
+void emit_of( void );
+void emit_endof( void );
+void emit_endcase( void );
+
+void announce_control_structs( int severity, char *call_cond,
+				          unsigned int abs_token_limit);
+void clear_control_structs_to_limit( char *call_cond,
+				          unsigned int abs_token_limit);
+void clear_control_structs( char *call_cond);
+
+#endif   /*  _TOKE_FLOWCONTROL_H    */

Modified: fcode-utils/toke/macros.c
===================================================================
--- fcode-utils/toke/macros.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/macros.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,6 +24,31 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Support functions for the  MACROS  vocabulary, implemented
+ *          as a TIC-Headerlist type of data structure, and linked in
+ *          to the Global Vocabulary.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          init_macros             Initialize the link-pointers in the
+ *                                      initial "Built-In" portion of
+ *                                      the  macros  vocabulary
+ *          add_user_macro          Add an entry to the  macros  vocabulary
+ *          skip_user_macro         Consume a Macro definition if Ignoring
+ *
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #if defined(__linux__) && ! defined(__USE_BSD)
@@ -32,107 +57,489 @@
 #include <string.h>
 #include <errno.h>

-#include "toke.h"
+#include "errhandler.h"
+#include "ticvocab.h"
+#include "stream.h"
+#include "scanner.h"
+#include "dictionary.h"
+#include "devnode.h"
+#include "macros.h"

-typedef struct macro {
-	u8  *name;
-	u8  *alias;
-	struct macro *next;
-} macro_t;
+/* **************************************************************************
+ *
+ *              Internal Static Variables
+ *          macros_tbl                    Initial array of "Built-In" Macros
+ *          number_of_builtin_macros      Number of "Built-In" Macro entries.
+ *
+ **************************************************************************** */

-static macro_t *macros=NULL;
+/* **************************************************************************
+ *
+ *      Revision History:
+ *          Thu, 27 Oct 2005 by David L. Paktor
+ *              Identify the macros that resolve to a single word.
+ *              Remove them from here and enter them as synonymous entries
+ *                  in the Tokens, Specials or Shared-words vocabularies.
+ *          Wed, 30 Nov 2005 by David L. Paktor
+ *              Allow user-definition of macros.
+ *          Fri, 06 Jan 2006 by David L. Paktor
+ *              Re-define the Macros as a TIC-Headerlist, and make them
+ *                  part of the Global Vocabulary
+ *
+ **************************************************************************** */

-char *lookup_macro(char *name)
+
+
+/* **************************************************************************
+ *
+ *      Function name:  macro_recursion_error
+ *      Synopsis:       Function that will go temporarily into the FUNCT
+ *                      field of a Macro's TIC-entry, to protect against
+ *                      recursive macro invocations.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield              Param field of the TIC-entry; unused.
+ *         Global Variables:
+ *             statbuf             The name being invoked erroneously.
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Printout:
+ *             Error Message.
+ *
+ *      Error Detection:
+ *          If this function is called, it is an ERROR
+ *
+ *      Extraneous Remarks:
+ *          This Tokenizer does not have the early-binding characterisitics
+ *              of FORTH; its Macros are strings, evaluated when invoked
+ *              rather than when they are defined.  A reference to a name
+ *              that matches the macro would cause recursion, possibly
+ *              infinite.  We will not allow that.
+ *
+ **************************************************************************** */
+
+static void macro_recursion_error( tic_param_t pfield)
 {
-	macro_t *curr;
-	
-	for (curr=macros; curr!=NULL; curr=curr->next)
-		if (!strcasecmp(name,(char *)curr->name))
-			break;
+    tokenization_error( TKERROR,
+	"Recursive invocation of macro named %s\n", statbuf);
+}

-	return curr?(char *)curr->alias:NULL;
+
+/* **************************************************************************
+ *
+ *      Function name:  eval_mac_string
+ *      Synopsis:       Function that goes into FUNCT field of a TIC-entry
+ *                      in the Macros list.  Protect against recursion.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield              Param field of the TIC-entry
+ *         Global Variables:
+ *             tic_found           The TIC-entry that has just been found;
+ *                                     it's the entry for this Macro.
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Global Variables:
+ *             report_multiline    Cleared to FALSE
+ *         Global Behavior:
+ *            The Macro will be evaluated as string input.
+ *
+ *      Error Detection:
+ *          An attempt at recursion will be detected because the FUNCT field
+ *              of the Macro entry will have been temporarily be replaced by
+ *              macro_recursion_error()
+ *
+ *      Process Explanation:
+ *          Save the address of the routine that is in the FUNCT field
+ *               of the entry for this Macro.  (Hey!  It's this routine...)
+ *          Replace the FUNCT field of the Macro entry with the Macro
+ *              Recursion Error Detection routine
+ *          Pass the address of the "resumption" routine and its argument
+ *              (the entry for this Macro), to  push_source()
+ *          Recast the type of the parameter field to a string
+ *          Make it the new Input Source Buffer.
+ *          Suspend multi-line warning; see comment in body of add_user_macro()
+ *              The multi-line warning flag is kept by  push_source()
+ *
+ *      Still to be done:
+ *          If an error is encountered during Macro evaluation, display
+ *              supplemental information giving the name of the Macro 
+ *              being run, and the file and line number in which it was
+ *              defined.
+ *          This will require changes to the way user Macros are added
+ *              and retained, and to the way error messages are displayed.
+ *
+ *      Revision History:
+ *          Updated Thu, 23 Feb 2006 by David L. Paktor
+ *              Do not process Macros (or, for that matter, User-defined
+ *                  Symbols or FLOADed files) with a routine that calls
+ *                  its own instance of  tokenize(), because the Macro
+ *                  (etc.) might contain a phrase (such as the start of
+ *                  a conditional) that must be terminated within the
+ *                  body of a file, thus causing an undeserved Error.
+ *                  Instead, they need to be handled in a more sophis-
+ *                  ticated way, tied in with the operation of get_word()
+ *                  perhaps, that will make a smooth transition between
+ *                  the body of the Macro and the resumption of processing
+ *                  the source file.  The end-of-file will only be seen
+ *                  at the end of an actual input file or when getting
+ *                  a delimited string.
+ *          Updated Fri, 24 Feb 2006 by David L. Paktor
+ *              Re-integrate Recursion Error Detection with the above.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *     In order to integrate Recursion Error Detection with the smooth
+ *         transition to resumption of processing the source file, we
+ *         need to create a "resumption" routine that will restore the
+ *         normal behavior of the macro after it's completed, by re-
+ *         instating the address of the normal Macro-invocation routine;
+ *         that routine, of course, is the one that passes the address
+ *         of the "resumption" routine to push_source().  In order to
+ *         get around this chicken-and-egg dilemma, we will create a
+ *         local static variable into which the address of the normal
+ *         Macro-invocation routine will be stored.  We actually only
+ *         need it once, but we'd rather avoid the overhead of checking,
+ *         every time, whether it has already been set; since it's always
+ *         the same, there's no harm in storing it every time.
+ *
+ **************************************************************************** */
+
+typedef void (*vfunct)();  /*  Pointer to function returning void  */
+static vfunct sav_mac_funct ;
+
+
+/* **************************************************************************
+ *
+ *     This "resumption" routine will be called by  pop_source()
+ *     The parameter is the Macro dictionary-entry whose behavior
+ *         is to be restored.
+ *
+ **************************************************************************** */
+
+static void mac_string_recovery( tic_hdr_t *macro_entry)
+{
+    (*macro_entry).funct = sav_mac_funct;
+    (*macro_entry).ign_func = sav_mac_funct;
 }

-int add_macro(char *name, char *alias)
+/* **************************************************************************
+ *
+ *     The normal Macro-invocation routine, at last...
+ *
+ **************************************************************************** */
+static void eval_mac_string( tic_param_t pfield)
 {
-	macro_t *curr;
+    int mac_str_len = strlen(pfield.chr_ptr);
+    /*  We can't use  (*tic_found).pfld_size  for the string length
+     *      because, if this is an alias for a macro, it will be zero...
+     */
+    /*  We can change that by de-coupling the decision to free the
+     *      param-field from whether pfld_size is non-zero (by intro-
+     *      ducing yet another field into the  tic_param_t  struct),
+     *      but we're not doing that today...
+     */

-	curr=malloc(sizeof(macro_t));
-	if(!curr) {
-		printf("Out of memory while adding macro.\n");
-		exit(-ENOMEM);
-	}
+    sav_mac_funct = *tic_found->funct;
+    (*tic_found).funct = macro_recursion_error;
+    (*tic_found).ign_func = macro_recursion_error;
+    push_source( mac_string_recovery, tic_found, FALSE);
+    report_multiline = FALSE;  /*  Must be done AFTER call to push_source()
+                                *      because  report_multiline  is part of
+                                *      the state that  push_source()  saves.
+				*/
+    init_inbuf( pfield.chr_ptr, mac_str_len);
+}

-	curr->next=macros;
-	curr->name=(u8 *)name;
-	curr->alias=(u8 *)alias;
+/* **************************************************************************
+ *
+ *     Builtin Macros do not need Recursion Error Detection.
+ *     Intermediate routine to convert parameter type.
+ *
+ **************************************************************************** */
+static void eval_builtin_mac( tic_param_t pfield)
+{
+    eval_string( pfield.chr_ptr);
+}
+/* **************************************************************************
+ *
+ *      Make a macro, because we might eliminate this layer later on.
+ *
+ **************************************************************************** */
+#define EVAL_MAC_FUNC  eval_mac_string
+#define BUILTIN_MAC_FUNC  eval_builtin_mac

-	macros=curr;
-	return 0;
+/* **************************************************************************
+ *
+ *      Initialization macro definition
+ *
+ **************************************************************************** */
+
+#define BUILTIN_MACRO(nam, alias) BUILTIN_MAC_TIC(nam, BUILTIN_MAC_FUNC, alias )
+
+static tic_mac_hdr_t macros_tbl[] = {
+	BUILTIN_MACRO( "(.)",		"dup abs <# u#s swap sign u#>") ,
+
+
+	BUILTIN_MACRO( "?",		"@ .") ,
+	BUILTIN_MACRO( "1+",		"1 +") ,
+	BUILTIN_MACRO( "1-",		"1 -") ,
+	BUILTIN_MACRO( "2+",		"2 +") ,
+	BUILTIN_MACRO( "2-",		"2 -") ,
+
+	BUILTIN_MACRO( "accept",      "span @ -rot expect span @ swap span !") ,
+	BUILTIN_MACRO( "allot", 	"0 max 0 ?do 0 c, loop") ,
+	BUILTIN_MACRO( "blank", 	"bl fill") ,
+	BUILTIN_MACRO( "carret",	"h# d") ,
+	BUILTIN_MACRO( ".d",		"base @ swap h# a base ! . base !") ,
+	BUILTIN_MACRO( "decode-bytes",  ">r over r@ + swap r@ - rot r>") ,
+	BUILTIN_MACRO( "3drop", 	"drop 2drop") ,
+	BUILTIN_MACRO( "3dup",		"2 pick 2 pick 2 pick") ,
+	BUILTIN_MACRO( "erase", 	"0 fill") ,
+	BUILTIN_MACRO( ".h",		"base @ swap h# 10 base ! . base !") ,
+	BUILTIN_MACRO( "linefeed",	"h# a") ,
+
+	BUILTIN_MACRO( "s.",		"(.) type space") ,
+	BUILTIN_MACRO( "space", 	"bl emit") ,
+	BUILTIN_MACRO( "spaces",	"0 max 0 ?do space loop") ,
+	BUILTIN_MACRO( "(u.)",		"<# u#s u#>") ,
+	BUILTIN_MACRO( "?leave",	"if leave then"),
+};
+
+static const int number_of_builtin_macros =
+	 sizeof(macros_tbl)/sizeof(tic_mac_hdr_t);
+
+/* **************************************************************************
+ *
+ *      Function name:  init_macros
+ *      Synopsis:       Initialize the link-pointers in the "Built-In"
+ *                          portion of the  macros  vocabulary, dynamically.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             tic_vocab_ptr                 Pointer to Global Vocab Pointer
+ *         Global Variables:
+ *             macros_tbl                    Initial "Built-In" Macros array
+ *             number_of_builtin_macros      Number of "Built-In" Macro entries
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Global Variables:    
+ *             The link-fields of the initial "Built-In" Macros array entries
+ *                  will be filled in.
+ *         Supplied Pointers:
+ *             *tic_vocab_ptr                Updated to "tail" of Macros array
+ *
+ **************************************************************************** */
+
+void init_macros( tic_hdr_t **tic_vocab_ptr )
+{
+    init_tic_vocab( (tic_hdr_t *)macros_tbl,
+        number_of_builtin_macros,
+	    tic_vocab_ptr );
 }

-void init_macros(void) 
+
+/* **************************************************************************
+ *
+ *      Function name:  print_if_mac_err
+ *      Synopsis:       Report a user-macro definition error, if so be.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             failure             TRUE if error was detected
+ *             func_cpy            STRDUP() of function name, for error message
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Memory Freed
+ *             Contents of func_cpy, error or not.
+ *         Printout:
+ *             Error message, if  failure  is TRUE.
+ *
+ **************************************************************************** */
+static void print_if_mac_err( bool failure, char *func_cpy)
 {
-	add_macro( "eval",	"evaluate");
-	add_macro( "(.)", 	"dup abs <# u#s swap sign u#>");
-	add_macro( "<<", 	"lshift");
-	add_macro( ">>",	"rshift");
-	add_macro( "?",		"@ .");
-	add_macro( "1+",	"1 +");
-	add_macro( "1-",	"1 -");
-	add_macro( "2+",	"2 +");
-	add_macro( "2-",	"2 -");
-	/* add_macro( "abort"",	"-2 throw"); */
-	add_macro( "accept",	"span @ -rot expect span @ swap span !");
-	add_macro( "allot",	"0 max 0 ?do 0 c, loop");
-	add_macro( "blank",	"bl fill");
-	add_macro( "/c*",	"chars");
-	add_macro( "ca1+",	"char+");
-	add_macro( "carret",	"h# d");
-	add_macro( ".d",	"base @ swap h# a base ! . base !");
-	add_macro( "decode-bytes", ">r over r@ + swap r@ - rot r>");
-	add_macro( "3drop",	"drop 2drop");
-	add_macro( "3dup",	"2 pick 2 pick 2 pick");
-	add_macro( "erase",	"0 fill");
-	add_macro( "false",	"0");
-	add_macro( ".h",	"base @ swap h# 10 base ! . base !");
-	add_macro( "linefeed",	"h# a");
-	add_macro( "/n*",	"cells");
-	add_macro( "na1+",	"cell+");
-	add_macro( "not",	"invert");
-	add_macro( "s.",	"(.) type space");
-	add_macro( "space",	"bl emit");
-	add_macro( "spaces",	"0 max 0 ?do space loop");
-	add_macro( "struct",	"0");
-	add_macro( "true",	"-1");
-	add_macro( "(u.)",	"<# u#s u#>");
+    if ( failure )
+    {
+	tokenization_error( TKERROR,
+	   "%s directive expects name and definition on the same line\n",
+	       strupr(func_cpy));
+    }
+    free( func_cpy);
+}

-	/* H.7 FCode name changes */
-	add_macro( "decode-2int",		"parse-2int");
-	add_macro( "delete-attribute",		"delete-property");
-	add_macro( "get-inherited-attribute",	"get-inherited-property");
-	add_macro( "get-my-attribute",		"get-my-property");
-	add_macro( "get-package-attribute",	"get-package-property");
-	add_macro( "attribute",	  "property");
-	add_macro( "flip",	  "wbflip");
-	add_macro( "is",	  "to");
-	add_macro( "lflips",	  "lwflips");
-	add_macro( "map-sbus",	  "map-low");
-	add_macro( "u*x",	  "um*");
-	add_macro( "version",	  "fcode-revision");
-	add_macro( "wflips",	  "wbflips");
-	add_macro( "x+",	  "d+");
-	add_macro( "x-",	  "d-");
-	add_macro( "xdr+",	  "encode+");
-	add_macro( "xdrbytes",	  "encode-bytes");
-	add_macro( "xdrint",	  "encode-int");
-	add_macro( "xdrphys",	  "encode-phys");
-	add_macro( "xdrstring",	  "encode-string");
-	add_macro( "xdrtoint",	  "decode-int");
-	add_macro( "xdrtostring", "decode-string");
-	add_macro( "xu/mod",	  "um/mod");
+
+/* **************************************************************************
+ *
+ *      Function name:  add_user_macro
+ *      Synopsis:       Parse input and add a user-defined Macro.
+ *
+ *      Associated Tokenizer directive:        [MACRO]
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Global Variables:
+ *             pc                      Input-source Scanning pointer
+ *             statbuf                 Symbol retrieved from input stream.
+ *             in_tokz_esc             TRUE if in "Tokenizer-Escape" mode
+ *             current_definitions     Pointer to Current Vocabulary pointer,
+ *                                         either Global or Current Device-Node
+ *             tokz_esc_vocab         "Tokenizer Escape" Vocab pointer
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Global Variables:
+ *             *current_definitions   { One of these will point  }
+ *             tokz_esc_vocab         {     to the new entry     }
+ *         Memory Allocated:
+ *             Copy of directive, for error message
+ *             Copy of Macro name
+ *             Copy of Macro body
+ *             Memory for the new entry will be allocated by support routine.
+ *         When Freed?
+ *             Copy of directive:  When error might be reported.
+ *             Macro name, body and entry:  Upon end of tokenization, or when
+ *                 RESET-SYMBOLS is issued in the same mode and Scope as when
+ *                 the Macro was defined. 
+ *
+ *      Error Detection:
+ *          At least two words in the input stream are expected to be on
+ *              the same line as the directive.  The  get_word_in_line()
+ *              and  get_rest_of_line()  routines will check for that;
+ *              we will issue the Error Message for either condition.
+ *          Check if the Macro name is a duplicate; warn_if_duplicate()
+ *              routine will issue message.
+ *
+ *      Process Explanation:
+ *          We start just after the directive has been recognized.
+ *          Get one word in line -- this is the macro name
+ *          Get input to end of line.  This is the "body" of the macro.
+ *          Add the Macro to the Current vocab, using support routine.
+ *          Set the definer field to MACRO_DEF and the Function to the
+ *              same one that's used for the built-in macros.
+ *          User-defined Macros may need to be processed while ignoring
+ *              (because they might include conditional-operators, etc.)
+ *              We will set the ign_func the same as the active function.
+ *
+ *      To be considered:
+ *          Do we want to do further filtration? 
+ *              Remove comments?
+ *              Compress whitespace?
+ *              Allow backslash at end of line to continue to next line?
+ *
+ *      Extraneous Remarks:
+ *          The scope of User-Macro definitions will follow the same rules
+ *              as all other definition types:  if Device-Definitions are
+ *              in effect, the scope of the new Macro definition will be
+ *              confined to the current Device-Node; if Global-Definitions
+ *              are in effect when it is defined, its scope will be Global;
+ *              if it was declared when we were in "Tokenizer Escape" mode,
+ *              then its scope will be limited to "Tokenizer Escape" mode.
+ *
+ **************************************************************************** */
+
+/*  This pointer is exported to this file only  */
+extern tic_hdr_t *tokz_esc_vocab ;
+
+void add_user_macro( void)
+{
+    char *macroname;
+    char *macrobody;
+    bool failure = TRUE;
+
+    /*  Copy of function name, for error message  */
+    char *func_cpy = strdup( statbuf);
+
+    if ( get_word_in_line( NULL ) )
+    {
+        /*  This is the Macro name  */
+	macroname = strdup( statbuf);
+
+	if ( INVERSE(get_rest_of_line() ) )
+	{
+	    /*  No body on line  */
+	    free( macroname);
    
-	/* non standard but often used macros */
-	add_macro( "name",	"device-name");
-	add_macro( "endif",	"then");
+	}else{
+	    /*  We have valid Macro body on line  */
+	    int mac_body_len = 0;
+
+	    tic_hdr_t **target_vocab = current_definitions;
+	    if ( in_tokz_esc ) target_vocab = &tokz_esc_vocab ;
+
+            warn_if_duplicate( macroname);
+	    trace_creation( MACRO_DEF, macroname);
+
+	    /*  Tack on a new-line, so that a remark will appear
+	     *      to be properly terminated.   This might trigger
+	     *      an undeserved multi-line warning if the Macro
+	     *      is an improperly terminated quote; we will work
+	     *      around that problem by temporarily suspending
+	     *      multi-line warnings during macro processing.
+	     */
+	    strcat( statbuf, "\n");
+	    macrobody = strdup( statbuf);
+	    mac_body_len = strlen(macrobody);
+
+	    add_tic_entry( macroname, EVAL_MAC_FUNC,
+	                       (TIC_P_DEFLT_TYPE)macrobody,
+			           MACRO_DEF, mac_body_len,
+				       EVAL_MAC_FUNC, target_vocab );
+	    failure = FALSE;
+	}
+    }
+
+    print_if_mac_err( failure, func_cpy);
 }
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_user_macro
+ *      Synopsis:       Consume the text of a user-defined Macro from the
+ *                          Input Stream, with no processing.  (Called when
+ *                          a user-Macro definer occurs in a segment that
+ *                          is being Ignored.)
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield             "Parameter field" pointer, to satisfy
+ *                                    the calling convention, but not used
+ *         Global Variables:
+ *             statbuf            Word currently being processed.
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *
+ *      Error Detection:
+ *          At least two words in the input stream are expected to be on
+ *              the same line as the user-Macro definer, same as when the
+ *              directives occurs in a segment that is not being Ignored.
+ *              The  get_word_in_line()  and  get_rest_of_line()  routines
+ *              will check for condition., we will issue the Error Message.
+ *
+ *      Process Explanation:
+ *          We need to protect against the case of a macro-definition that
+ *              invokes a directive that alters Conditional processing...
+ *
+ **************************************************************************** */
+void skip_user_macro( tic_bool_param_t pfield )
+{
+    bool failure = TRUE;
+    char *func_cpy = strdup( statbuf);
+    if ( get_word_in_line( NULL ) )
+    {
+	if ( get_rest_of_line() )
+	{
+	    failure = FALSE;
+	}
+    }
+
+    print_if_mac_err( failure, func_cpy);
+
+}

Added: fcode-utils/toke/macros.h
===================================================================
--- fcode-utils/toke/macros.h	                        (rev 0)
+++ fcode-utils/toke/macros.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,58 @@
+#ifndef _TOKE_MACROS_H
+#define _TOKE_MACROS_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Prototypes for functions that operate on the  MACROS vocabulary,
+ *          which is implemented as a String-Substitution-type vocab.
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+
+#include "types.h"
+#include "ticvocab.h"
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+void init_macros( tic_hdr_t **tic_vocab_ptr );
+void add_user_macro( void);
+void skip_user_macro( tic_bool_param_t pfield );
+#if  0  /*  What's this doing here?  */
+char *lookup_macro(char *name);
+bool exists_as_macro(char *name);
+bool create_macro_alias( char *new_name, char *old_name );
+void reset_macros_vocab( void );
+#endif  /*  What's this doing here?  */
+
+#endif   /*  _TOKE_MACROS_H    */

Added: fcode-utils/toke/nextfcode.c
===================================================================
--- fcode-utils/toke/nextfcode.c	                        (rev 0)
+++ fcode-utils/toke/nextfcode.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,708 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Functions for Managing FCode Assignment Pointer and for
+ *          Detection of Overlapping-FCode Error in Tokenizer
+ *
+ *      (C) Copyright 2006 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      The fact that the user is able to specify, at any time, the
+ *          next FCode number to be assigned introduces the risk that
+ *          a symbol might inadvertently be assigned an FCode that is
+ *          still in use by another symbol, which could lead to bizarre
+ *          and hard-to-trace failures at run-time.
+ *
+ *      This module will support a means whereby to detect and report,
+ *          as an Error, overlapping FCode assignments.
+ *
+ *      The task is further complicated by the fact that, under some
+ *          circumstances, "recycling" a range of FCodes can be done
+ *          safely, and the programmer may do so intentionally.
+ *
+ *      We will define a way for the programmer to specify that FCode
+ *          assignments are being intentionally "recycled", and that
+ *          overlaps with previously-assigned FCodes are not to be
+ *          treated as Errors.  We will not attempt to analyze whether
+ *          it is, indeed, safe to do so; the programmer who does this
+ *          is presumed to be sufficiently responsible.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          reset_fcode_ranges   (Re-)Initialize the range of assignable
+ *                                   FCode numbers and clear the records
+ *                                   of prevously-assigned ranges of FCodes
+ *          list_fcode_ranges    Display a final report of assigned FCode
+ *                                   ranges at end of tokenization or when
+ *                                   the  reset_fcodes()  routine is called.
+ *          set_next_fcode       Set the start of a new Range of FCode
+ *                                   numbers to be assigned.
+ *          assigning_fcode      The next FCode number is about to be assigned;
+ *                                   test for out-of-bounds, overlap, etc.
+ *                                   errors.
+ *          bump_fcode           Increment the next FCode number prior to the
+ *                                    next assignment.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Revision History:
+ *          06 Jun 06  -- Need became apparent after verification test
+ *                            against existing device-drivers.
+ *              
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Detect and report when the Current Range stops overlapping
+ *              one particular Range and starts overlapping another.
+ *
+ *          Detect and report when the Current Range overlaps more than
+ *              one Range at a time (e.g., if other Ranges themselves
+ *              overlap, and the Current Range is stepping through them)
+ *              but, again, only display one message per overlapped Range.
+ *
+ **************************************************************************** */
+
+
+
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              iname              Name of current input file
+ *              lineno             Line Number within current input file
+ *
+ **************************************************************************** */
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "toke.h"
+#include "nextfcode.h"
+#include "errhandler.h"
+#include "stream.h"
+#include "scanner.h"
+
+
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *              nextfcode          The next FCode-number to be assigned
+ *
+ **************************************************************************** */
+
+u16  nextfcode;         /*  The next FCode-number to be assigned              */
+
+/* **************************************************************************
+ *
+ *      Internal (Static) Structure:
+ *          fcode_range_t         The record of a Range of assigned FCodes
+ *
+ *   Fields:
+ *       fcr_start            FCode number at the start of a this Range
+ *       fcr_end              Last FCode number in this Range that has
+ *                                actually been assigned.  0 if none made yet.
+ *       fcr_infile           File name of where this Range was started
+ *       fcr_linenum          Line number where this Range was started
+ *       fcr_not_lapped       TRUE if an overlap error has not been reported
+ *                                against this Range.  Use this to prevent
+ *                                issuing an Error message every time an
+ *                                overlapping FCode is assigned; once per
+ *                                Range is sufficient.
+ *       fcr_next             Pointer to following entry in linked-list.
+ *
+ *   We will keep the list of assigned FCode Ranges as a forward-linked,
+ *       rather than the more usual backward-linked, list for convenience
+ *       at listing-time.  Otherwise, it doesn't make much difference.
+ *
+ **************************************************************************** */
+
+typedef struct fcode_range
+    {
+        u16                 fcr_start;
+	u16                 fcr_end ;
+	char               *fcr_infile;
+	int                 fcr_linenum;
+	bool                fcr_not_lapped;
+	struct fcode_range *fcr_next;
+    }  fcode_range_t ;
+
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *     ranges_exist              TRUE if FCode Ranges have been created;
+ *                                   Prevents unnecessary overlap checking.
+ *     changes_listed            TRUE if no changes to any of the Ranges
+ *                                   have taken place since the last time
+ *                                   a Listing was displayed.  Prevents
+ *                                   unnecessary repetions (e.g., if a
+ *                                    reset_fcode_ranges()  was called
+ *                                   right after an operation that causes
+ *                                   a Listing
+ *
+ *          These four apply if no FCode Ranges have been created:
+ *
+ *     range_start               First FCode in the current first (only) range.
+ *     range_end                 Last FCode in current first (only) range that
+ *                                   has actually been assigned.  0 if none yet.
+ *     first_fcr_infile          File name of where first (only) range started.
+ *     first_fcr_linenum         Line number where first (only) range started.
+ *
+ *          These two apply after FCode Ranges have been created:
+ *
+ *     first_fc_range            Pointer to the first entry in the linked
+ *                                   list of FCode Ranges.
+ *     current_fc_range          Pointer to the entry in the linked list of
+ *                                   Ranges that contains the Current Range.
+ *
+ **************************************************************************** */
+
+static bool           ranges_exist      = FALSE;
+static bool           changes_listed    = FALSE;
+
+static u16            range_start       = FCODE_START;
+static u16            range_end         = 0;
+static char          *first_fcr_infile  = NULL;
+static int            first_fcr_linenum = 0;
+
+static fcode_range_t *first_fc_range    = NULL;
+static fcode_range_t *current_fc_range  = NULL;
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_fcode_ranges
+ *      Synopsis:       (Re-)Initialize the range of assignable FCode numbers
+ *                          and clear the records of prevously-assigned Ranges
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Global Variables:
+ *             iname                    Name of current input file
+ *             lineno                   Line Number within current input file
+ *         Local Static Variables:
+ *             ranges_exist             TRUE if FCode Ranges have been created
+ *             first_fc_range           Pointer to First FCode Range
+ *             current_fc_range         Pointer to the current Range
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Global Variables:
+ *             nextfcode                Initialized to Standard start value
+ *                                          for user-assigned FCodes
+ *         Local Static Variables:
+ *             ranges_exist             Cleared to FALSE
+ *             changes_listed           Reset to FALSE
+ *             range_start              Reset to Standard start value
+ *             range_end                Reset to 0
+ *             first_fcr_infile         Copy of iname
+ *             first_fcr_linenum        Copy of lineno
+ *             first_fc_range           Reset to NULL
+ *             current_fc_range         Reset to NULL
+ *         Memory Allocated
+ *             For  first_fcr_infile , which will have a copy of  iname
+ *         When Freed?
+ *             The next time this routine is called.
+ *         Memory Freed
+ *             Any FCode Ranges that were in effect will be freed, and the
+ *                 memory for their  fcr_infile  fields will be freed.
+ *             If  iname  has changed, memory for  first_fcr_infile  will
+ *                 be freed before the new copy is made.
+ *
+ *      Process Explanation:
+ *          This will be called either as part of normal initialization
+ *              or in response to a User-issued directive.
+ *          In the former case, it may be called twice:  once before any
+ *              Source is examined, and once again in response to the first
+ *              (and only to the first) invocation of an FCode-Starter, at
+ *              which time the input file name and line number will be updated.
+ *          In the latter case, the calling routine will be responsible for
+ *              displaying any Advisory Messages and listings of previously
+ *              assigned FCode Ranges.
+ *
+ **************************************************************************** */
+
+void reset_fcode_ranges( void)
+{
+    if ( ranges_exist )
+    {
+	while ( current_fc_range != NULL )
+	{
+	    current_fc_range = first_fc_range->fcr_next;
+	    free( first_fc_range->fcr_infile);
+	    free( first_fc_range);
+	    first_fc_range = current_fc_range;
+	}
+	ranges_exist = FALSE;
+    }else{
+	if ( first_fcr_infile != NULL )
+	{
+	    if ( strcmp( first_fcr_infile, iname) != 0 )
+	    {
+		free( first_fcr_infile);
+		first_fcr_infile = NULL;
+	    }
+	}
+    }
+
+    changes_listed    = FALSE;
+    range_start       = FCODE_START;
+    range_end         = 0;
+
+    if ( first_fcr_infile == NULL )
+    {
+	first_fcr_infile = strdup( iname);
+    }
+    first_fcr_linenum = lineno;
+    nextfcode         = FCODE_START;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  list_fcode_ranges
+ *      Synopsis:       Display an Advisory of assigned FCode ranges at
+ *                          end of tokenization or upon reset_fcodes() 
+ *
+ *      Inputs:
+ *         Parameters:
+ *             final_tally               TRUE if printing a final tally at
+ *                                           end of tokenization.  
+ *         Global Variables:
+ *             verbose                   Do not print anything if not set.
+ *             nextfcode                 FCode next after last-assigned
+ *         Local Static Variables:
+ *             changes_listed            If TRUE, only print new-line.
+ *             ranges_exist              TRUE if more than one Range exists.
+ *             range_start               FCode at start of only range
+ *             range_end                 Last FCode in only range; 0 if none.
+ *             first_fc_range            Ptr to start of FCode Ranges list
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Local Static Variables:
+ *             changes_listed           Set to TRUE. 
+ *         Printout:
+ *            Printed to Standard Out on final tally, or to STDERR otherwise.
+ *            One of three formats:
+ *            1   No FCodes assigned.
+ *            2   Last assigned FCode = 0xXXX
+ *            3   FCodes assigned:
+ *                    [ No FCodes assigned in the range that started ... ]
+ *                    [ From 0xXXX to 0xYYY ** in the range that started ... ]
+ *                        (** = Indicator if the Range has an Overlap Error.)
+ *
+ *      Process Explanation:
+ *          This is called to complete an Advisory or Standard-Out Message
+ *              that doesn't end in a new-line.
+ *
+ *      Extraneous Remarks:
+ *          If we ever decide not to keep entries for Ranges in which no
+ *              assignments were made, let's not remove the code that lists
+ *              them.  It's harmless to keep it around, and we remain ready...
+ *
+ **************************************************************************** */
+
+void list_fcode_ranges( bool final_tally)
+{
+    if ( verbose )
+    {
+	FILE *message_dest = ( final_tally ? stdout : ERRMSG_DESTINATION );
+	if ( changes_listed )
+	{
+	    fprintf(message_dest, "\n");
+	}else{
+	    changes_listed = TRUE;
+
+	    if ( INVERSE(ranges_exist) )
+	    {   /*  List the first and only range  */
+		if ( range_end == 0 )
+		{
+		    fprintf(message_dest, "No FCodes assigned.\n");
+		}else{
+		    if ( range_start == FCODE_START )
+		    {
+			fprintf(message_dest,
+			    "Last assigned FCode = 0x%x\n", range_end);
+		    }else{
+			fprintf(message_dest,
+			    "FCodes assigned:  0x%x to 0x%x\n",
+				range_start, range_end);
+		    }
+		}
+		/*  We are done listing the first and only range  */
+	    }else{   /*  List the collection of Ranges  */
+
+		/*  Pionter to function returning void  */
+		typedef void (*vfunct)();
+
+		/*  Function for the  started_at()  part of the message  */
+		vfunct start_at_funct =
+	            ( final_tally ? print_started_at : started_at );
+
+
+		fcode_range_t *next_range = first_fc_range;
+
+		fprintf(message_dest, "FCodes assigned:\n");
+
+		while ( next_range != NULL )
+		{
+		    if ( next_range->fcr_end == 0 )
+		    {
+			fprintf(message_dest, "    None assigned");
+		    }else{
+			fprintf(message_dest, "    From 0x%x to 0x%x",
+			    next_range->fcr_start, next_range->fcr_end );
+			if ( INVERSE( next_range->fcr_not_lapped) )
+			{
+			    fprintf(message_dest, " ***Overlap***" );
+			}
+		    }
+		    fprintf(message_dest, " in the range");
+		    (*start_at_funct)(
+			next_range->fcr_infile, next_range->fcr_linenum );
+
+		    next_range = next_range->fcr_next;
+		}
+	    }
+	}
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  set_next_fcode
+ *      Synopsis:       Set the start of a new Range of FCode numbers
+ *
+ *      Inputs:
+ *         Parameters:
+ *             new_fcode                 Start of the new range of FCodes
+ *         Global Variables:
+ *             nextfcode                 The next FCode-number to be assigned
+ *             iname                     Name of current input file
+ *             lineno                    Line Number within current input file
+ *         Local Static Variables:
+ *             ranges_exist              TRUE if FCode Ranges have been created
+ *             range_start               First FCode in the current (first
+ *                                           and only) range
+ *             range_end                 Last FCode in only range; 0 if none.
+ *             current_fc_range          Pointer to the current Range if there
+ *                                           are more than one.
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:
+ *             nextfcode                 Set to the start of the new range
+ *         Local Static Variables:
+ *             ranges_exist              May be set to TRUE
+ *             first_fc_range            May be initialized
+ *             current_fc_range          May be initialized or moved
+ *             first_fcr_infile          May be updated, may be made NULL
+ *             first_fcr_linenum         May be updated, may be made irrelevant
+ *             range_start               May be set to start of the new range
+ *                                           or rendered irrelevant
+ *             range_end                 Reset to 0 (by  reset_fcode_ranges() )
+ *             changes_listed            Reset to FALSE
+ *         Memory Allocated
+ *             For new Range data structure; for copy of  iname
+ *         When Freed?
+ *             By reset_fcode_ranges()
+ *
+ *      Error Detection:
+ *          Not here.
+ *          The calling routine will detect and report attempts to set an
+ *             illegal new range. 
+ *          Overlap with earlier Ranges will be detected and reported when
+ *              the FCode is actually Assigned.
+ *
+ *      Process Explanation:
+ *          The calling routine will not call this routine with a new starting
+ *              FCode that is not legal per the Standard.
+ *          It may call with a new starting FCode that is equal to  nextfcode
+ *          If no Ranges exist yet, and the new starting FCode is equal to
+ *              the current value of  nextfcode , this is a continuation of
+ *              the first and only range; do not change the file name or line
+ *              number; just go away.
+ *          If no Ranges exist yet, and no FCode assignments have been made
+ *              in the current range, then this is a new start for the first
+ *              and only range; detect the latter condition by range_end == 0
+ *              Call the  reset_fcode_ranges()  routine to update the file name
+ *              and line number, then update the remaining variables for the
+ *              current (first and only) range, and you are done here.
+ *          If no Ranges exist yet, and if FCode assignments _have_ been made
+ *              in the current (first and only) range, create a data structure
+ *              for the first (and now no longer only) Range and point both
+ *              the  first_fc_range  and  current_fc_range  pointers at it.
+ *              Set the  ranges_exist  flag to TRUE.
+ *          If Ranges exist, whether from being newly-created, above, or from
+ *              earlier, create a data structure for the new Current Range
+ *              and move the  current_fc_range  pointer to point at it.  If
+ *              the new starting FCode is equal to  nextfcode  we still want
+ *              to create a new Range that will be listed separately.
+ *          If no assignments were made within the Current Range, we will not
+ *              overwrite or delete it; it will be listed at the appropriate
+ *              time, and will be harmless in the overlap test.
+ *
+ *      Extraneous Remarks:
+ *          We will trade off the strict rules of structured code here,
+ *              in exchange for ease of coding.
+ *
+ **************************************************************************** */
+
+void set_next_fcode( u16  new_fcode)
+{
+    if ( INVERSE( ranges_exist) )
+    {    /*  The current range is the first and only.  */
+
+	if ( new_fcode == nextfcode )
+	{
+	    /*  Do nothing here  */
+	    return;
+	}
+
+	if ( range_end == 0 )
+	{   /*  No FCode assignments have been made in the current range  */
+	    /*  This is still the first and only range.                   */
+
+	    reset_fcode_ranges();   /*  Update file name and line number  */
+	    range_start = new_fcode;
+	    nextfcode = new_fcode;
+
+	    /*  We are done here  */
+	    return;
+
+	}else{  /*  Create the data structure for the first Range  */
+	    first_fc_range = safe_malloc( sizeof( fcode_range_t),
+				 "creating first FCode Range" );
+	    first_fc_range->fcr_start        = range_start;
+	    first_fc_range->fcr_end          = range_end;
+	    first_fc_range->fcr_infile       = first_fcr_infile;
+	    first_fc_range->fcr_linenum      = first_fcr_linenum;
+	    first_fc_range->fcr_not_lapped   = TRUE;
+	    first_fc_range->fcr_next         = NULL;
+
+	    first_fcr_infile  = NULL;
+	    first_fcr_linenum = 0;
+	    range_start       = FCODE_START;
+	    range_end         = 0;
+
+	    current_fc_range  = first_fc_range;
+
+	    ranges_exist      = TRUE;
+	}
+    }
+
+    /*  Previous Ranges exist now for sure!  */
+    current_fc_range->fcr_next  = safe_malloc( sizeof( fcode_range_t),
+				      "creating new FCode Range" );
+    current_fc_range = current_fc_range->fcr_next;
+
+    nextfcode                         = new_fcode;
+    current_fc_range->fcr_start       = nextfcode;
+    current_fc_range->fcr_end         = 0;
+                                  /*  Will be filled in by first assignment  */
+    current_fc_range->fcr_infile      = strdup( iname);
+    current_fc_range->fcr_linenum     = lineno;
+    current_fc_range->fcr_not_lapped  = TRUE;
+    current_fc_range->fcr_next        = NULL;
+
+    changes_listed                    = FALSE;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  find_overlap
+ *      Synopsis:       Compare the FCode under test against existing
+ *                          FCode Ranges and return a pointer to the
+ *                          Range against which it overlaps, if any.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             test_fcode                 FCode to be tested
+ *         Global Variables:
+ *             
+ *         Local Static Variables:
+ *             ranges_exist               If not TRUE, no need to test
+ *             first_fc_range             Start of Ranges to test
+ *             current_fc_range           Do not test Current Range
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to FCode Range in which an
+ *                                            overlap exists, or NULL if none.
+ *
+ *      Error Detection:
+ *          The calling routine will treat any findings as it deems appropriate.
+ *
+ *      Process Explanation:
+ *          A Range within which no assignments were made will never "hit"
+ *              the overlap test because its  fcr_end  field will be zero
+ *              and its  fcr_start  field will be non-zero; there's no
+ *              number that will be "between" them!
+ *
+ **************************************************************************** */
+
+static fcode_range_t *find_overlap( u16 test_fcode)
+{
+    fcode_range_t *retval = NULL;
+    if ( ranges_exist )
+    {
+	fcode_range_t *test_range = first_fc_range;
+	while ( test_range != current_fc_range )
+	{
+	    if ( ( test_fcode <= test_range->fcr_end ) &&
+	         ( test_fcode >= test_range->fcr_start )  )
+	    {
+		retval = test_range;
+		break;
+	    }
+	    test_range = test_range->fcr_next;
+	}
+    }
+
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  assigning_fcode
+ *      Synopsis:       Commit the next FCode number to be assigned;
+ *                          test for out-of-bounds, overlap, etc. errors.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             
+ *         Global Variables:
+ *             nextfcode
+ *         Local Static Variables:
+ *             ranges_exist         TRUE if FCode Ranges have been created
+ *             first_fc_range       First entry in linked list of Ranges.
+ *             current_fc_range     List entry for Current Range.
+ *
+ *      Outputs:
+ *         Returned Value:                  NONE
+ *         Global Variables:
+ *         Local Static Variables:
+ *             changes_listed               Reset to FALSE
+ *                    One of these two will be set to  nextfcode 
+ *             range_end                       ... if  ranges_exist  is FALSE
+ *             current_fc_range->fcr_end       ... if  ranges_exist  is TRUE
+ *
+ *      Error Detection:
+ *          FATAL if the value of  nextfcode  is larger than the legal
+ *              maximum for an FCode
+ *          ERROR if the value of  nextfcode   falls within one of the
+ *              existing Ranges (other than the current one, of course)
+ *
+ **************************************************************************** */
+
+void assigning_fcode( void)
+{
+    if ( nextfcode > FCODE_LIMIT )
+    {
+	/*  Let's give a last summarization before we crap out */
+	tokenization_error( INFO, "");
+	list_fcode_ranges( FALSE);
+
+	tokenization_error( FATAL,
+	    "Too many definitions.  "
+	    "Assigned FCode exceeds limit "
+	    "specified by IEEE-1275.");
+	/*
+	 *  No need to  return()  from here.
+	 *  FATAL error exits program.
+	 */
+    }
+
+    changes_listed = FALSE;
+
+    if ( INVERSE(ranges_exist) )
+    {    /*  No Overlap Error checking needed here.  */
+	range_end = nextfcode;
+    }else{
+	current_fc_range->fcr_end = nextfcode;
+
+	/*   Detect and report Overlap Error only once per Range  */
+	if ( current_fc_range->fcr_not_lapped )
+	{
+	    fcode_range_t *found_lap = find_overlap( nextfcode);
+	    if ( found_lap != NULL )
+	    {
+		tokenization_error( TKERROR,
+		    "Assigning FCode of 0x%x, "
+			"which overlaps the range", nextfcode);
+		started_at( found_lap->fcr_infile, found_lap->fcr_linenum);
+
+		current_fc_range->fcr_not_lapped = FALSE;
+	    }
+	}
+    }
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  bump_fcode
+ *      Synopsis:       Increment the next assignable FCode number
+ *                          prior to the next assignment.
+ *
+ *      Inputs:
+ *         Parameters:                   NONE
+ *         Global Variables:
+ *             nextfcode                 The next FCode-number to be assigned
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Global Variables:
+ *             nextfcode                 Incremented
+ *
+ *      Extraneous Remarks:
+ *          This looks like a no-brainer now, but if we ever need this
+ *              function to perform any more sophisticated background
+ *              activity, we can limit the scope of our modifications
+ *              to this routine.
+ *
+ **************************************************************************** */
+
+void bump_fcode( void)
+{
+    nextfcode++;
+}

Added: fcode-utils/toke/nextfcode.h
===================================================================
--- fcode-utils/toke/nextfcode.h	                        (rev 0)
+++ fcode-utils/toke/nextfcode.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,80 @@
+#ifndef _TOKE_NEXTFCODE_H
+#define _TOKE_NEXTFCODE_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Function Prototypes for Managing FCode Assignment Pointer 
+ *         and for Detection of Overlapping-FCode Error in Tokenizer
+ *
+ *      (C) Copyright 2006 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "types.h"
+
+
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */
+
+extern u16  nextfcode;         /*  The next FCode-number to be assigned      */
+
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+void reset_fcode_ranges( void);
+void list_fcode_ranges( bool final_tally);
+void set_next_fcode( u16  new_fcode);
+void assigning_fcode( void);
+void bump_fcode( void);
+
+/* **************************************************************************
+ *
+ *          Macros:
+ *
+ *          FCODE_START             Standard-specified starting number for
+ *                                      user-generated FCodes.
+ *
+ *          FCODE_LIMIT             Standard-specified maximum number for
+ *                                      FCodes of any origin.
+ *
+ *      I know these are not likely to change, but I still feel better
+ *          making them named symbols, just on General Principles...
+ *
+ **************************************************************************** */
+
+#define FCODE_START  0x0800
+#define FCODE_LIMIT  0x0fff
+
+#endif   /*  _TOKE_NEXTFCODE_H    */

Added: fcode-utils/toke/parselocals.c
===================================================================
--- fcode-utils/toke/parselocals.c	                        (rev 0)
+++ fcode-utils/toke/parselocals.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,944 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Parsing functions for IBM-style Local Values
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          declare_locals      Pick up the Locals' names after the {
+ *          handle_local        Insert the code to access a Local
+ *          exists_as_local     Confirm whether a name is in the Locals vocab
+ *          assign_local        Process the "Assign to a Local" operator ( -> )
+ *          finish_locals       Insert the code for exiting a routine
+ *                                  that uses locals
+ *          forget_locals       Remove the locals' names from the search   
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          These are the names of the three routines that will be invoked
+ *          when Locals are used.  Their definitions exist in a separate
+ *          Local Values Support FCode source-file that must be FLOADed
+ *          into the user's tokenization source.
+ *
+ **************************************************************************** */
+
+/*  Note that the enclosing curly-braces are part of the name  */
+static const char* push_locals = "{push-locals}"; /* ( #ilocals #ulocals -- ) */
+static const char* pop_locals = "{pop-locals}";   /* ( #locals -- )           */
+static const char* local_addr = "_{local}";       /* ( local# -- addr )       */
+
+/*  Switchable Fetch or Store operator to apply to  local_addr.   */
+static const char* local_op = "@";   /*  Initially Fetch  */
+
+
+/* **************************************************************************
+ *
+ *      Revision History:
+ *          Updated Wed, 13 Jul 2005 by David L. Paktor
+ *              Command-line control for:
+ *                  Support for Locals in general
+ *                  Whether to accept the "legacy" separator (semicolon)
+ *                  Whether to issue a message for the "legacy" separator
+ *          Updated Tue, 10 Jan 2006 by David L. Paktor
+ *              Convert to  tic_hdr_t  type vocabulary.
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "ticvocab.h"
+#include "dictionary.h"
+#include "scanner.h"
+#include "parselocals.h"
+#include "errhandler.h"
+#include "clflags.h"
+#include "stream.h"
+#include "devnode.h"
+#include "flowcontrol.h"
+
+/* **************************************************************************
+ *
+ *      Global Variables Imported
+ *          statbuf
+ *          pc    
+ *          opc
+ *          incolon
+ *          lastcolon
+ *          ibm_locals_legacy_separator      Accept ; as the "legacy" separator
+ *          ibm_legacy_separator_message     Issue a message for "legacy" sep'r
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *              local_names        Vocabulary for new local-names
+ *              num_ilocals        Number of initialized local variables
+ *              num_ulocals        Number of uninitialized local variables
+ *              localno            Running Local-Number to be assigned
+ *              eval_buf           Internally-generated string to be parsed
+ *              l_d_lineno         Locals Declaration Line Number
+ *
+ **************************************************************************** */
+
+static tic_hdr_t *local_names = NULL;
+static int num_ilocals = 0;
+static int num_ulocals = 0;
+static int localno = 0;
+static char eval_buf[64];
+static unsigned int l_d_lineno;  	/*  For Error Messages   */
+
+/* **************************************************************************
+ *
+ *     The  local_names  vocabulary follows the same  tic_hdr_t   structure
+ *         as the dictionaries of tokens, special-functions, etcetera.   Its
+ *         "parameter field" is an integer, used to store the Local's number,
+ *         an its "function" is invoke_local(), defined further below.
+ *
+ *     The vocabulary is initially empty, so there's no need for an "init"
+ *         or a "reset" routine.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Function name:  int_to_str
+ *      Synopsis:       Convert an integer into a compilable string.
+ *                      Suport routine for invoke_local().
+ *
+ *      Inputs:
+ *         Parameters:
+ *             num             The number to convert
+ *             bufr            The buffer into which to place it.
+ *                                 Needn't be very long:
+ *                                 five at least, ten is enough
+ *
+ *      Outputs:
+ *         Returned Value:     Pointer to  bufr
+ *             bufr            Contents are changed.
+ *
+ *      Process Explanation:
+ *          Convert into decimal.  If the number is greater than 8,
+ *              prepend a  d#  in front of it.  If less, don't.
+ *              We specifically want to avoid a  d#  in front of
+ *              the numbers 0 1 2 and 3, which are also named constants;
+ *              there's no need to treat 'em as literals.
+ *          The calling routine will be responsible for allocating
+ *              and freeing the buffer.
+ *
+ *      Extraneous Remarks:
+ *          Too bad  atoi()  isn't a Standard C function; I could convert
+ *             using the current base, and be guaranteed that it would be
+ *             interpreted in the same base.
+ *          Instead, I have to fiddle-faddle around with  d#  ...
+ *
+ **************************************************************************** */
+
+static char *int_to_str( int num, char *bufr)
+{
+     char* prefix = "d# ";
+     if ( num < 8 ) prefix = "";
+     sprintf(bufr,"%s%d",prefix, num);
+     return (bufr);
+}
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  invoke_local
+ *      Synopsis:       Compile-in the code to access the Local whose
+ *                      assigned Number is given.  This function is
+ *                      entered into the Local-Names Vocabulary entry.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             pfield              The Vocabulary entry's Param field, taken
+ *                                     from the Assigned Number of the Local.
+ *         Local Static Variables:
+ *             local_addr          Name of  _{local}  routine, invoked
+ *                                     when a Local is used 
+ *             local_op            Fetch or Store operator to apply.
+ *
+ *      Outputs:
+ *         Returned Value:         None
+ *         Local Static Variables:
+ *             eval_buf            Phrase constructed here; will become new
+ *                                     Source Input Buffer, temporarily
+ *
+ *      Error Detection:
+ *          If the Local Values Support FCode source-file was not
+ *              FLOADed into the user's tokenization source, then
+ *              the function  _{local}  will be an "unknown name".
+ *
+ *      Process Explanation:
+ *          We are going to generate a string of the form:
+ *                  " #local _{local} OP"
+ *              and pass it to the Parser for evaluation.
+ *          The call to  _{local}  is preceded by its parameter, which is
+ *               its Assigned Local-Number, and followed by the appropriate
+ *               OPerator, which will be "Fetch" if the Local's name was
+ *               invoked by itself, or "Store" if its invocation was made
+ *               in conjuction with the  ->  operator.
+ *          The string-buffer may be local, but it must be stable.
+ *
+ *      Revision History:
+ *      Updated Thu, 24 Mar 2005 by David L. Paktor
+ *          Factored-out to permit  lookup_local()  to be a "pure"
+ *          function that can be used for duplicate-name detection.
+ *      Updated Tue, 10 Jan 2006 by David L. Paktor
+ *          Accommodate conversion to  tic_hdr_t  type vocabulary.
+ *
+ **************************************************************************** */
+
+static void invoke_local( tic_param_t pfield )
+{
+    char local_num_buf[10];
+    int loc_num = (int)pfield.deflt_elem;
+
+    int_to_str(loc_num, local_num_buf);
+    sprintf( eval_buf, "%s %s %s", local_num_buf, local_addr, local_op );
+    eval_string( eval_buf);
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  locals_separator
+ *      Synopsis:       Test whether the given character is the separator
+ *                          between initted and uninitted Local Names.
+ *                      Optionally, allow Semi-Colon as a separator and issue
+ *                          an optional Advisory.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             subj     One-character "subject" of the test
+ *         Global Variables:    
+ *             ibm_locals_legacy_separator     Allow Semi-Colon as a separator?
+ *             ibm_legacy_separator_message    Issue an Advisory message?
+ *
+ *      Outputs:
+ *         Returned Value:   TRUE if the character is the separator
+ *
+ *      Error Detection:
+ *          If the separator is Semi-Colon, and  ibm_locals_legacy_separator 
+ *              is TRUE, then if  ibm_legacy_separator_message  is TRUE,
+ *              issue an Advisory message.
+ *          If the flag to allow Semi-Colon is FALSE, then simply do not
+ *              acknowledge a valid separator.  Other routines will report
+ *              an erroneous attempt to use an already-defined symbol.
+ *
+ *      Revision History:
+ *          Updated Wed, 13 Jul 2005 by David L. Paktor
+ *          Bring the questions of whether to accept semicolon as a separator
+ *              -- and whether to issue a message for it -- under the control 
+ *              of external flags (eventually set by command-line switches),
+ *              rather than being hard-compiled.
+ *
+ *      Extraneous Remarks:
+ *          In the interest of avoiding too deeply nested "IF"s, I will
+ *              not be adhering strictly to the rules of structure.
+ *
+ **************************************************************************** */
+
+static bool locals_separator( char subj )
+{
+    bool retval = FALSE;
+    /*  Is it the preferred (i.e., non-legacy) separator?   */
+    if ( subj == '|' )
+    {  
+	retval = TRUE;
+	return ( retval );
+    }
+	
+    if ( ibm_locals_legacy_separator )
+    {
+	if ( subj == ';' )
+	{
+	    retval = TRUE;
+	    if ( ibm_legacy_separator_message )
+	    {
+		tokenization_error ( WARNING , "Semicolon as separator in "
+		    "Locals declaration is deprecated in favor of '|'\n");
+	    }
+	}
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  add_local
+ *      Synopsis:       Given a pointer to a name and a number, enter
+ *                      them into the vocabulary for new Local names.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             lnum              The assigned number
+ *             lname             Pointer to the name
+ *         Local Static Variables:
+ *             local_names       The vocabulary for new Local names
+ *
+ *      Outputs:
+ *         Returned Value:       NONE
+ *         Local Static Variables:
+ *             local_names       Enter the new Local's name and number.
+ *         Memory Allocated:
+ *             A place into which the name will be copied
+ *         When Freed?
+ *             When  forget_locals()  routine frees up all memory
+ *                 allocations in the "Local Names" Vocabulary.
+ *
+ *      Process Explanation:
+ *          Allocate a stable place in memory for the name, via  strdup().
+ *          The entry's "action" will be the  invoke_local()  function,
+ *              defined above.  The "parameter field" size is zero.
+ *
+ **************************************************************************** */
+
+static void add_local( TIC_P_DEFLT_TYPE lnum, char *lname)
+{
+    char *lnamecopy ;
+
+    lnamecopy = strdup( lname);
+    add_tic_entry( lnamecopy, invoke_local, lnum,
+		       LOCAL_VAL, 0, NULL, &local_names );
+    trace_creation( LOCAL_VAL, lname);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  gather_locals
+ *      Synopsis:       Collect Local names, for both initted and uninitted
+ *                      Return an indication as to whether to continue
+ *                      gathering Locals' names
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             initted        TRUE if we are gathering initted Local names.
+ *             counter        Pointer to variable that's counting names.
+ *         Global Variables:
+ *             statbuf        The symbol just retrieved from the input stream.
+ *         Local Static Variables:
+ *             localno        Running Local-Number to be assigned
+ *             l_d_lineno     Line # of Locals Declar'n start (for err mssg)
+ *
+ *      Outputs:
+ *         Returned Value:    TRUE = Ended with initted/uninitted separator
+ *         Local Static Variables:
+ *             localno        Incremented for each Local name declared
+ *             local_names    Enter the new locals' names into the Vocabulary.
+ *                            Numeric field is assigned local number.
+ *
+ *      Error Detection:
+ *          A Local-name that duplicates an existing name is an ERROR.
+ *              Especially if that name is <Semicolon> and the flag
+ *              called  ibm_locals_legacy_separator  was not set.
+ *          Issue an Error if close-curly-brace terminator is not found,
+ *              or if imbedded comment is not terminated, before end of file.
+ *          If the Separator is found a second-or-more time, issue an Error
+ *              and continue collecting uninitted Local names.
+ *
+ *      Revision History:
+ *      Updated Thu, 24 Mar 2005 by David L. Paktor
+ *          Allow comments to be interspersed among the declarations.
+ *          Error-check duplicate Local-name.
+ *      Updated Wed, 30 Mar 2005 by David L. Paktor
+ *          Warning when name length exceeds ANSI-specified max (31 chars). 
+ *      Updated Thu, 07 Jul 2005 by David L. Paktor
+ *          Protect against PC pointer-overrun due to unterminated
+ *              comment or declaration.
+ *          Error-check for numbers.
+ *          No name-length check; doesn't go into FCode anyway.
+ *
+ **************************************************************************** */
+
+static bool gather_locals( bool initted, int *counter )
+{
+    signed long wlen;
+    bool retval = FALSE;
+
+    while ( TRUE )
+    {
+        wlen = get_word();
+
+	if ( wlen <= 0 )
+	{
+	    warn_unterm( TKERROR, "Local-Values Declaration", l_d_lineno);
+	    break;
+	}
+
+	/*  Allow comments to be interspersed among the declarations.  */
+	if ( filter_comments( statbuf) )
+	{
+	    /*  Unterminated and Multi-line checking already handled   */
+	    continue;
+	}
+	/*  Is this the terminator or the separator? */
+	if ( wlen == 1 )    /*  Maybe   */
+	{
+	    /*  Check for separator   */
+	    if (locals_separator( statbuf[0] ) )
+	    {
+	        /*  If gathering initted Local names, separator is legit  */
+	        if ( initted )
+		{
+		    retval = TRUE;
+		    break;
+		}else{
+		    tokenization_error ( TKERROR,
+		        "Excess separator -- %s -- found "
+			    "in Local-Values declaration", statbuf);
+		    in_last_colon();
+		    continue;
+		}
+	    }
+	    /*  Haven't found the separator.  Check for the terminator  */
+	    if ( statbuf[0] == '}' )
+	    {
+		break;
+	    }
+	}
+	/*  It was not the terminator or the separator  */
+	{
+	    long tmp;
+	    char *where_pt1;  char *where_pt2;
+	    /*  Error-check for duplicated names  */
+	    if ( word_exists ( statbuf, &where_pt1, &where_pt2 ) )
+	    {
+		tokenization_error ( TKERROR, "Cannot declare %s "
+		    "as a Local-Name; it's already defined %s%s",
+			statbuf, where_pt1, where_pt2 );
+		show_node_start();
+		continue;
+	    }
+	    /*  Error-check for numbers.  */
+	    if ( get_number(&tmp) )
+	    {
+		tokenization_error ( TKERROR, "Cannot declare %s "
+		    "as a Local-Name; it's a number.\n", statbuf );
+		continue;
+	    }
+
+	    /*  We've got a valid new local-name    */
+	    /*  Don't need to check name length; it won't go into the FCode  */
+
+	    /*  Increment our counting-v'ble  */
+	    *counter += 1;
+
+	    /*  Define our new local-name in the Locals' vocabulary  */
+	    add_local( localno, statbuf );
+
+	    /*  Bump the running Local-Number  */
+	    localno++;
+
+	}
+    }
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  activate_locals
+ *      Synopsis:       Compile-in the call to  {push-locals}  that
+ *                      the new definition under construction will need,
+ *                      now that the Locals have been declared.
+ *      
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             num_ilocals              First argument to  {push-locals}
+ *             num_ulocals              Second argument to  {push-locals}
+ *             push_locals              Name of  {push-locals}  routine.
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Local Static Variables:
+ *             eval_buf               Phrase constructed here; will become
+ *                                        new Source Input Buffer, temporarily
+ *
+ *      Error Detection:
+ *          If the Local Values Support FCode source-file was not
+ *              FLOADed into the user's tokenization source, then
+ *              the function  {push-locals}  will be an "unknown name".
+ *
+ *      Process Explanation:
+ *          We are going to generate a string of the form:
+ *                  " #ilocals #ulocals {push-locals}"
+ *              and pass it to the Parser for evaluation.
+ *          The string-buffer may be local, but it must be stable.
+ *
+ *      Question under consideration.:
+ *          Do we want to check if  {push-locals}  is an unknown name,
+ *              and give the user a hint of what's needed?  And, if so,
+ *              do we do it only once, or every time?
+ *
+ **************************************************************************** */
+
+static void activate_locals( void )
+{
+    char ilocals_buf[10];
+    char ulocals_buf[10];
+     
+    int_to_str(num_ilocals, ilocals_buf ); 
+    int_to_str(num_ulocals, ulocals_buf );
+    sprintf( eval_buf,"%s %s %s",ilocals_buf, ulocals_buf, push_locals);
+    eval_string( eval_buf);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  error_check_locals
+ *      Synopsis:       Indicate whether Locals declaration is erronious
+ *      
+ *      Inputs:
+ *         Parameters:     NONE
+ *         Global Variables:
+ *             incolon             TRUE if colon def'n is in effect.
+ *             opc                 FCode Output buffer Position Counter
+ *             lastcolon           Value of opc when Colon def'n was started
+ *
+ *      Outputs:
+ *         Returned Value:         TRUE if found errors severe enough to
+ *                                     preclude further processing of Decl'n
+ *
+ *      Errors Detected:
+ *           Colon definition not in effect.  ERROR and return TRUE.
+ *           Locals declaration inside body of colon-definition (i.e., after
+ *                something has been compiled-in to it) is potentially risky,
+ *                but may be valid, and is a part of legacy practice.  It
+ *                will not be treated as an outright ERROR, but it will
+ *                generate a WARNING...
+ *           Multiple locals declarations inside a single colon-definition
+ *                are completely disallowed.  ERROR and return TRUE.
+ *           Locals declaration inside a control-structure is prohibited.
+ *                Generate an ERROR, but return FALSE to allow processing
+ *                of the declaration to continue.
+ *
+ **************************************************************************** */
+
+/*  The value of  lastcolon  when Locals Declaration is made.
+ *      If it's the same, that detects multiple locals declaration attempt.
+ */
+static int last_local_colon = 0;
+
+static bool error_check_locals ( void )
+{
+    bool retval = FALSE;
+    
+    if ( ! incolon )
+    {
+	tokenization_error ( TKERROR,
+	    "Can only declare Locals inside of a Colon-definition.\n");
+        retval = TRUE;
+    } else {
+	if ( last_local_colon == lastcolon )
+	{
+	    tokenization_error ( TKERROR, "Excess Locals Declaration");
+	    in_last_colon();
+	    retval = TRUE;
+	}else{
+            last_local_colon = lastcolon;
+	    if ( opc > lastcolon )
+	    {
+		tokenization_error ( WARNING,
+		    "Declaring Locals after the body of a Colon-definition "
+		    "has begun is not recommended.\n");
+	    }
+	    announce_control_structs( TKERROR,
+		"Local-Values Declaration encountered",
+		    last_colon_abs_token_no);
+	}
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  declare_locals
+ *      Synopsis:       Process (or Ignore) the Declaration of Locals,
+ *                          upon encountering Curly-brace ( {  )
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             ignoring            TRUE if "Ignoring"
+ *         Global Variables:
+ *             statbuf             Next symbol to process.
+ *             lineno              Current Line Number in Input File
+ *             report_multiline    FALSE to suspend multiline warning
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Global Variables:
+ *             statbuf             Advanced to end of Locals Declaration.
+ *             pc                  Bumped past the close-curly-brace
+ *         Local Static Variables:
+ *             localno             Init'd, then updated by gather_locals()
+ *             l_d_lineno          Line Number of start of Locals Declaration
+ *
+ *      Error Detection:
+ *          See  error_check_locals()
+ *              After Error messages, will bypass further processing until the
+ *                  terminating close-curly-brace of a Locals Declaration.
+ *                  If the terminating close-curly-brace missing under those
+ *                  circumstances, issue an Error
+ *              If terminating close-curly-brace is missing when the Locals
+ *                  Declaration is otherwise valid, gather_locals() will
+ *                  detect and report the Error.
+ *              Warning if multiline declaration.  Because embedded comments
+ *                  may also supppress the multiline warning, we need to save
+ *                  and restore the state of the report_multiline switch...
+ *
+ **************************************************************************** */
+
+void declare_locals ( bool ignoring)
+{
+    num_ilocals = 0;
+    num_ulocals = 0;
+    localno = 0;
+    
+    l_d_lineno = lineno;
+    bool sav_rep_mul_lin = report_multiline;
+    report_multiline = TRUE;
+
+    if ( ignoring || error_check_locals() )
+    {
+       if ( skip_until ( '}' ) )
+       {
+	   warn_unterm(TKERROR,
+	       "misplaced Local-Values Declaration", l_d_lineno);
+       }else{
+	   pc++ ;  /*  Get past the close-curly-brace  */
+       }
+    }else{
+       if (gather_locals( TRUE,  &num_ilocals ) )
+       {
+	   gather_locals( FALSE, &num_ulocals );
+       }
+    }
+
+    /*  If PC has reached the END,  gather_locals()  will
+     *      have already issued an "unterminated" Error;
+     *      a "multiline" warning would be redundant
+     *      repetitive, unnecessary, excessive, unaesthetic
+     *      and -- did I already mention? -- redundant.
+     */
+    if ( pc < end )
+    {
+	report_multiline = sav_rep_mul_lin;
+	warn_if_multiline( "Local-Values declaration", l_d_lineno);
+    }
+
+    /*  Don't do anything if no Locals were declared    */
+    /*  This could happen if the  {  }  field is empty  */
+    if ( localno != 0 )
+    {
+	activate_locals();
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_local
+ *      Synopsis:       Process the given name as a Local Name;
+ *                      indicate if it was a valid Local Name.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             lname                The "Local" name for which to look
+ *         Local Static Variables:  
+ *             local_names          The vocabulary for Local names
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if the name is a valid "Local Name"
+ *
+ **************************************************************************** */
+
+static bool handle_local( char *lname)
+{
+    bool retval = handle_tic_vocab( lname, local_names );
+    return ( retval ) ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_local
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                      word, only if it was a valid Local Name.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             lname                The "Local" name for which to look
+ *         Local Static Variables:  
+ *             local_names          The vocabulary for Local names
+ *
+ *      Outputs:
+ *         Returned Value:          Pointer to the data-structure, or
+ *                                      NULL if not found.
+ *
+ **************************************************************************** */
+
+tic_hdr_t *lookup_local( char *lname)
+{
+    tic_hdr_t *retval = lookup_tic_entry( lname, local_names );
+    return ( retval ) ;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  create_local_alias
+ *      Synopsis:       Create an alias in the "Local Names" Vocabulary
+ *
+ *      Associated FORTH word:              ALIAS
+ *
+ *      Inputs:
+ *         Parameters:
+ *             old_name             Name of existing entry
+ *             new_name             New name for which to create an entry
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if  old_name  found in "Locals" vocab
+ *         Global Variables:    
+ *             local_names          Will point to the new entry
+ *         Memory Allocated:
+ *             Memory for the new entry, by the support routine
+ *         When Freed?
+ *             When  forget_locals()  routine frees up all memory
+ *                 allocations in the "Local Names" Vocabulary.
+ *
+ **************************************************************************** */
+
+bool create_local_alias(char *new_name, char *old_name)
+{
+    bool retval = create_tic_alias( new_name, old_name, &local_names );
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_as_local
+ *      Synopsis:       Simply confirm whether a given name exists
+ *                      within the Locals vocabulary.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             stat_name          Name to look up
+ *
+ *      Outputs:
+ *         Returned Value:        TRUE if stat_name was a Local
+ *
+ **************************************************************************** */
+
+bool exists_as_local( char *stat_name )
+{
+    bool retval = exists_in_tic_vocab(stat_name, local_names );
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  assign_local
+ *      Synopsis:       Process the "Assign to a Local" operator ( -> )
+ *      
+ *      Inputs:
+ *         Parameters:             NONE
+ *         Global Variables:
+ *             statbuf          Next symbol to process
+ *             pc               Input-source Scanning pointer
+ *             lineno           Input-source Line Number. Used for Err Mssg.
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Global Variables:
+ *             statbuf          Advanced to next symbol
+ *             pc               Advanced; may be unchanged if error.
+ *             lineno           Advanced; may be unchanged if error
+ *             local_op         Will be set to Store and then reset to Fetch.
+ *         Global Behavior:
+ *             Construct a phrase and pass it to the Tokenizer.
+ *
+ *      Error Detection:
+ *          If next symbol is not a Local name, print ERROR message    
+ *              and restore  pc  so that the next symbol will be    
+ *              processed by ordinary means.
+ *          In the extremely unlikely case that -> is last symbol in  
+ *              the source-file, report an ERROR.
+ *
+ *      Process Explanation:
+ *          Save the PC.
+ *          Get the next symbol; check for end-of-file.
+ *          Set Local Operator ( local_op ) to "Store", to prepare to apply it.
+ *          Pass the next symbol to  handle_local() .
+ *          If  handle_local()  failed to find the name, you have
+ *              detected an error; restore  pc .
+ *          Otherwise, you have invoked the local and applied "Store" to it.
+ *          At the end, reset  local_op  to "Fetch".
+ *
+ **************************************************************************** */
+
+void assign_local ( void )
+{
+    signed long wlen;
+    bool is_okay;
+    u8 *savd_pc = pc;
+    unsigned int savd_lineno = lineno;
+
+    wlen = get_word();
+
+	if ( wlen <= 0 )
+	{
+	    warn_unterm(TKERROR, "Locals Assignment", lineno);
+	    return;
+	}
+
+    local_op = "!";   /*  Set to Store  */
+
+    is_okay = handle_local( statbuf);
+    if( INVERSE(is_okay)  )
+    {
+        tokenization_error ( TKERROR,
+	    "Cannot apply -> to %s, only to a declared Local.\n", statbuf );
+        pc = savd_pc;
+	lineno = savd_lineno;
+    }
+    local_op = "@";   /*  Reset to Fetch  */
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  finish_locals
+ *      Synopsis:       Compile-in the call to  {pop-locals}  that the
+ *                      new definition under construction will need
+ *                      when it's about to complete execution, i.e.,
+ *                      before an EXIT or a SemiColon.  But only if the
+ *                      current definition under construction is using Locals.
+ *      
+ *      Inputs:
+ *         Parameters:       NONE
+ *             
+ *         Local Static Variables:
+ *             localno            Total # of Locals.
+ *                                    Both a param to  {pop-locals} 
+ *                                    and an indicator that Locals are in use.
+ *             pop_locals         Name of  {pop-locals}  routine.
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *         Local Static Variables:
+ *             eval_buf            Phrase constructed here; will become new
+ *                                     Source Input Buffer, temporarily
+ *
+ *      Error Detection:
+ *          If the Local Values Support FCode source-file was not
+ *              FLOADed into the user's tokenization source, then
+ *              the function  {pop-locals}  will be an "unknown name".
+ *
+ *      Revision History:
+ *          Updated Fri, 24 Feb 2006 by David L. Paktor
+ *              The eval_string() routine no longer calls its own
+ *                  instance of  tokenize() .  In order to make a
+ *                  smooth transition between the processing the
+ *                  internally-generated string and the resumption
+ *                  of processing the source file, it simply sets
+ *                  up the string to be processed next.
+ *              In this case, however, we need to have the string
+ *                  processed right away, as the calling routine
+ *                  emits a token that must follow those generated
+ *                  by this.
+ *              Fortunately, we know the exact contents of the string.
+ *                  Two calls to tokenize_one_word() will satisfy the
+ *                  requirement.
+ *
+ **************************************************************************** */
+
+void finish_locals ( void )
+{
+     /*    Don't do anything if Locals are not in use    */
+    if ( localno > 0 )
+    {
+        char nlocals_buf[10];
+     
+        int_to_str(localno, nlocals_buf ); 
+        sprintf( eval_buf,"%s %s",nlocals_buf, pop_locals);
+        eval_string( eval_buf);
+	tokenize_one_word( get_word() );
+	tokenize_one_word( get_word() );
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  forget_locals
+ *      Synopsis:       Remove the Locals' names from the special Vocabulary
+ *                      free-up their allocated memory, and reset the Locals'
+ *                      counters (which are also the indication that Locals
+ *                      are in use).  This is done at the time a SemiColon
+ *                      is processed.  But only if the current definition
+ *                      under construction is using Locals.
+ *      
+ *      Inputs:
+ *         Parameters:              NONE
+ *         Local Static Variables:
+ *             local_names          The vocabulary for new Local names
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Local Static Variables:
+ *             local_names          Emptied and pointing at NULL.
+ *             num_ilocals          Reset to zero
+ *             num_ulocals              ditto
+ *             localno                  ditto
+ *         Memory Freed
+ *             All memory allocations in the "Local Names" Vocabulary.
+ *
+ **************************************************************************** */
+
+void forget_locals ( void )
+{
+     /*    Don't do anything if Locals are not in use    */
+    if ( localno != 0 )
+    {
+        reset_tic_vocab( &local_names, NULL ) ;
+
+        num_ilocals = 0;
+        num_ulocals = 0;
+        localno = 0;
+    }
+}

Added: fcode-utils/toke/parselocals.h
===================================================================
--- fcode-utils/toke/parselocals.h	                        (rev 0)
+++ fcode-utils/toke/parselocals.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,48 @@
+#ifndef _TOKE_LOCALX_H
+#define _TOKE_LOCALX_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *     Function Prototypes / External definitions for Parsing
+ *          functions for IBM-style Local Values in Tokenizer
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "ticvocab.h"
+
+void declare_locals ( bool ignoring);
+tic_hdr_t *lookup_local( char *lname);
+bool exists_as_local( char *stat_name );
+bool create_local_alias(char *new_name, char *old_name);
+void assign_local ( void );
+void finish_locals ( void );
+void forget_locals ( void );
+
+#endif   /*  _TOKE_LOCALX_H    */

Modified: fcode-utils/toke/scanner.c
===================================================================
--- fcode-utils/toke/scanner.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/scanner.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,6 +24,12 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -34,74 +40,596 @@
 #include <time.h>
 #include <ctype.h>

-#include "toke.h"
+#include "macros.h"
 #include "stack.h"
 #include "stream.h"
 #include "emit.h"
+#include "toke.h"
 #include "dictionary.h"
+#include "vocabfuncts.h"
+#include "scanner.h"
+#include "errhandler.h"
+#include "tokzesc.h"
+#include "conditl.h"
+#include "flowcontrol.h"
+#include "usersymbols.h"
+#include "clflags.h"
+#include "devnode.h"
+#include "tracesyms.h"
+#include "nextfcode.h"

-#define ERROR    do { if (!noerrors) exit(-1);  } while (0)
+#include "parselocals.h"

-extern u8 *start, *pc, *end, *opc, *ostart;
-extern int verbose, noerrors;
+/* **************************************************************************
+ *
+ *  Some VERY IMPORTANT global variables follow
+ *
+ **************************************************************************** */

-u8   *statbuf=NULL;
-u16  nextfcode;
-u8   base=0x0a;
+u8  *statbuf=NULL;      /*  The word just read from the input stream  */
+u8   base=0x0a;         /*  The numeric-interpretation base           */

-/* header pointers */
-u8  *fcode_hdr=NULL;
-u8  *pci_hdr=NULL;
-
 /* pci data */
 bool pci_is_last_image=TRUE;
-bool pci_want_header=FALSE;
-u16  pci_revision=0x0001;
+u16  pci_image_rev=0x0001;  /*  Vendor's Image, NOT PCI Data Structure Rev */
 u16  pci_vpd=0x0000;

-bool offs16=TRUE;
-static bool intok=FALSE, incolon=FALSE;
-bool haveend=FALSE;

-static u8 *skipws(u8 *str)
+/*  Having to do with the state of the tokenization  */
+bool offs16       = TRUE;    /*  We are using 16-bit branch- (etc) -offsets */
+bool in_tokz_esc  = FALSE;   /*  TRUE if in "Tokenizer Escape" mode   */
+bool incolon      = FALSE;   /*  TRUE if inside a colon definition    */
+bool haveend      = FALSE;   /*  TRUE if the "end" code was read.     */
+int do_loop_depth = 0;       /*  How deep we are inside DO ... LOOP variants  */
+
+/*  Used for error-checking of IBM-style Locals  */
+int lastcolon;   /*  Location in output stream of latest colon-definition. */
+
+/*  Used for error reporting   */
+char *last_colon_defname = NULL;   /*  Name of last colon-definition        */
+char *last_colon_filename = NULL;  /*  File where last colon-def'n made     */
+unsigned int last_colon_lineno;    /*  Line number of last colon-def'n      */
+bool report_multiline = TRUE;      /*  False to suspend multiline warning   */
+unsigned int last_colon_abs_token_no;
+
+/* **************************************************************************
+ *  Local variables
+ **************************************************************************** */
+static u16  last_colon_fcode;  /*  FCode-number assigned to last colon-def'n  */
+                               /*      Used for RECURSE  */
+
+static bool do_not_overload = TRUE ;  /*  False to suspend dup-name-test     */
+static bool got_until_eof = FALSE ;   /*  TRUE to signal "unterminated"      */
+
+static unsigned int last_colon_do_depth = 0;
+
+/*  State of headered-ness for name-creation  */
+typedef enum headeredness_t {
+       FLAG_HEADERLESS ,
+       FLAG_EXTERNAL ,
+       FLAG_HEADERS }  headeredness ;
+static headeredness hdr_flag = FLAG_HEADERLESS ;  /*  Init'l default state  */
+
+/*  Local variables having to do with:                                      */
+/*       ...  the state of the tokenization                                 */
+static bool is_instance = FALSE;        /*  Is "instance" is in effect?     */
+static char *instance_filename = NULL;  /*  File where "instance" invoked   */
+static unsigned int instance_lineno;    /*  Line number of "instance"       */
+static bool fcode_started = FALSE ;     /*  Only 1 fcode_starter per block. */
+static bool first_fc_starter = TRUE;    /*  Only once per tokenization...   */
+
+/*       ... with the state of the input stream,                            */
+static bool need_to_pop_source;
+
+/*       ... with the use of the return stack,                              */
+static int ret_stk_depth = 0;          /*  Return-Stack-Usage-Depth counter */
+
+/*       ... and with control of error-messaging.                           */
+           /*  Should a warning about a dangling "instance" 
+	    *      be issued at the next device-node change?
+	    */
+static bool dev_change_instance_warning = TRUE;
+
+           /*  Has a gap developed between "instance" and its application?  */
+static bool instance_definer_gap = FALSE;
+
+           /*  Shared phrases                                               */
+static char *in_tkz_esc_mode = "in Tokenizer-Escape mode.\n";
+
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_ws
+ *      Synopsis:       Advance the PC past all whitespace.
+ *                      Protect against pointer over-runs 
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Global Variables:        
+ *             pc                       Input-source Scanning pointer
+ *             end                      End of input-source buffer
+ *
+ *      Outputs:
+ *         Returned Value:      TRUE if PC reached END before non-blank char
+ *         Global Variables:    
+ *             pc            Advanced to first non-blank char, or to END  
+ *             lineno        Incremented if encountered new-line along the way
+ *
+ *      Error Detection:
+ *          Return a TRUE if End of input-source buffer reached before
+ *              non-blank character.  Not necessarily an error; allow
+ *              calling routine to decide...
+ *
+ **************************************************************************** */
+
+static bool skip_ws(void)
 {
-	while (str && (*str=='\t' || *str==' ' || *str=='\n' )) {
-		if (*str=='\n') 
-			lineno++;
-		str++;
+    bool retval = TRUE;
+    char ch_tmp;
+
+    for (  ; pc < end; pc++ )
+{
+        ch_tmp = *pc;
+	if ( (ch_tmp != '\t') && (ch_tmp != ' ') && (ch_tmp != '\n' ) )
+	{
+	    retval = FALSE;
+	    break;
    }
+        if ( ch_tmp == '\n')  lineno++;
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_until
+ *      Synopsis:       Advance the PC to the given character.
+ *                      Do not copy anything into statbuf.
+ *                      Protect against pointer over-runs 
+ *
+ *      Inputs:
+ *         Parameters:
+ *             lim_ch                   Limiting Character
+ *         Global Variables:        
+ *             pc                       Input-source Scanning pointer
+ *             end                      End of input-source buffer
+ *
+ *      Outputs:
+ *         Returned Value:      TRUE if PC reached END before finding LIM_CH
+ *         Global Variables:    
+ *             pc            Advanced to first occurrence of LIM_CH, or to END  
+ *             lineno        Incremented if encountered new-line along the way
+ *
+ *      Error Detection:
+ *          Return a TRUE if End of input-source buffer reached before
+ *              desired character.  Not necessarily an error; allow calling
+ *              routine to decide...
+ *
+ **************************************************************************** */
+
+bool skip_until( char lim_ch)
+{
+    bool retval = TRUE;
+    char ch_tmp;
+
+    for (  ; pc < end; pc++ )
+    {
+        ch_tmp = *pc;
+        if ( ch_tmp == lim_ch )
+	{
+	    retval = FALSE;
+	    break;
+	}
+        if ( ch_tmp == '\n')  lineno++;
+	}
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  get_until
+ *      Synopsis:       Return, in  statbuf, the string from  PC  to the first
+ *                      occurrence of the given delimiter-character..
+ *
+ *      Inputs:
+ *         Parameters:
+ *             needle          The given delimiter-character
+ *         Global Variables:
+ *             pc              Input-source Scanning Pointer
+ *
+ *      Outputs:
+ *         Returned Value:     Length of the string obtained
+ *         Global Variables:
+ *             statbuf         The string obtained from the input stream;
+ *                                 does not include the delimiter-character.
+ *             pc              Bumped past the delimiter-character, unless
+ *                                 it's a new-line, in which case leave it
+ *                                 to be handled by  get_word()
+ *         Local Static Variables:
+ *             got_until_eof   Pass this as a signal that the end of the
+ *                                 buffer was reached before the delimiter;
+ *                                 Testing whether PC has reached END is
+ *                                 not a sufficient indication.
+ *
+ *      Error Detection:
+ *          If string overflows  statbuf  allocation, ERROR, and 
+ *              return "no string" (i.e., length = 0).
+ *          Otherwise, if delimiter not found before eof, keep string.
+ *              Protection against PC pointer-over-run past END is
+ *              provided by  skip_until() .  Reaching END will be
+ *              handled by calling routine; pass indication along
+ *              via Local Static Variable.
+ *
+ *      Process Explanation:
+ *          Skip the delimiter-character from further input, unless it's a
+ *              new-line which will be skipped anyway.  Let  skip_until() 
+ *              and  get_word()  handle incrementing line-number counters.
+ *          If skip_until()  indicated reaching end-of-file, don't bump PC
+ *
+ *      Revision History:
+ *          Updated Thu, 14 Jul 2005 by David L. Paktor
+ *              More robust testing for when PC exceeds END 
+ *                  Involved replacing  firstchar()
+ *
+ **************************************************************************** */
    
-	return str;
+static signed long get_until(char needle)
+{                                                                               
+	u8 *safe;                                                         
+	unsigned long len = 0;
+
+	safe=pc;
+
+	got_until_eof = skip_until(needle);
+
+	len = pc - safe;
+	if (len >= GET_BUF_MAX )
+	{
+	    tokenization_error( TKERROR,
+		"get_until buffer overflow.  Max is %d.\n", GET_BUF_MAX-1 );
+	    len = GET_BUF_MAX-1;
 }

-static u8 *firstchar(u8 needle, u8 *str)
+	memcpy(statbuf, safe, len);
+	statbuf[len]=0;
+
+	if ( INVERSE(got_until_eof) )
 {
-	while (str && *str!=needle) {
-		if (*str=='\n') 
-			lineno++;
-		str++;
+	    if ( needle != '\n' )  pc++;
    }
+	return len;
+}

-	return str;
+
+/* **************************************************************************
+ *
+ *          We are going to use a fairly sophisticated mechanism to
+ *              make a smooth transition between processing the body
+ *              of a Macro, a User-defined Symbol or an FLOADed file 
+ *              and the resumption of processing the source file, so
+ *              that the end-of-file will only be seen at the end of
+ *              the primary input file (the one from the command-line).
+ *         This mechanism will be tied in with the get_word() routine
+ *
+ *         We are going to define a private data-structure in which
+ *              we will save the state of the current source file,
+ *              and from which, of course, we will recover it.  Its
+ *              fields will be:
+ *                   A pointer to the next structure in the list.
+ *                   The saved values of  START  END  and  PC
+ *                   The saved values of  INAME  and  LINENO
+ *                   A flag indicating that get-word should "pause"
+ *                        before popping the source-stream because
+ *                        the input file will be changing.
+ *                   A place from which to save and recover the state of
+ *                        whether we're testing for "Multi-line" strings;
+ *                        to prevent undeserved "Multi-line" warnings
+ *                        during Macro processing.
+ *                   A pointer to a "resumption" routine, to call
+ *                        when resuming processing the source file;
+ *                        the routine takes a pointer parameter
+ *                        and has no return value.  The pointer
+ *                        may be NULL if no routine is needed.
+ *                   The pointer to pass as the parameter to the
+ *                        resumption routine.
+ *
+ **************************************************************************** */
+
+typedef struct source_state
+    {
+	struct source_state   *next;
+	u8                    *old_start;
+	u8                    *old_pc;
+	u8                    *old_end;
+	char                  *old_iname;
+	unsigned int           old_lineno;
+	bool                   pause_before_pop;
+	bool                   sav_rep_multlin;
+	void                 (*resump_func)();
+	_PTR                   resump_param;
+    } source_state_t ;
+
+static source_state_t  *saved_source = NULL;
+
+
+/* **************************************************************************
+ *
+ *      Function name:  push_source
+ *      Synopsis:       Save the state of the current source file, in the
+ *                          source_state data-structure LIFO linked-list.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             res_func              Pointer to routine to call when resuming
+ *                                       processing the saved source file.
+ *             res_param             Parameter to pass to res_func.
+ *                                   Either or both pointers may be NULL.
+ *             file_chg              TRUE if input file is going to change.
+ *         Global Variables:
+ *             start                 Points to current input buffer
+ *             end                   Points to end of current input buffer
+ *             pc                    Input point in current buffer
+ *             iname                 Name of current source file
+ *             lineno                Line number in current source file
+ *             report_multiline      Whether we're testing for "Multi-line"
+ *         Local Static Variables:
+ *             saved_source          Pointer to the source_state data-structure
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *         Local Static Variables:
+ *             saved_source          Points to new source_state entry
+ *         Memory Allocated
+ *             for the new source_state entry
+ *         When Freed?
+ *             When resuming processing the source file, by drop_source().
+ *
+ *      Process Explanation:
+ *          The calling routine will establish the new input buffer via
+ *              a call to init_inbuf() or the like.
+ *
+ **************************************************************************** */
+
+void push_source( void (*res_func)(), _PTR res_parm, bool file_chg )
+{
+    source_state_t  *new_sav_src;
+
+    new_sav_src = safe_malloc( sizeof(source_state_t), "pushing Source state");
+
+    new_sav_src->next = saved_source;
+    new_sav_src->old_start = start;
+    new_sav_src->old_pc = pc;
+    new_sav_src->old_end = end;
+    new_sav_src->old_iname = iname;
+    new_sav_src->old_lineno = lineno;
+    new_sav_src->pause_before_pop = file_chg;
+    new_sav_src->sav_rep_multlin = report_multiline;
+    new_sav_src->resump_func = res_func;
+    new_sav_src->resump_param = res_parm;
+
+    saved_source = new_sav_src;
 }

-static unsigned long get_word(void)
+/* **************************************************************************
+ *
+ *      Function name:  drop_source
+ *      Synopsis:       Remove last saved state of source processing
+ *                          from the source_state LIFO linked-list,
+ *                          without (or after) restoring.
+ *
+ *      Inputs:
+ *         Parameters:               NONE
+ *         Local Static Variables:
+ *             saved_source          Pointer to the source_state data-structure
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *         Local Static Variables:
+ *             saved_source          Points to previous source_state entry
+ *         Memory Freed
+ *             Saved source_state entry that was just "dropped"
+ *
+ *      Error Detection:
+ *          None.  Called only when linked-list is known not to be at end.  
+ *
+ **************************************************************************** */
+
+static void drop_source( void)
 {
+    source_state_t  *former_sav_src = saved_source;
+
+    saved_source = saved_source->next ;
+    free( former_sav_src);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  pop_source
+ *      Synopsis:       Restore the state of source processing as it was
+ *                          last saved in the source_state linked-list.
+ *
+ *      Inputs:
+ *         Parameters:               NONE
+ *         Local Static Variables:
+ *             saved_source          Pointer to the source_state data-structure
+ *             need_to_pop_source    If TRUE, don't check before popping.
+ *
+ *      Outputs:
+ *         Returned Value:           TRUE if reached end of linked-list
+ *         Global Variables:
+ *             start                 Points to restored input buffer
+ *             end                   Points to end of restored input buffer
+ *             pc                    Input point in restored buffer
+ *             iname                 Name of restored source file
+ *             lineno                Line number in restored source file
+ *             report_multiline      Restored to saved value.
+ *         Local Static Variables:
+ *             saved_source          Points to previous source_state entry
+ *             need_to_pop_source    TRUE if postponed popping till next time
+ *         Memory Freed
+ *             Saved source-state entry that was just "popped"
+ *
+ *      Process Explanation:
+ *          First check the need_to_pop_source flag.
+ *          If it is set, we will clear it and go ahead and pop.
+ *          If it is not set, we will check the  pause_before_pop  field
+ *                  of the top entry in the source_state linked-list.
+ *              If the  pause_before_pop  field is set, we will set the
+ *                  need_to_pop_source flag and return.
+ *              If it is not, we will go ahead and pop.
+ *          If we are going to go ahead and pop, we will call the
+ *              "Resume-Processing" routine (if it's not NULL) before
+ *              we restore the saved source state.
+ *
+ **************************************************************************** */
+
+static bool pop_source( void )
+{
+    bool retval = TRUE;
+
+    if ( saved_source != NULL )
+    {
+	retval = FALSE;
+	if ( need_to_pop_source )
+	{
+	    need_to_pop_source = FALSE;
+	}else{
+	    if ( saved_source->pause_before_pop )
+	    {
+	        need_to_pop_source = TRUE;
+		return( retval);
+	    }
+	}
+
+	if ( saved_source->resump_func != NULL )
+	{
+	    saved_source->resump_func( saved_source->resump_param);
+	}
+	report_multiline = saved_source->sav_rep_multlin;
+	lineno = saved_source->old_lineno ;
+	iname = saved_source->old_iname ;
+	end = saved_source->old_end ;
+	pc = saved_source->old_pc ;
+	start = saved_source->old_start ;
+
+	drop_source();
+    }
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  get_word
+ *      Synopsis:       Gather the next "word" (aka Forth Token) from the
+ *                          input stream.
+ *                      A Forth Token is, of course, a string of characters
+ *                          delimited by white-space (blank, tab or new-line).
+ *                      Do not increment line-number counters here; leave
+ *                          the delimiter after the word unconsumed.
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Global Variables:
+ *             pc                      Input-stream Scanning Pointer
+ *         Local Static Variables:
+ *             need_to_pop_source      If TRUE, pop_source() as first step
+ *
+ *      Outputs:
+ *         Returned Value:             Length of "word" gotten;
+ *                                     0 if  reached end of file.
+ *                                     -1 if reached end of primary input
+ *                                         (I.e., end of all source)
+ *         Global Variables:
+ *             statbuf                 Copy of "gotten" word
+ *             pc                      Advanced to end of "gotten" word,
+ *                                         (i.e., the next word is "consumed")
+ *                                         unless returning zero.
+ *             abs_token_no            Incremented, if valid "word" (token)
+ *                                         was gotten.
+ *
+ *      Process Explanation:
+ *          Skip whitespace to the start of the token, 
+ *             then skip printable characters to the end of the token.
+ *          That part's easy, but what about when skipping whitespace
+ *              brings you to the end of the input stream?
+ *          First, look at the  need_to_pop_source  flag.  If it's set,
+ *              we came to the end of the input stream the last time
+ *              through.  Now we need to  pop_source()  first.
+ *          Next, we start skipping whitespace; this detects when we've
+ *                  reached the end of the input stream.  If we have,
+ *                  then we need to  pop_source()  again.
+ *              If  pop_source()  returned a TRUE, we've reached the end
+ *                  of the primary input file.  Return -1.
+ *              If  pop_source()  turned the  need_to_pop_source  flag
+ *                  to TRUE again, then we need to "pause" until the
+ *                  next time through; return zero.
+ *          Otherwise, we proceed with collecting the token as described.
+ *
+ *      Revision History:
+ *          Updated Thu, 23 Feb 2006 by David L. Paktor
+ *              Tied this routine in with a more sophisticated mechanism that
+ *                  makes a smooth transition between processing the body of
+ *                  a Macro, a User-defined Symbol or an FLOADed file, and 
+ *                  the resumption of processing the source file, so that the
+ *                  end-of-file will only be seen at the end of the primary
+ *                  input file (the one that came from the command-line)
+ *          Updated Fri, 24 Feb 2006 by David L. Paktor
+ *              This is trickier than I thought.  Added a global indicator
+ *                  of whether a file-boundary was crossed while getting
+ *                  the word; previously, that was indicated by a return
+ *                  value of zero, which now means something else...
+ *              The flag,  closed_stream , will be cleared every time this
+ *                  routine is entered, and set whenever close_stream() is
+ *                  entered.
+ *         Updated Tue, 28 Feb 2006 at 10:13 PST by David L. Paktor
+ *              Trickier still.  On crossing a file-boundary, must not
+ *                  consume the first word in the resumed file, for one
+ *                  call; instead, return zero.  Consume it on the next
+ *                  call.  The  closed_stream  flag is now irrelevant and
+ *                  has gone away.
+ *
+ **************************************************************************** */
+
+signed long get_word( void)
+{
    size_t len;
    u8 *str;
+	bool keep_skipping;
+	bool pop_result;

-	pc=skipws(pc);
-	if (pc>=end)
+	if ( need_to_pop_source )
+	{
+	    pop_result = pop_source();
+	}
+
+	do {
+	    keep_skipping = skip_ws();
+	    if ( keep_skipping )
+	    {
+		pop_result = pop_source();
+		if ( pop_result || need_to_pop_source )
+		{
+		    statbuf[0] = 0;
+		    if ( pop_result )
+		    {
+			return -1;
+		    }
    	return 0;
+		}
+	    }
+	} while ( keep_skipping );

str=pc;
-	while (str && *str && *str!='\n' && *str!='\t' && *str!=' ')
+	while ( (str < end) && *str && *str!='\n' && *str!='\t' && *str!=' ')
    	str++;

len=(size_t)(str-pc);
-	if (len>1023) {
-		printf("%s:%d: error: buffer overflow.\n", iname, lineno);
-		ERROR;
+	if (len >= GET_BUF_MAX )
+	{
+	    tokenization_error ( FATAL,
+		"get_word buffer overflow.  Max is %d.", GET_BUF_MAX-1 );
    }

memcpy(statbuf, pc, len); 
@@ -112,39 +640,297 @@
    		iname, lineno, statbuf, len);
 #endif
    pc+=len;
+	abs_token_no++;
    return len;
 }

-static unsigned long get_until(char needle)
+
+/* **************************************************************************
+ *
+ *      Function name:  get_word_in_line
+ *      Synopsis:       Get the next word on the same line as the current
+ *                      line of input.  If the end of line was reached
+ *                      before a word was found, print an error message
+ *                      and return an indication.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             func_nam        Name of the function expecting the same-line
+ *                                 input; for use in the Error Message.
+ *                                 If NULL, do not issue Error Message
+ *         Global Variables:
+ *             pc              Input character pointer.  Saved for comparison
+ *             lineno          Current input line number.  Saved for comparison
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE = success.  Word was acquired on same line.
+ *         Global Variables:
+ *             statbuf         Advanced to the next word in the input stream.
+ *             pc              Advanced if no error; restored otherwise.
+ *
+ *      Error Detection:
+ *          If no next word is gotten (i.e., we're at end-of-file), or if
+ *              one is gotten but not on the same line, the routine will
+ *              return FALSE; if  func_nam  is not NULL, an ERROR Message
+ *              will be issued.
+ *          Also, the values of  PC  LINENO  and  ABS_TOKEN_NO  will be reset
+ *              to the positions they had when this routine was entered.
+ *
+ **************************************************************************** */
+
+bool get_word_in_line( char *func_nam)
 {                                                                               
-	u8 *safe;                                                         
-	unsigned long len;                                                      
+    signed long wlen;
+    bool retval = TRUE;
+    u8 *save_pc = pc;
+    unsigned int save_lineno = lineno;
+    unsigned int save_abs_token_no = abs_token_no;

-	safe=pc;
-	pc=firstchar(needle,safe);
-	if (pc>=end)
-		return 0;
+    /*  Copy of function name, for error message  */
+    char func_cpy[FUNC_CPY_BUF_SIZE+1];

-	len=(unsigned long)pc-(unsigned long)safe;
-	if (len>1023) {
-		printf("%s:%d: error: buffer overflow\n", iname, lineno);
-		ERROR;
+    /*  Do this first, in the likely event that  func_nam  was  statbuf   */
+    if ( func_nam != NULL )
+    {
+	strncpy( func_cpy, func_nam, FUNC_CPY_BUF_SIZE);
+	func_cpy[FUNC_CPY_BUF_SIZE] = 0;  /*  Guarantee a null terminator  */
+    }
+
+    wlen = get_word();
+    if ( ( lineno != save_lineno ) || ( wlen <= 0 ) )
+    {
+	abs_token_no = save_abs_token_no;
+	lineno = save_lineno;
+	pc = save_pc;
+	retval = FALSE;
+	if ( func_nam != NULL )
+	{
+	    tokenization_error ( TKERROR,
+	       "Operator %s expects its target on the same line\n",
+		   strupr(func_cpy));
    }
+    }
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  get_rest_of_line
+ *      Synopsis:       Get all the remaining text on the same line as
+ *                      the current line of input.  If there is no text
+ *                      (not counting whitespace) before the end of line,
+ *                      return an indication.
+ *
+ *      Inputs:
+ *         Parameters:         NONE
+ *         Global Variables:
+ *             pc              Input character pointer.  Saved for restoration
+ *             lineno          Current input line number.  Saved for comparison
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE = success.  Text was acquired on same line.
+ *         Global Variables:
+ *             statbuf         Contains the text found in the input stream.
+ *             pc              Advanced to end of line or of whitespace, if
+ *                                 no error; restored otherwise.
+ *             lineno          Preserved if no error; otherwise, restored.
+ *             abs_token_no    Restored if error; otherwise, advanced as normal.
+ *
+ *      Error Detection:
+ *          Routine will return FALSE if no text is gotten on the same line.
+ *
+ **************************************************************************** */
+
+bool get_rest_of_line( void)
+{
+    bool retval = FALSE;
+    u8 *save_pc = pc;
+    unsigned int save_lineno = lineno;
+    unsigned int save_abs_token_no = abs_token_no;
+
+    if ( INVERSE( skip_ws() ) )
+    {
+        if ( lineno == save_lineno )
+	{
+	    signed long wlen = get_until('\n');
+	    if ( wlen > 0 ) retval = TRUE;
+	}else{
+	    abs_token_no = save_abs_token_no;
+	    lineno = save_lineno;
+	    pc = save_pc;
+	}
+    }
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  warn_unterm
+ *      Synopsis:       Message for "Unterminated ..." something
+ *                      Show saved line-number, where the "something" started,
+ *                      and the definition, if any, in which it occurred.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             severity              Type of error/warning message to display
+ *                                       usually either WARNING or TKERROR
+ *             something             String to print after "Unterminated"
+ *             saved_lineno          Line-Number where the "something" started
+ *         Global Variables:
+ *             lineno                Saved, then restored.
+ *             last_colon_defname    Used only if unterm_is_colon is TRUE;
+ *         Local Static Variables:
+ *             unterm_is_colon       See 07 Mar 2006 entry under Rev'n History
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *         Global Variables:
+ *             lineno                Saved, then restored.
+ *         Local Static Variables:
+ *             unterm_is_colon       Reset to FALSE
+ *         Printout:
+ *             Warning or Error message
+ *
+ *      Revision History:
+ *          Updated Mon, 06 Mar 2006 by David L. Paktor
+ *              Added call to in_last_colon()
+ *          Updated Tue, 07 Mar 2006 by David L. Paktor
+ *              Call to in_last_colon() works okay in most cases except for
+ *                  when the "something" is a Colon Definition; there, it
+ *                  results in the phrase: ... Definition in definition of ...
+ *                  which is awkward.  To eliminate that, I am introducing 
+ *                  a Local Static Variable flag called  unterm_is_colon
+ *                  which will be set only in the appropriate place and
+ *                  re-cleared here.  It's a retro-fit, of course; it could
+ *                  have been a parameter had the need for it occurred when
+ *                  this routine was first constructed... 
+ *
+ **************************************************************************** */
+
+static bool unterm_is_colon = FALSE;
+void warn_unterm( int severity, char *something, unsigned int saved_lineno)
+{
+    unsigned int tmp = lineno;
+    lineno = saved_lineno;
+    if ( unterm_is_colon )
+    {
+	tokenization_error( severity, "Unterminated %s of %s\n",
+	    something, strupr( last_colon_defname) );
+	unterm_is_colon = FALSE;
+    }else{
+	tokenization_error( severity, "Unterminated %s", something);
+	in_last_colon();
+    }
+    lineno = tmp;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  warn_if_multiline
+ *      Synopsis:       Test for "Multi-line ..." something and issue WARNING
+ *                      Show saved line-number, where the "something" started
+ *
+ *      Inputs:
+ *         Parameters:
+ *             something          String to print after "Unterminated"
+ *             start_lineno       Line-Number where the "something" started
+ *         Global Variables:
+ *             lineno             Line-Number where we are now
+ *             iname              Input file name, to satisfy ...where_started()
+ *                                    (Not crossing any actual file boundary.)
+ *             report_multiline   TRUE = go ahead with the message
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *         Global Variables:
+ *             report_multiline   Restored to TRUE.
+ *
+ *      Error Detection:
+ *          Only issue message if the current  lineno  doesn't equal
+ *              the start_lineno  
+ *
+ *      Process Explanation:
+ *          The directive "multi-line" allows the user to specify that
+ *              the next "Multi-line ..." something is intentional, and
+ *              will cause its warning to be suppressed.  It remains in
+ *              effect until it's "used"; afterwards, it's reset.
+ *
+ **************************************************************************** */
+
+void warn_if_multiline( char *something, unsigned int start_lineno )
+{
+    if ( report_multiline && ( start_lineno != lineno ) )
+    {
+	tokenization_error( WARNING, "Multi-line %s, started", something);
+	where_started( iname, start_lineno);
+    }
+    report_multiline = TRUE;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  string_remark
+ *      Synopsis:       Suspend string parsing past end of line and
+ *                      whitespace at start of the new line.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             errmsg_txt            Text to be used for error-message.
+ *         Global Variables:
+ *             pc                    Input-source Scanning pointer
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *         Global Variables:
+ *             pc                    Will point to first non-blank in new line
+ *
+ *      Error Detection:
+ *          The return value of the skip_until() or skip_ws() routine
+ *             will indicate if PC goes past END.  Issue a WARNING.
+ *             The calling routine will handle things from there.
+ *
+ **************************************************************************** */
+
+static void string_remark(char *errmsg_txt)
+{
+    unsigned int sav_lineno = lineno;
+    bool eof = skip_until('\n');
+    if ( ! eof )
+    {
+	eof = skip_ws();
+    }
+    if ( eof )
+    {
+	warn_unterm(WARNING, errmsg_txt, sav_lineno);
+	}
    
-	memcpy(statbuf, safe, len);
-	statbuf[len]=0;
-	return len;
 }

+
+/*  Convert the given string to a number in the supplied base   */
+/*  Allow -- and ignore -- embedded periods.    */
+/*  The  endptr  param represents a pointer that will be updated
+ *      with the address of the first non-numeric character encountered,
+ *      (unless it is a NULL, in which case it is ignored).
+ */
+/*  There is no test for a completely invalid string;
+ *  the calling routine is responsible for ascertaining
+ *  the validity of the string being passed.
+ */
 static long parse_number(u8 *start, u8 **endptr, int lbase) 
 {
    long val = 0;
-	int negative = 0, curr;
+	bool negative = FALSE ;
+	int  curr;
    u8 *nptr=start;

curr = *nptr;
-	if (curr == '-') {
-		negative=1;
+	if (curr == '-')
+	{
+		negative = TRUE ;
    	nptr++;
    }
    
@@ -177,265 +963,3486 @@
    	*endptr=nptr;

if (negative)
-		return -val;
+	{
+		val = -val;
+	}
    return val;
 }

-static u8 *get_sequence(u8 *walk)
+/* **************************************************************************
+ *
+ *      Function name:  add_byte_to_string
+ *      Synopsis:       Add the given byte (or character) to the string
+ *                          being accumulated in statbuf, but protect
+ *                          against a buffer overflow.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             nu_byte           The given character to be added
+ *             walk              Pointer to pointer to the position
+ *                                   in  statbuf  where the character
+ *                                   is to be placed
+ *         Global Variables:
+ *             statbuf           Buffer where the string is accumulated
+ *         Macros:
+ *             GET_BUF_MAX       Size of the buffer
+ *
+ *      Outputs:
+ *         Returned Value:       NONE
+ *         Supplied Pointers:
+ *             **walk            Given character is placed here
+ *             *walk             Incremented in any case
+ *
+ *      Error Detection:
+ *          If  walk  has reached end of string buffer, do not place
+ *              the character, but continue to increment  walk .
+ *              Calling routine will detect overflow.
+ *
+ **************************************************************************** */
+				
+static void add_byte_to_string( u8 nu_byte, u8 **walk )
 {
-	u8 val, pval[3];
+    if ( *walk - statbuf < GET_BUF_MAX )
+    {
+	**walk = nu_byte;
+	}
+    (*walk)++;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  c_string_escape
+ *      Synopsis:       Process C-style escape syntax in strings
+ *
+ *      Inputs:
+ *         Parameters:
+ *             walk                    Pointer to pointer to area into
+ *                                         which to put acquired values
+ *         Global Variables:
+ *             pc                      Input-source Scanning pointer
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Global Variables:
+ *             pc                      Point to last character processed.
+ *         Supplied Pointers:
+ *             *walk                   Advanced by number of bytes acquired
+ *
+ *      Error Detection:
+ *          WARNING conditions.  See under "Process Explanation" below.
+ *
+ *      Process Explanation:
+ *          Start with  PC  pointing to the first character to process
+ *              i.e., after the backslash.
+ *          We recognize newline, tab and numbers
+ *          A digit-string in the current base can be converted to a number.
+ *          The first non-numeric character ends the numeric sequence
+ *              and gets swallowed up.
+ *          If the number exceeds the size of a byte, use the truncated
+ *          	value and issue a WARNING.
+ *          If the first character in the "digit"-string was non-numeric,
+ *              use the character literally and issue a WARNING.
+ *          If the character that ended the numeric sequence is a quote,
+ *              it might be the end of the string, or the start of a
+ *              special-character or even of an "( ... ) hex-sequence,
+ *              so don't swallow it up.
+ *
+ *      Still to be done:
+ *          Better protection against PC pointer-over-run past END.
+ *              Currently, this works, but it's held together by threads:
+ *              Because  init_stream  forces a null-byte at the end of
+ *              the input buffer, parse_number() exits immediately upon
+ *              encountering it.  This situation could be covered more
+ *              robustly...
+ *
+ **************************************************************************** */
+
+static void c_string_escape( u8 **walk)
+{
+    char c = *pc;
+    u8 val;
+    /*  We will come out of this "switch" statement
+     *      with a value for  val  and a decision
+     *      as to whether to write it.
+     */
+    bool write_val = TRUE;
    
-	pc++; /* skip the ( */
+    switch (c)
+    {
+			case 'n':
+				/* newline */
+	    val = '\n';
+				break;
+			case 't':
+				/* tab */
+	    val = '\t';
+				break;
+			default:
+
+	    /*  Digit-string?  Convert it to a number, using the current base.
+	     *  The first non-numeric character ends the numeric sequence
+	     *      and gets swallowed up.
+	     *  If the number exceeds the size of a byte, use the truncated
+	     *      value and issue a WARNING.
+	     *  If the first character in the "digit"-string was non-numeric,
+	     *      use the character literally and issue a WARNING.
+	     */
+
+	     /*
+	     *  If the sequence ender is a quote, it might be the end of
+	     *      the string, or the start of a special-character or even
+	     *      of an "( ... ) hex-sequence, so don't swallow it up.
+	     */
+	    {
+		long lval;
+		u8 *sav_pc = pc;
+		lval=parse_number(pc, &pc, base);
+		val = (u8)lval;
 #ifdef DEBUG_SCANNER
-	printf("%s:%d: debug: hex field:", iname, lineno);
+				if (verbose)
+					printf( "%s:%d: debug: escape code "
+						"0x%x\n",iname, lineno, val);
 #endif
-	pval[1]=0; pval[2]=0;
+		if ( lval > 0x0ff )
+		{
+		    tokenization_error ( WARNING,
+			"Numeric String after \ overflows byte.  "
+			    "Using 0x%02x.\n", val);
+			}

-	for(;;) {
-		while (!isxdigit(*pc) && (*pc) != ')')
-			pc++;
+		if ( pc == sav_pc )
+		{
+		    /*  NOTE:  Here, PC hasn't been advanced past its
+		     *      saved value, so we can count on  C  remaining
+		     *      unchanged since the start of the routine.
+		     */ 
+		    /*  Don't use the null-byte at the end of the buffer  */
+		    if ( ( pc >= end ) 
+		    /*        or a sequence-ending quote.                 */
+			 || ( c == '"' ) )
+		    {
+			write_val = FALSE;
+		    }else{
+			/*  In the WARNING message, print the character
+			 *      if it's printable or show it in hex
+			 *      if it's not.
+			 */
+			if ( (c > 0x20 ) && ( c <= 0x7e) )
+			{
+			    tokenization_error ( WARNING,
+				"Unrecognized character, %c, "
+				    "after \ in string.  "
+					"Using it literally.\n", c);
+			}else{
+			    tokenization_error ( WARNING,
+				"Unrecognized character, 0x%02x, "
+				    "after \ in string.  "
+					"Using it literally.\n", c);
+			}
+			val = c;
+		    }
+		}
+		/*  NOTE:  Here, however, PC may have been advanced...  */
+		/*  Don't swallow the sequence-ender if it's a quote.   */
+		if ( *pc == '"' )
+		{
+		    pc--;
+		}

-		pval[0]=*pc;
-		if (pval[0]==')')
-			break;
+	    }   /*  End of the  "default"  clause  */
+    }    /*  End of the  "switch"  statement  */

-		pc++; /* this cannot be a \n */
+    if ( write_val ) add_byte_to_string( val, walk );

-		pval[1]=*pc;
-		if ( *pc!=')' && *pc++=='\n' )
-			lineno++;
-				
-		val=parse_number(pval, NULL, 16);
-		*(walk++)=val;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  get_sequence
+ *      Synopsis:       Process the Hex-Number option in strings
+ *                      Protect against PC pointer-over-run past END.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             **walk           Pointer to pointer to area into which
+ *                                  to put acquired values
+ *         Global Variables:
+ *             pc               Input-source Scanning pointer
+ *             end              End of input-source buffer
+ *
+ *      Outputs:
+ *         Returned Value:      TRUE = "Normal Completion" (I.e., not EOF)
+ *         Global Variables:
+ *             pc               Points at terminating close-paren, or END
+ *             lineno           Input File Line-Number Counter, may be incr'd
+ *         Supplied Pointers:
+ *             *walk            Advanced by number of values acquired
+ *
+ *      Error Detection:
+ *          End-of-file encountered before end of hex-sequence:
+ *              Issue a Warning, conclude processing, return FALSE.
+ *
+ *      Process Explanation:
+ *          SETUP and RULES:
+ *              Start with  PC  pointing to the first character
+ *                  after the '('  (Open-Paren)     
+ *              Bytes are gathered from digits in pairs, except
+ *                  when separated they are treated singly.
+ *              Allow a backslash in the middle of the sequence
+ *                  to skip to the end of the line and past the
+ *                  whitespace at the start of the next line,
+ *                  i.e., it acts as a comment-escape.
+ *
+ *          INITIALIZE:
+ *              PV_indx = 0
+ *              Set return-indicator to "Abnormal Completion"
+ *              Ready_to_Parse = FALSE
+ *              Stuff NULL into PVAL[2]
+ *          WHILE PC is less than END
+ *              Pick up character at PC into Next_Ch
+ *              IF  Next_Ch  is close-paren :
+ *                  Set return-indicator to "Normal Completion".
+ *                  Done!  Break out of loop.
+ *              ENDIF
+ *              IF comment-escape behavior (controlled by means of a
+ *                      command-line switch) is allowed
+ *                  IF  Next_Ch  is backslash :
+ *                      Skip to end-of line, skip whitespace.
+ *                      If that makes PC reach END :  WARNING message.
+ *                          (Don't need to break out of loop;
+ *                               normal test will terminate.)
+ *                      CONTINUE Loop.
+ *                          (Don't increment PC; PC is already at right place).
+ *                  ENDIF
+ *              ENDIF
+ *              IF  Next_Ch  is a valid Hex-Digit character :
+ *                  Stuff it into  PVAL[PV_indx]
+ *                  IF (PV_indx is 0) :
+ *                      Increment PV_indx
+ *                  ELSE
+ *                      Set Ready_to_Parse to TRUE
+ *          	    ENDIF
+ *              ELSE
+ *                  IF  Next_Ch  is a New-Line, increment Line Number counter
+ *                  IF (PV_indx is 1) :
+ *                      Stuff NULL into PVAL[1]
+ *                      Set Ready_to_Parse to TRUE
+ *          	    ENDIF
+ *              ENDIF
+ *              IF Ready_to_Parse
+ *                  Parse PVAL
+ *                  Stuff into WALK
+ *                  Reset PV_indx to zero
+ *                  Reset Ready_to_Parse to FALSE
+ *              ENDIF
+ *              Increment PC
+ *          REPEAT
+ *          Return with Normal/Abnormal completion indicator
+ *
+ **************************************************************************** */
+
+static bool get_sequence(u8 **walk)
+{
+    int pv_indx = 0;
+    bool retval = FALSE;   /*  "Abnormal Completion" indicator  */
+    bool ready_to_parse = FALSE;
+    char next_ch;
+    char pval[3];
+
 #ifdef DEBUG_SCANNER
+	printf("%s:%d: debug: hex field:", iname, lineno);
+#endif
+    pval[2]=0;
+
+    while ( pc < end )
+    {
+	next_ch = *pc;
+	if ( next_ch == ')' )
+	{
+	    retval = TRUE;
+				break;
+	}
+	if ( hex_remark_escape )
+	{
+	    if ( next_ch == '\' )
+	    {
+		string_remark("string hex-sequence remark");
+		continue;
+	    }
+	}
+	if ( isxdigit(next_ch) )
+	{
+	    pval[pv_indx] = next_ch;
+	    if ( pv_indx == 0 )
+	    {
+		pv_indx++;
+	    }else{
+		ready_to_parse = TRUE;
+	    }
+	}else{
+	    if ( next_ch == '\n' )  lineno++ ;
+	    if ( pv_indx != 0 )
+	    {
+		pval[1] = 0;
+		ready_to_parse = TRUE;
+	    }
+	}
+	if ( ready_to_parse )
+	{
+	    u8 val = parse_number(pval, NULL, 16);
+	    *((*walk)++)=val;
+#ifdef DEBUG_SCANNER
    	printf(" %02x",val);
 #endif
+	    pv_indx = 0;
+	    ready_to_parse = FALSE;
    }
+	pc++;
+    }
 #ifdef DEBUG_SCANNER
    printf("\n");
 #endif
-
-	return walk;
+    return ( retval );
 }

-static unsigned long get_string(void)
+/* **************************************************************************
+ *
+ *    Return the length of the string.
+ *    Pack the string, without the terminating '"' (Quote), into statbuf
+ *    Protect against PC pointer-over-run past END.
+ *    Enable Quote-Backslash as a String-Remark Escape.
+ *    Allowability of Quote-Backslash as a String-Remark is under control
+ *        of a command-line switch (string_remark_escape ).
+ *    Allowability of C-style escape characters is under control
+ *        of a command-line switch ( c_style_string_escape ).
+ *
+ *    Truncate string to size of Forth Packed-String (i.e., uses leading
+ *        count-byte, so limited to 255, number that one byte can express)
+ *        unless the string is being gathered for a Message or is being
+ *        consumed for purposes of ignoring it, in either of which case
+ *        that limit need not be enforced.  Parameter "pack_str" controls
+ *        this:  TRUE  if limit needs to be enforced.
+ *
+ *    Issue WARNING if string length gets truncated.
+ *    Issue WARNING if string crosses line.
+ *        The issuance of the Multi-line WARNING is under control of a
+ *           one-shot directive similar to OVERLOAD , called  MULTI-LINE
+ *
+ *    Still to be decided:
+ *        Do we want to bring the allowability of strings crossing
+ *            lines under control of a command-line switch?
+ *
+ ************************************************************************** */
+
+static signed long get_string( bool pack_str)
 {
    u8 *walk;
    unsigned long len;
-	bool run=1;
-	u8 c, val;
+	char c;
+	bool run = TRUE;
+	unsigned long start_lineno = lineno;    /*  For warning message  */
    
+	/*
+	 *  Bump past the single whitespace character that delimits
+	 *      the command -- e.g.,  ."  or  "  or suchlike -- that
+	 *      starts the string.  Allow new-line to be a command-
+	 *      -delimiting whitespace character.  Regard any sub-
+	 *      sequent whitespace characters as part of the string
+	 */
+	if (*pc++=='\n') lineno++;
+
+	got_until_eof = TRUE ;
+
    walk=statbuf;
    while (run) {
-		switch ((c=*pc)) {
-		case '\':
-			switch ((c=*(++pc))) {
-			case 'n':
-				/* newline */
-				*(walk++)='\n';
+		switch ((c=*pc))
+		{
+		    /*  Standard use of '"' (Quote)  for special-char escape  */
+		    case '"':
+			/*  Skip the '"' (Quote) */
+				pc++;
+			/*  End of the buffer also ends the string cleanly  */
+			if ( pc >= end )
+			{
+			    run = FALSE;
+			    got_until_eof = FALSE ;
    			break;
-			case 't':
-				/* tab */
-				*(walk++)='\t';
+			}
+			/*  Pick up the next char after the '"' (Quote) */
+			c=*pc;
+			switch (c)
+			{
+			    case '(':
+				pc++; /* skip the '(' */
+				run = get_sequence(&walk);
    			break;
-			default:
-				val=parse_number(pc, &pc, base);
-#ifdef DEBUG_SCANNER
-				if (verbose)
-					printf( "%s:%d: debug: escape code "
-						"0x%x\n",iname, lineno, val);
-#endif
-				*(walk++)=val;
-			}
-			break;
-		case '"':
-			pc++; /* skip the " */

-			/* printf("switching: %c\n",*pc); */
-			switch(*pc) {
-			case '(':
-				walk=get_sequence(walk);
-				break;
-			case '"':
-				*(walk++)='"';
-				pc++;
-				break;
    		case 'n':
-				*(walk++)='\n';
-				pc++;
+				add_byte_to_string( '\n', &walk);
    			break;
    		case 'r':
-				*(walk++)='\r';
-				pc++;
+				add_byte_to_string( '\r', &walk);
    			break;
    		case 't':
-				*(walk++)='\t';
-				pc++;
+				add_byte_to_string( '\t', &walk);
    			break;
    		case 'f':
-				*(walk++)='\f';
-				pc++;
+				add_byte_to_string( '\f', &walk);
    			break;
    		case 'l':
-				*(walk++)='\n';
-				pc++;
+				add_byte_to_string( '\n', &walk);
    			break;
    		case 'b':
-				*(walk++)=0x08;
-				pc++;
+				add_byte_to_string( 0x08, &walk);
    			break;
    		case '!':
-				*(walk++)=0x07;
-				pc++;
+				add_byte_to_string( 0x07, &walk);
    			break;
    		case '^':
-				pc++;
-				c=toupper(*(pc++));
-				*(walk++)=c-'A';
+				pc++;    /*   Skip the up-arrow (Caret) */
+				add_byte_to_string( *pc & 0x1f , &walk);
    			break;
-			case '\n':
-				lineno++;
+			    /*  We're done after any of the whitespace
+			     *     characters follows a quote.
+			     */
    		case ' ':
    		case '\t':
-				run=0;
+				/*  Advance past the terminating whitespace
+				 *       character, except newline.
+				 *  Let  get_word()  handle that.
+				 */
+				pc++;
+			    case '\n':
+				run=FALSE;
+				got_until_eof = FALSE ;
    			break;
    		default:
-				*(walk++)=*(pc++);
+				/*  Control allowability of Quote-Backslash
+				 *      as a String-Remark by means of a
+				 *      command-line switch.
+				 */
+				if ( string_remark_escape )
+				{
+				    if ( c == '\' )
+				    {
+					string_remark("string-escape remark");
+					/* The first non-blank in the new line
+					 *     has not been processed yet.
+					 *     Back up to allow it to be.
+					 */
+					pc--;
    			break;
    		}
+				}
+				add_byte_to_string( c, &walk);
+			}
    		break;
+		    case '\n':
+			/*  Allow strings to cross lines.  Include the
+			 *      newline in the string.  Account for it.
+			 */
+			lineno++;
    	default:
-			*(walk++)=c;
+			/*  Control allowability of C-style escape-character
+			 *      syntax by means of a command-line switch.
+			 */
+			if ( c_style_string_escape )
+			{
+			    if ( c == '\' )
+			    {
+				pc++;
+				c_string_escape(&walk );
+				break;
+			    }
+			}
+			add_byte_to_string( c, &walk);
    	}
-		if (*pc++=='\n') lineno++;
+		/*  Advance past the char processed, unless we're done.     */
+		if ( run ) pc++;
+		/*  Done if we hit end of file before string was concluded  */
+		if ( pc >= end )
+		{
+		    run = FALSE;
+		    if ( got_until_eof )
+		    {
+			warn_unterm( WARNING, "string", start_lineno);
+			/*  Prevent multiple messages for one error  */
+			got_until_eof = FALSE;
+		    }
+		}
    }
    
-	*(walk++)=0;
-	
-	if (pc>=end)
-		return 0;
-	
-	len=(unsigned long)walk-(unsigned long)statbuf;
-	if (len>1023) {
-		printf("%s:%d: error: buffer overflow\n", iname, lineno);
-		ERROR;
+	warn_if_multiline( "string", start_lineno);
+
+	len = walk - statbuf;
+	if (len >= GET_BUF_MAX )
+	{
+	    tokenization_error ( TKERROR,
+		"get_string buffer overflow.  Max is %d\n.", GET_BUF_MAX-1 );
+	    len = GET_BUF_MAX-1;
    }
 #ifdef DEBUG_SCANNER
    if (verbose)
    	printf("%s:%d: debug: scanned string: '%s'\n", 
    				iname, lineno, statbuf);
 #endif
+	if ( pack_str && (len > STRING_LEN_MAX) )
+	{
+	    tokenization_error ( WARNING,
+		"String length being truncated to %d.\n", STRING_LEN_MAX );
+	    len = STRING_LEN_MAX;
+	}
+	statbuf[len] = 0;

-	return len>255?255:len;
+	return len ;
 }

-static int get_number(long *result)
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_user_message
+ *      Synopsis:       Collect a user-generated tokenization-time message;
+ *                          either print it or discard it.  Shared code
+ *                          for  user_message()  and  skip_user_message()
+ *
+ *      Inputs:
+ *         Parameters:
+ *             delim                End-of-string delimiter character.
+ *                                  If it's a double-quote ("), we will use
+ *                                      the get-string() routine, with all
+ *                                      its options, to collect the message.
+ *                                  Otherwise, we'll capture plain text from
+ *                                      the input stream.
+ *             print_it             TRUE if we should print the message 
+ *         Local Static Variables:
+ *             got_until_eof        TRUE if reached end of buffer before delim.
+ *         Global Variables:
+ *             lineno               Save, then restore.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Global Variables:
+ *             statbuf              The string will be collected in here
+ *             
+ *         Printout (if  print_it  is TRUE):
+ *            The string, with new-line tacked on, will be printed from
+ *                the  tokenization_error()  routine as a MESSAGE.
+ *            The line-number will be shown as of the origin of the message
+ *
+ *      Error Detection:
+ *          Error-reports will be printed regardless of  print_it  param.
+ *          If delimiter was not found, show "Unterminated" warning message.
+ *          If delimiter was " (double-quote), the get_string() routine
+ *              already checked for a multi-line construct; if delimiter is
+ *              a new-line, then a multi-line construct is impossible.
+ *              otherwise, we will do the multi-line check here.
+ *
+ **************************************************************************** */
+
+static void handle_user_message( char delim, bool print_it )
 {
-	u8 lbase, *until;
-	long val;
+    signed long wlen;
+    unsigned int start_lineno = lineno;
+    unsigned int multiline_start = lineno;    /*  For warning message  */
+    bool check_multiline = FALSE;
+    const char *ug_msg = "user-generated message";

-	lbase=intok?0x10:base;
-	val=parse_number(statbuf,&until, lbase);
+    if ( delim == '"' )
+    {
+	wlen = get_string( FALSE);
+    }else{
+	/*
+	 *  When the message-delimiter is a new-line, and the
+	 *      command-delimiter was a new-line, it means the
+	 *      string length is zero; we won't bump the PC.
+	 *  Otherwise, we will honor the convention we extend
+	 *      to  .(  whereby, if the command is delimited
+	 *      by a new-line, we allow the string to begin
+	 *      on the next line.
+	 */
+	if ( delim == '\n' )
+	{
+	    if ( *pc != '\n') pc++;
+	}else{
+		if (*pc++=='\n') lineno++;
+	    multiline_start = lineno;
+	    check_multiline = TRUE;
+	}
+	wlen = get_until( delim );
+    }
+
+    if ( print_it )
+    {
+	unsigned int tmp_lineno = lineno;
+	lineno = start_lineno;
+	/*  Don't add a new-line to body of the message.
+	 *  Routine already takes care of that.
+	 *  Besides, buffer might be full...
+	 */
+	tokenization_error( MESSAGE, statbuf);
+	lineno = tmp_lineno;
+    }
+
+    if ( got_until_eof )   /*  Crude but effective retrofit... */
+    {
+	warn_unterm(WARNING, (char *)ug_msg, start_lineno);
+    }else{
+	if ( check_multiline )
+	{
+	    warn_if_multiline( (char *)ug_msg, multiline_start);
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  user_message
+ *      Synopsis:       Collect a user-generated message and
+ *                          print it at tokenization-time.
+ *
+ *      Tokenizer directive (either mode):
+ *          Synonyms                              String Delimiter
+ *             [MESSAGE]  #MESSAGE  [#MESSAGE]        end-of-line
+ *             #MESSAGE"                                  "  
+ *      "Tokenizer-Escape" mode directive         String Delimiter
+ *          .(                                            )
+ *          ."                                            "
+ *
+ *      Inputs:
+ *         Parameter is the "parameter field" of the TIC entry, which
+ *             was initialized to the end-of-string delimiter character.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Printout:                User-message, parsed from input.
+ *
+ *      Extraneous Remarks:
+ *          We would have preferred to simply use the "character value"
+ *              aspect of the union, but we found portability issues
+ *              between big- and little- -endian processors, so we still
+ *              have to recast its type here.
+ *
+ **************************************************************************** */
+
+void user_message( tic_param_t pfield )
+{
+    char delim = (char)pfield.deflt_elem ;
+    handle_user_message( delim, TRUE);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  skip_user_message
+ *      Synopsis:       Collect a user-generated message and discard it.
+ *                          Used when ignoring a Conditional section.
+ *
+ *      Tokenizer directive (either mode):
+ *          Synonyms                              String Delimiter
+ *             [MESSAGE]  #MESSAGE  [#MESSAGE]        end-of-line
+ *             #MESSAGE"                                  "  
+ *      "Tokenizer-Escape" mode directive         String Delimiter
+ *          .(                                            )
+ *          ."                                            "
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield               "Parameter field" of the TIC entry, which
+ *                                      was initialized to the delimiter.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Printout:                NONE
+ *
+ **************************************************************************** */
+
+void skip_user_message( tic_param_t pfield )
+{
+    char delim = (char)pfield.deflt_elem ;
+    handle_user_message( delim, FALSE);
+}
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  get_number
+ *      Synopsis:       If the word retrieved from the input stream is a
+ *                      valid number (under the current base) convert it.
+ *                      Return an indication if it was not.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             *result             Pointer to place to return the number
+ *         Global Variables:
+ *             statbuf             The word just read that is to be converted.
+ *             base                The current numeric-interpretation base.
+ *
+ *      Outputs:
+ *         Returned Value:         TRUE = Input was a valid number
+ *         Supplied Pointers:
+ *             *result             The converted number, if valid
+ *                                     otherwise undefined
+ *
+ *      Revision History:
+ *          Updated Mon, 28 Mar 2005 by David L. Paktor
+ *              Always use the current base.
+ *              Reversed sense of return-flag.
+ *
+ **************************************************************************** */
+
+bool get_number( long *result)
+{
+    u8 *until;
+    long val;
+    bool retval = FALSE ;
+
+    val = parse_number(statbuf, &until, base);
    
 #ifdef DEBUG_SCANNER
-	printf("%s:%d: debug: parsing number: base 0x%x, val 0x%lx, "
-			"processed %ld of %ld bytes\n", iname, lineno, 
-			lbase, val,(size_t)(until-statbuf), strlen((char *)statbuf));
+    printf("%s:%d: debug: parsing number: base 0x%x, val 0x%lx, "
+		"processed %ld of %ld bytes\n", iname, lineno, 
+		 base, val,(size_t)(until-statbuf), strlen((char *)statbuf));
 #endif

-	if (until==(statbuf+strlen((char *)statbuf))) {
-		*result=val;
-		return 0;
-	}
-		
-	return -1;
+    /*  If number-parsing ended before the end of the input word,
+     *      then the input word was not a valid number.
+     */
+    if (until==(statbuf+strlen((char *)statbuf)))
+    {
+	*result=val;
+	retval = TRUE;
+    }
+
+    return ( retval );
 }

+/* **************************************************************************
+ *
+ *      Function name:  deliver_number
+ *      Synopsis:       Deliver the supplied number according to the
+ *                              state of the tokenizer:
+ *                          In normal tokenization mode, emit it as an
+ *                              FCode literal.
+ *                          In  "Tokenizer-Escape" mode, push it onto
+ *                              the Data Stack.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             numval                  The number, verified to be valid.
+ *         Global Variables:
+ *             in_tokz_esc   TRUE if tokenizer is in "Tokenizer Escape" mode.
+ *
+ *      Outputs:
+ *         Returned Value:             NONE 
+ *         Items Pushed onto Data-Stack:
+ *             Top:                    The number, if  in_tokz_esc  was TRUE
+ *         FCode Output buffer:
+ *             If  in_tokz_esc  was FALSE, a  b(lit)  token will be written,
+ *                 followed by the number.
+ *
+ **************************************************************************** */
+
+static void deliver_number( long numval)
+{
+    if ( in_tokz_esc )
+    {
+        dpush( numval );
+    } else {
+        emit_literal(numval);
+    }
+}
+/* **************************************************************************
+ *
+ *      Function name:  handle_number
+ *      Synopsis:       Convert the word just retrieved from the input stream
+ *                              to a number.
+ *                      Indicate whether the string was a valid number and
+ *                              deliver it, as appropriate.
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Global Variables:
+ *             statbuf       The word that was just read, and to be converted.
+ *
+ *      Outputs:
+ *         Returned Value:    TRUE = Input string was a valid number
+ *         If input string was a valid number, the converted number will
+ *             be delivered, as appropriate, by  deliver_number(). 
+ *
+ **************************************************************************** */
+
+static bool handle_number( void )
+{
+    bool retval ;
+    long numval;
+
+    retval = get_number( &numval );
+    if ( retval )
+    {
+        deliver_number( numval );
+    }
+
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  ascii_right_number
+ *      Synopsis:       Convert a character sequence to a number, justified
+ *                          toward the right (i.e., the low-order bytes) and
+ *                          deliver it, as appropriate.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             in_str                  The input string
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         The converted number will be delivered by  deliver_number(). 
+ *
+ *      Process Explanation:
+ *          The last four characters in the sequence will become the number.
+ *          If there are fewer than four, they will fill the low-order part
+ *              of the number.
+ *          Example:  PCIR            is converted to  h# 50434952
+ *                    CPU             is converted to  h# 00435055
+ *             and
+ *          	      LotsOfStuff     is equivalent to  a# tuff 
+ *                                    and is converted to  h# 74756666
+ *
+ **************************************************************************** */
+
+static void ascii_right_number( char *in_str)
+{
+    u8 nxt_ch;
+    char *str_ptr = in_str;
+    long numval = 0;
+
+    for ( nxt_ch = (u8)*str_ptr ;
+	    ( nxt_ch = (u8)*str_ptr ) != 0 ;
+        	str_ptr++ )
+    {
+        numval = ( numval << 8 ) + nxt_ch ;
+    }
+    deliver_number( numval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  ascii_left_number
+ *      Synopsis:       Similar to  ascii_right_number()  except justified
+ *                          toward the left (i.e., the high-order bytes).
+ *                      
+ *
+ *      Inputs:
+ *         Parameters:
+ *             in_str                  The input string
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         The converted number will be delivered by  deliver_number().
+ *
+ *      Process Explanation:
+ *          If there are fewer than four characters in the sequence, they
+ *              will fill the high-order part of the number.
+ *                    CPU             is converted to  h# 43505500
+ *          In all other respects, similar to  ascii_right_number()
+ *
+ **************************************************************************** */
+
+static void ascii_left_number( char *in_str)
+{
+    u8 nxt_ch;
+    char *str_ptr = in_str;
+    long numval = 0;
+    int shift_amt = 24;
+    bool shift_over = FALSE ;
+
+    for ( nxt_ch = (u8)*str_ptr ;
+	    ( nxt_ch = (u8)*str_ptr ) != 0 ;
+        	str_ptr++ )
+    {
+        if ( shift_over )  numval <<= 8;
+	if ( shift_amt == 0 )  shift_over = TRUE ;
+	numval += ( nxt_ch << shift_amt );
+	if ( shift_amt > 0 ) shift_amt -= 8;
+    }
+    deliver_number( numval );
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  init_scanner
+ *      Synopsis:       Allocate memory the Scanner will need.
+ *                          Only need to call once per program run.
+ *
+ **************************************************************************** */
+
 void init_scanner(void)
 {
-	statbuf=malloc(1024);
-	if (!statbuf) {
-		printf ("no memory.\n");
-		exit(-1);
-	}
+	statbuf=safe_malloc(GET_BUF_MAX, "initting scanner");
 }

+/* **************************************************************************
+ *
+ *      Function name:  exit_scanner
+ *      Synopsis:       Free up memory the Scanner used
+ *
+ **************************************************************************** */
+
 void exit_scanner(void)
 {
    free(statbuf);
 }

-#define FLAG_EXTERNAL 0x01
-#define FLAG_HEADERS  0x02
-char *name, *alias;
-int flags=0;
+/* **************************************************************************
+ *
+ *      Function name:  set_hdr_flag
+ *      Synopsis:       Set the state of the "headered-ness" flag to the
+ *                          value given, unless over-ridden by one or both
+ *                          of the "always-..." Command-Line Flags
+ *
+ *      Inputs:
+ *         Parameters:
+ *             new_flag                  New setting
+ *         Global Variables:
+ *             always_headers            Override HEADERLESS and make HEADERS
+ *             always_external           Override HEADERLESS and HEADERS;
+ *                                           make EXTERNAL
+ *
+ *      Outputs:
+ *         Returned Value:               None
+ *         Local Static Variables:
+ *             hdr_flag                  Adjusted to new setting
+ *
+ *      Process Explanation:
+ *          If  always_headers  is TRUE, and  new_flag  is not FLAG_EXTERNAL
+ *              then set to FLAG_HEADERS
+ *          If  always_external  is TRUE, set to FLAG_EXTERNAL, regardless.
+ *              (Note:  always_external  over-rides  always_headers).
+ *          Otherwise, set to  new_flag
+ *
+ **************************************************************************** */

-static int create_word(void)
+static void set_hdr_flag( headeredness new_flag)
 {
-	unsigned long wlen;
+    headeredness new_state = new_flag;
+    switch ( new_flag)
+    {
+	case FLAG_HEADERLESS:
+	    {
+		if ( always_headers )
+		{   new_state = FLAG_HEADERS;
+		}
+	    /*  No  break.  Intentional...   */
+	    }
+	case FLAG_HEADERS:
+	    {
+		if ( always_external )
+		{   new_state = FLAG_EXTERNAL;
+		}
+	    /*  No  break.  Intentional...   */
+	    }
+	case FLAG_EXTERNAL:
+	    break;  /*  Satisfy compiler's error-checking...   */
+	/*  No default needed here...   */
+    }

-	if (incolon) {
-		printf("%s:%d: error: creating words not allowed "
-			"in colon definition.\n", iname, lineno);
-		ERROR;
+    hdr_flag = new_state;
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  init_scan_state
+ *      Synopsis:       Initialize various state variables for each time
+ *                          a new tokenization scan is started.
+ *
+ *      Inputs:
+ *         Parameters:             NONE
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Global Variables:   Initialized to:
+ *             base                            0x0a (I.e., base = "decimal")
+ *             nextfcode                       By  reset_fcode_ranges()
+ *             pci_is_last_image               TRUE
+ *             incolon                         FALSE
+ *         Local Static Variables:
+ *             hdr_flag                  FLAG_HEADERLESS (unless over-ridden)
+ *             is_instance                     FALSE
+ *             last_colon_filename             NULL
+ *             instance_filename               NULL
+ *             dev_change_instance_warning     TRUE
+ *             instance_definer_gap            FALSE
+ *             need_to_pop_source              FALSE
+ *             first_fc_starter                TRUE
+ *             ret_stk_depth                   0
+ *         Memory Freed
+ *             Copies of input-file name in  last_colon_filename  and
+ *                 instance_filename , if allocated.
+ *
+ **************************************************************************** */
+
+void init_scan_state( void)
+{
+    base = 0x0a;
+    pci_is_last_image = TRUE;
+    incolon = FALSE;
+    is_instance = FALSE;
+    set_hdr_flag( FLAG_HEADERLESS);
+    reset_fcode_ranges();
+    first_fc_starter = TRUE;
+    if ( last_colon_filename != NULL ) free( last_colon_filename);
+    if ( instance_filename != NULL ) free( instance_filename);
+    last_colon_filename = NULL;
+    instance_filename = NULL;
+    dev_change_instance_warning = TRUE;
+    instance_definer_gap = FALSE;
+    need_to_pop_source = FALSE;
+    ret_stk_depth = 0;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  collect_input_filename
+ *      Synopsis:       Save a copy of the current input file name in the
+ *                          given variable, for error-reporting purposes
+ *
+ *      Inputs:
+ *         Parameters:
+ *             saved_nam                    Pointer to pointer for copy of name
+ *         Global Variables:
+ *             iname                        Current input file name
+ *         Local Static Variables:
+ *
+ *      Outputs:
+ *         Returned Value:                  NONE
+ *         Supplied Pointers:
+ *             *saved_nam                   Copy of name
+ *         Memory Allocated
+ *             For copy of input file name
+ *         When Freed?
+ *             Subsequent call to this routine with same pointer
+ *             (Last copy made will be freed if starting a new tokenization,
+ *                 otherwise, will persist until end of program.) 
+ *         Memory Freed
+ *             Previous copy in same pointer.
+ *
+ *      Process Explanation:
+ *          If there is a previous copy, and it still matches the current
+ *              input-file name, we don't need to free or re-allocate.
+ *
+ **************************************************************************** */
+
+static void collect_input_filename( char **saved_nam)
+{
+    bool update_lcfn = TRUE;    /*  Need to re-allocate?  */
+    if ( *saved_nam != NULL )
+    {
+	if ( strcmp( *saved_nam, iname) == 0 )
+	{
+	    /*  Last collected filename unchanged from iname  */
+	    update_lcfn = FALSE;
+	}else{
+	    free( *saved_nam);
    }
-	
-	if(nextfcode > 0xfff) {
-		printf("%s:%d: error: maximum number of fcode words "
-			"reached.\n", iname, lineno);
-		ERROR;
+    }
+    if ( update_lcfn )
+    {
+	*saved_nam = strdup(iname);
+    }
+} 
+
+/* **************************************************************************
+ *
+ *      Function name:  test_in_colon
+ *      Synopsis:       Error-check whether a word is being used in the
+ *                      correct state, relative to being inside a colon
+ *                      definition; issue a message if it's not.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             wname            The name of the word in question
+ *             sb_in_colon      TRUE if the name should be used inside
+ *                                  a colon-definition only; FALSE if
+ *                                  it may only be used outside of a
+ *                                  colon-definition.
+ *             severity         Type of error/warning message to call.
+ *                                  usually either WARNING or TKERROR
+ *             use_instead      Word the error-message should suggest be
+ *                                  used "instead".  This may be a NULL,
+ *                                  in which case the "suggestion" part
+ *                                  of the message will simply be omitted.
+ *         Global Variables:
+ *             incolon          State of the tokenization; TRUE if inside
+ *                                  a colon definition
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE if no error.
+ *         Printout:           Error messages as indicated.
+ *
+ *      Error Detection:
+ *          If the state, relative to being inside a colon-definition,
+ *              is not what the parameter says it should be, issue a
+ *              message of the indicated severity, and return FALSE.
+ *
+ **************************************************************************** */
+
+static bool test_in_colon ( char *wname,
+                           bool sb_in_colon,    /*  "Should Be IN colon"  */
+			        int severity,
+				     char *use_instead)
+{
+    bool is_wrong;
+    bool retval = TRUE ;
+
+    is_wrong = BOOLVAL(( sb_in_colon != FALSE ) != ( incolon != FALSE )) ;
+    if ( is_wrong )
+    {  
+        char *ui_pt1 = "";
+        char *ui_pt2 = "";
+        char *ui_pt3 = "";
+	retval = FALSE;
+	if ( use_instead != NULL )
+	{
+	    ui_pt1 = "  Use  ";
+	    ui_pt2 = use_instead;
+	    ui_pt3 = "  instead.";
    }
+	tokenization_error ( severity, "The word  %s  should not be used "
+	    "%sside of a colon definition.%s%s%s\n", strupr(wname),
+	        sb_in_colon ? "out" : "in", ui_pt1, ui_pt2, ui_pt3 );
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  must_be_deep_in_do
+ *      Synopsis:       Check that the statement in question is called 
+ *                          from inside the given depth of structures
+ *                          of the  DO ... LOOP -type (i.e., any combination
+ *                          of DO  or ?DO  and  LOOP  or  +LOOP ).
+ *                      Show an error if it is not.
+ *
+ **************************************************************************** */
+
+static void must_be_deep_in_do( int how_deep )
+{
+    int functional_depth = do_loop_depth;
+    if ( incolon )
+    {
+        functional_depth -= last_colon_do_depth;
+    }
+    if ( functional_depth < how_deep )
+    {
+	char deep_do[64] = "";
+	int indx;
+	bool prefix = FALSE;
+
+	for ( indx = 0; indx < how_deep ; indx ++ )
+	{
+	    strcat( deep_do, "DO ... ");
+	}
+	for ( indx = 0; indx < how_deep ; indx ++ )
+	{
+	    if ( prefix )
+	    {
+		strcat( deep_do, " ... ");
+	    }
+	    strcat( deep_do, "LOOP");
+	    prefix = TRUE;
+	}
+
+	tokenization_error( TKERROR,
+	    "%s outside of  %s  structure", strupr(statbuf), deep_do);
+	in_last_colon();
+    }
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  bump_ret_stk_depth
+ *      Synopsis:       Increment or decrement the Return-Stack-Usage-Depth
+ *                          counter.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             bump              Amount by which to increment;
+ *                                   negative number to decrement.
+ *         Local Static Variables:
+ *             ret_stk_depth     The Return-Stack-Usage-Depth counter
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *         Local Static Variables:
+ *             ret_stk_depth     Incremented or decremented
+ *
+ *      Process Explanation:
+ *          This simple-seeming function is actually a place-holder
+ *             for future expansion.  Proper error-detection of
+ *             Return-Stack usage is considerably more complex than
+ *             what we are implementing here, and is deferred for a
+ *             later revision.
+ *
+ *      Still to be done:
+ *          Full detection of whether the Return-Stack has been cleared
+ *              when required, including analysis of Return-Stack usage
+ *              within Flow-Control constructs, and before Loop elements...
+ *
+ *      Extraneous Remarks:
+ *          Some FORTHs use a Loop-Control stack separate from the Return-
+ *              -Stack, but others use the Return-Stack to keep LOOP-control
+ *              elements.  An FCode program must be portable between different
+ *              environments, and so must adhere to the restrictions listed
+ *              in the ANSI Spec:
+ *
+ *       3.2.3.3   Return stack  
+ *        . . . . . .
+ *       A program may use the return stack for temporary storage during the
+ *          execution of a definition subject to the following restrictions:
+ *       	A program shall not access values on the return stack (using R@,
+ *       	    R>, 2R@ or 2R>) that it did not place there using >R or 2>R;
+ *       	A program shall not access from within a do-loop values placed
+ *       	    on the return stack before the loop was entered;
+ *       	All values placed on the return stack within a do-loop shall
+ *       	    be removed before I, J, LOOP, +LOOP, UNLOOP, or LEAVE is
+ *       	    executed;
+ *       	All values placed on the return stack within a definition
+ *       	    shall be removed before the definition is terminated
+ *       	    or before EXIT is executed.
+ *
+ **************************************************************************** */
+
+static void bump_ret_stk_depth( int bump)
+{
+    ret_stk_depth += bump;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  ret_stk_balance_rpt
+ *      Synopsis:       Display a Message if usage of the Return-Stack
+ *                          appears to be out of balance.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             before_what         Phrase to use in Message;
+ *                                     if NULL, use statbuf...
+ *             clear_it            TRUE if this call should also clear the
+ *                                     Return-Stack-Usage-Depth counter
+ *         Global Variables:
+ *             statbuf             Word currently being processed
+ *         Local Static Variables:
+ *             ret_stk_depth       The Return-Stack-Usage-Depth counter
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Local Static Variables:
+ *             ret_stk_depth       May be cleared
+ *
+ *      Error Detection:
+ *          Based simply on whether the Return-Stack-Usage-Depth counter
+ *              is zero.  This is a weak and uncertain implementation;
+ *              therefore, the Message will be a WARNING phrased with
+ *              some equivocation.
+ *
+ *      Process Explanation:
+ *          Proper detection of Return-Stack usage errors is considerably
+ *              more complex, and is deferred for a future revision.
+ *
+ *      Still to be done:
+ *          Correct analysis of Return-Stack usage around Flow-Control
+ *              constructs.  Consider, for instance, the following:
+ * 
+ *          blablabla >R  yadayada IF  R> gubble ELSE flubble R>  THEN
+ * 
+ *              It is, in fact, correct, but the present scheme would
+ *              tag it as a possible error.  Conversely, something like:
+ * 
+ *          blablabla >R  yadayada IF  R> gubble THEN
+ * 
+ *              would not get tagged, even though it is actually an error.
+ * 
+ *          The current simple scheme also does not cover Return-Stack
+ *              usage within Do-Loops or before Loop elements like I and
+ *              J or UNLOOP or LEAVE.  Implementing something like that
+ *              would probably need to be integrated in with Flow-Control
+ *              constructs, and will be noted in  flowcontrol.c
+ *
+ **************************************************************************** */
+
+static void ret_stk_balance_rpt( char *before_what, bool clear_it)
+{
+    if ( ret_stk_depth != 0 )
+    {
+	char *what_flow = ret_stk_depth < 0 ? "deficit" : "excess" ;
+	char *what_phr =  before_what != NULL ? before_what : strupr(statbuf);
+
+	tokenization_error( WARNING,
+	    "Possible Return-Stack %s before %s", what_flow, what_phr);
+	in_last_colon();
+
+	if ( clear_it )
+	{
+	    ret_stk_depth = 0;
+	}
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  ret_stk_access_rpt
+ *      Synopsis:       Display a Message if an attempt to access a value
+ *                          on the Return-Stack appears to occur before
+ *                          one was placed there.
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             statbuf                Word currently being processed
+ *         Local Static Variables:
+ *             ret_stk_depth          The Return-Stack-Usage-Depth counter
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *
+ *      Error Detection:
+ *          Equivocal WARNING, based simply on whether the Return-Stack-
+ *              -Usage-Depth counter not positive.
+ *
+ *      Process Explanation:
+ *          Proper detection is deferred...
+ *
+ *      Still to be done:
+ *          Correct analysis of Return-Stack usage...
+ *
+ **************************************************************************** */
+
+static void ret_stk_access_rpt( void)
+{
+    if ( ret_stk_depth <= 0 )
+    {
+	tokenization_error( WARNING,
+	    "Possible Return-Stack access attempt by %s "
+		"without value having been placed there",
+		strupr(statbuf) );
+	in_last_colon();
+    }
+}
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  encode_file
+ *      Synopsis:       Input a (presumably binary) file and encode it
+ *                      as a series of strings which will be accumulated
+ *                      and encoded in a manner appropriate for a property.
+ *
+ *      Associated Tokenizer directive:        encode-file        
+ *
+ *      Error Detection:
+ *          Handled by support routines.
+ *
+ **************************************************************************** */
+
+static void encode_file( const char *filename )
+{
+	FILE *f;
+	size_t s;
+	int num_encoded=0;
    
-	wlen=get_word();
-	name=strdup((char *)statbuf);
+	tokenization_error( INFO, "ENCODing File %s\n", filename );

+	f = open_expanded_file( filename, "rb", "encoding");
+	if( f != NULL )
+	{
+	    while( (s=fread(statbuf, 1, STRING_LEN_MAX, f)) )
+	    {
+		    emit_token("b(")");
+		    emit_string(statbuf, s);
+		    emit_token("encode-bytes");
+		    if( num_encoded )
+			    emit_token("encode+");
+		    num_encoded += s;
+	    }
+	    fclose( f );
+	    tokenization_error ( INFO, "ENCODed %d bytes.\n", num_encoded);
+	}
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  check_name_length
+ *      Synopsis:       If the length of a user-defined name exceeds the
+ *                          ANSI-specified maximum of 31 characters, issue
+ *                          a message.  This is a hard-coded limit.
+ *                      Although our Tokenizer can handle longer names,
+ *                          they will cause big problems when encountered
+ *                          by an FCode interpreter.
+ *                      If the name is going to be included in the binary
+ *                          output, the message severity must be an ERROR.
+ *                      Otherwise, if the name is HEADERLESS, the severity
+ *                          can be reduced to a Warning; if the name is only
+ *                          defined in "Tokenizer Escape" mode the message
+ *                          severity can be further reduced to an Advisory.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             wlen                 Length of the newly-created word
+ *         Global Variables: 
+ *             in_tokz_esc          TRUE if in "Tokenizer Escape" mode.
+ *         Local Static Variables:
+ *             hdr_flag             State of headered-ness for name-creation
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Global Variables:        
+ *         Printout:                ERROR message if applicable.
+ *
+ *      Error Detection:
+ *             The whole point of this routine.  
+ *
+ *      Revision History:
+ *          Updated Wed, 20 Jul 2005 by David L. Paktor
+ *               Escalated from merely an informative warning to a TKERROR 
+ *          Updated Fri, 21 Oct 2005 by David L. Paktor
+ *               Adjust severity if name doesn't go into the FCode anyway...
+ *
+ **************************************************************************** */
+
+void check_name_length( signed long wlen )
+{
+    if ( wlen > 31 )
+    {
+	int severity = TKERROR;
+	if ( in_tokz_esc )
+	{   severity = INFO;
+	}else{
+	    if (hdr_flag == FLAG_HEADERLESS)
+	    {   severity = WARNING;
+	    }
+	}
+	tokenization_error( severity,
+	    "ANSI Forth does not permit definition of names "
+		"longer than 31 characters.\n" );
+    }
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  definer_name
+ *      Synopsis:       Given a defining-word internal token, return
+ *                      a printable string for the definer, for use
+ *                      in an error-message.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             definer             Internal token for the defining-word
+ *             reslt_ptr           Pointer to string-pointer that takes
+ *                                     the result, if successful
+ *
+ *      Outputs:
+ *         Returned Value:         TRUE if definer was recognized
+ *         Supplied Pointers:
+ *             *reslt_ptr          If successful, points to printable string;
+ *                                     otherwise, left unchanged.
+ *
+ *
+ **************************************************************************** */
+
+static bool definer_name(fwtoken definer, char **reslt_ptr)
+{
+    bool retval = TRUE;
+    switch (definer)
+    {
+	case VARIABLE:
+	    *reslt_ptr = "VARIABLE";
+	    break;
+	case DEFER:
+	    *reslt_ptr = "DEFER";
+	    break;
+	case VALUE:
+	    *reslt_ptr = "VALUE";
+	    break;
+	case BUFFER:
+	    *reslt_ptr = "BUFFER";
+	    break;
+	case CONST:
+	    *reslt_ptr = "CONSTANT";
+	    break;
+	case COLON:
+	    *reslt_ptr = "COLON";
+	    break;
+	case CREATE:
+	    *reslt_ptr = "CREATE";
+	    break;
+	case FIELD:
+	    *reslt_ptr = "FIELD";
+	    break;
+	case MACRO_DEF:
+	    *reslt_ptr = "MACRO";
+	    break;
+	case ALIAS:
+	    *reslt_ptr = "ALIAS";
+	    break;
+	case LOCAL_VAL:
+	    *reslt_ptr = "Local Value name";
+	    break;
+	default:
+	    retval = FALSE;
+    }
+
+    return ( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  as_a_what
+ *      Synopsis:       Add the phrase "as a[n] <DEF'N_TYPE>" for the given
+ *                          definition-type to the given string buffer.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             definer                 Internal token for the defining-word
+ *             as_what                 The string buffer to which to add.
+ *
+ *      Outputs:
+ *         Returned Value:             TRUE if an assigned name was found
+ *                                         for the given definer and text
+ *                                         was added to the buffer.
+ *         Supplied Pointers:
+ *             *as_what                Text is added to this buffer.
+ *
+ *      Process Explanation:
+ *          The calling routine is responsible to make sure the size of
+ *              the buffer is adequate.  Allow 25 for this routine.
+ *          The added text will not have spaces before or after; if any
+ *              are needed, they, too, are the responsibility of the
+ *              calling routine.  The return value gives a helpful clue.
+ *
+ **************************************************************************** */
+
+bool as_a_what( fwtoken definer, char *as_what)
+{
+    char *defn_type_name;
+    bool retval = definer_name(definer, &defn_type_name);
+    if ( retval )
+    {
+	strcat( as_what, "as a");
+	/*  Handle article preceding definer name
+	 *      that starts with a vowel.
+	 */
+	/*  HACK:  Only one definer name -- ALIAS --
+	 *      begins with a vowel.  Take advantage
+	 *      of that...
+	 *  Otherwise, we'd need to do something involving
+	 *      strchr( "AEIOU", defn_type_name[0] )
+	 */
+	if ( definer == ALIAS ) strcat( as_what, "n" );
+
+	strcat( as_what, " ");
+	strcat( as_what, defn_type_name);
+    }
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_word
+ *      Synopsis:       Find the TIC-entry for the given word in the Current
+ *                          mode -- relative to "Tokenizer-Escape" -- and
+ *                          Scope into which definitions are being entered.
+ *                      Optionally, prepare text for various Message types.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             stat_name               Word to look up
+ *             where_pt1               Pointer to result-display string, part 1
+ *                                         NULL if not preparing text
+ *             where_pt2               Pointer to result-display string, part 2
+ *                                         NULL if not preparing text
+ *         Global Variables:
+ *             in_tokz_esc             TRUE if in "Tokenizer Escape" mode.
+ *             scope_is_global         TRUE if "global" scope is in effect
+ *             current_device_node     Current dev-node data-struct
+ *             ibm_locals              TRUE if IBM-style Locals are enabled
+ *
+ *      Outputs:
+ *         Returned Value:             Pointer to TIC-entry; NULL if not found
+ *         Supplied Pointers:
+ *             *where_pt1              Result display string, part 1 of 2
+ *             *where_pt2              Result display string, part 2 of 2
+ *
+ *      Process Explanation:
+ *          We will set the two-part result-display string in this routine
+ *              because only here do we know in which vocabulary the word
+ *              was found.
+ *          Pre-load the two parts of the result-display string.
+ *          If we are in "Tokenizer Escape" mode, look up the word:  first,
+ *              in the "Tokenizer Escape" Vocabulary, or, if not found,
+ *              among the "Shared" words.
+ *          Otherwise, we're in Normal" mode.  Look it up:  first, among the
+ *              Locals, if IBM-style Locals are enabled (it can possibly be
+ *              one if "Tokenizer Escape" mode was entered during a colon-
+ *              -definition); then, if it was not found and if "Device"
+ *              scope is in effect, look in the current device-node; then,
+ *              if not found, in the "core" vocabulary.
+ *          Load the second part of the result-display string with the
+ *               appropriate phrase for whereever it was found.
+ *          Then adjust the first part of the result-display string with
+ *               the definer, if known.
+ *
+ *          The two strings will be formatted to be printed adjacently,
+ *              without any additional spaces in the printf() format.
+ *          The first part of the result-display string will not start with
+ *              a space, but will have an intermediate space if necessary.
+ *          The second part of the result-display string will not start
+ *              with a space, and will contain the terminating new-line
+ *              if appropriate.  It might or might not have been built
+ *              with a call to  in_what_node().
+ *
+ *          If the calling routine displays the result-display strings,
+ *              it should follow-up with a call to  show_node_start()
+ *              This will be harmless if  in_what_node()  was not used
+ *              in the construction of the result-display string.
+ *          If the calling routine is NOT going to display the result strings,
+ *              it should pass NULLs for the string-pointer pointers.
+ *
+ *          The second part of the string consists of pre-coded phrases;
+ *              therefore, we can directly assign the pointer.
+ *          The first part of the string, however, has developed into
+ *              something constructed "on the fly".  Earlier, it, too,
+ *              had been a directly-assignable pointer; all the callers
+ *              to this routine expect that.  Rather than change all the
+ *              callers, we will assign a local buffer for it.
+ *
+ *      Extraneous Remarks:
+ *          We had to add the rule allowing where_pt1 or where_pt2 to be
+ *              NULL after we introduced the  in_what_node()  function.
+ *              We had cases where residue from a lookup for processing
+ *              showed up later in an unrelated Message.  The NULL rule
+ *              should prevent that.
+ *
+ **************************************************************************** */
+
+static char lookup_where_pt1_buf[32];
+
+tic_hdr_t *lookup_word( char *stat_name, char **where_pt1, char **where_pt2 )
+{
+    tic_hdr_t *found = NULL;
+    bool trail_space = TRUE;
+    bool doing_lookup = BOOLVAL( ( where_pt1 != NULL )
+			      && ( where_pt2 != NULL ) );
+    char *temp_where_pt2 = "in the core vocabulary.\n";
+
+    lookup_where_pt1_buf[0] = 0;             /*  Init'lz part-1 buffer  */
+
+    /*  "Core vocab" refers both to shared fwords and built-in tokens.  */
+
+    /*  Distinguish between "Normal" and "Tokenizer Escape" mode  */
+    if ( in_tokz_esc )
+    {   /*  "Tokenizer Escape" mode.  */
+	found = lookup_tokz_esc( stat_name);
+	if ( found != NULL )
+	{
+	    temp_where_pt2 = in_tkz_esc_mode;
+	}else{
+	    /*  "Core vocabulary".  */
+	    found = lookup_shared_word( stat_name);
+	}
+    }else{
+	/*  "Normal" tokenization mode  */
+	if ( ibm_locals )
+	{
+	    found = lookup_local( stat_name);
+	    if ( doing_lookup && ( found != NULL ) )
+	    {
+		trail_space = FALSE;
+		temp_where_pt2 = ".\n";
+	    }
+	}
+
+	if ( found == NULL )
+	{
+	    found = lookup_in_dev_node( stat_name);
+	    if ( found != NULL )
+	    {
+		if ( doing_lookup )
+		{
+		    temp_where_pt2 = in_what_node( current_device_node);
+		}
+	    }else{
+		/*  "Core vocabulary".  */
+		found = lookup_core_word( stat_name);
+	    }
+	}
+    }
+
+    if ( ( doing_lookup ) && ( found != NULL ) )
+    {
+	if ( as_a_what( found->fword_defr, lookup_where_pt1_buf) )
+	{
+	    if ( trail_space )
+	    {
+		strcat(lookup_where_pt1_buf, " ");
+	    }
+	}
+	*where_pt1 = lookup_where_pt1_buf;
+	*where_pt2 = temp_where_pt2;
+    }
+    return( found);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  word_exists
+ *      Synopsis:       Check whether a given word is already defined in the
+ *                          Current mode -- relative to "Tokenizer-Escape" --
+ *                          and Scope into which definitions are being entered. 
+ *                      Used for error-reporting.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             stat_name                 Word to look up
+ *             where_pt1                 Pointer to string, part 1 of 2,
+ *                                          to display in result
+ *             where_pt2                 Pointer to string, part 2 of 2,
+ *                                          to display in result
+ *
+ *      Outputs:
+ *         Returned Value:               TRUE if the name exists.
+ *         Supplied Pointers:
+ *             *where_pt1                Result display string, part 1 of 2
+ *             *where_pt2                Result display string, part 2 of 2
+ *
+ *      Process Explanation:
+ *          If the calling routine displays the result-display strings,
+ *              it should follow-up with a call to  show_node_start()
+ *
+ *      Extraneous Remarks:
+ *          This used to be a much heftier routine; now it's just
+ *              a wrapper around  lookup_word() .
+ *
+ **************************************************************************** */
+
+bool word_exists( char *stat_name, char **where_pt1, char **where_pt2 )
+{
+    bool retval = FALSE;
+    tic_hdr_t *found = lookup_word( stat_name, where_pt1, where_pt2 );
+
+    if ( found != NULL )
+    {
+	retval = TRUE;
+    }
+
+    return( retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  warn_if_duplicate
+ *      Synopsis:       Check whether a given word is already defined in
+ *                          the current mode and issue a warning if it is.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             stat_name                Word to check
+ *         Global Variables:
+ *             verbose_dup_warning      Whether to run the check at all.
+ *         Local Static Variables:
+ *             do_not_overload          FALSE if  OVERLOAD  is in effect.
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Local Static Variables:
+ *             do_not_overload          Restored to TRUE
+ *         Printout:
+ *             Warning message if a duplicate.
+ *
+ *      Error Detection:
+ *             None.  This is merely an informative warning.
+ *
+ *      Process Explanation:
+ *          "Current mode" -- meaning, whether the tokenizer is operating
+ *              in "Tokenizer Escape" mode or in normal tokenization mode --
+ *              will be recognized by the  word_exists()  function.
+ *
+ *      Extraneous Remarks:
+ *          The  OVERLOAD  directive is our best shot at creating a more
+ *              fine-grained way to temporarily bypass this test when
+ *              deliberately overloading a name.  It would be nice to have
+ *              a mechanism, comparable to the classic
+ *                     WARNING @ WARNING OFF  .....  WARNING !
+ *              that could be applied to a range of definitions, but:
+ *              (1)  That would require more of a true FORTH infrastructure;
+ *                       hence, more effort than I am willing to invest, at
+ *                       this juncture, for such a small return,
+ *              and
+ *              (2)  Most intentional-overloading ranges only cover a
+ *                       single definition anyway.
+ *
+ **************************************************************************** */
+
+void warn_if_duplicate( char *stat_name)
+{
+    if ( verbose_dup_warning && do_not_overload )
+    {
+	char *where_pt1;
+	char *where_pt2; 
+	if ( word_exists( stat_name, &where_pt1, &where_pt2) )
+	{
+	    tokenization_error( WARNING, 
+		"Duplicate definition:   %s  already exists %s%s",
+		    stat_name, where_pt1, where_pt2 );
+	    show_node_start();
+	}
+    }
+    do_not_overload = TRUE;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  glob_not_allowed
+ *      Synopsis:       Print a Message that "XXX is not allowed."
+ *                          because Global Scope is in effect.
+ *                      Used from several places...
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             severity              Severity of the Message
+ *             not_ignoring          FALSE = "Ignoring", for the part of the
+ *                                       message about "How It's being Handled"
+ *         Global Variables:
+ *             statbuf               Disallowed word currently being processed
+ *
+ *      Outputs:
+ *         Returned Value:           NONE
+ *         Printout:                 Message of given severity.
+ *
+ **************************************************************************** */
+
+static void glob_not_allowed( int severity, bool not_ignoring)
+{
+    tokenization_error( severity, "Global Scope is in effect; "
+			"%s not allowed.  %s.\n",
+			    strupr(statbuf), 
+				 not_ignoring ?
+				     "Attempting to compensate.." :
+					  "Ignoring" );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  not_in_dict
+ *      Synopsis:       Print the message "XXX is not in dictionary."
+ *                      Used from several places...
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             stat_name                Word that could not be processed
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Printout:         Error message.
+ *
+ **************************************************************************** */
+
+static void not_in_dict( char *stat_name)
+{
+    tokenization_error ( TKERROR,
+        "Word  %s  is not in dictionary.\n", stat_name);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  tokenized_word_error
+ *      Synopsis:       Report an error when a word could not be processed
+ *                          by the tokenizer.  Messages will vary...
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             stat_name                Word that could not be processed
+ *         Global Variables:
+ *             in_tokz_esc    TRUE if tokenizer is in "Tokenizer Escape" mode.
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Printout:          Error message.  Possible Advisory about
+ *
+ *      Error Detection:
+ *          Error was detected by the calling routine...
+ *
+ *      Process Explanation:
+ *          If the tokenizer is in "Tokenizer Escape" mode, the word might
+ *              be one that can be used in normal tokenization mode;
+ *          Conversely, if the tokenizer is in normal-tokenization mode,
+ *              the word might be one that can be used in the "Escape" mode.
+ *          Or, the word is completely unknown.
+ *          Recognizing the current mode is handled by  word_exists()
+ *          However, we need to test for the *converse* of the current mode,
+ *              so before we call  word_exists()  we are going to save and
+ *              invert the setting of  in_tokz_esc  (and afterwards, of
+ *              course, restore it...)
+ *
+ **************************************************************************** */
+
+static void tokenized_word_error( char *stat_name)
+{
+    char *where_pt1;
+    char *where_pt2;
+    bool found_somewhere;
+    
+    bool sav_in_tokz_esc = in_tokz_esc;
+    in_tokz_esc = INVERSE(sav_in_tokz_esc);
+
+    found_somewhere = word_exists( stat_name, &where_pt1, &where_pt2);
+    if ( found_somewhere )
+    {
+	tokenization_error ( TKERROR, "The word %s is %s recognized "
+	    "in tokenizer-escape mode.\n",
+		 stat_name, sav_in_tokz_esc ? "not" :  "only" );
+    } else {
+	not_in_dict( stat_name);
+    }
+
+    if ( INVERSE(exists_in_ancestor( stat_name)) )
+    {
+        if ( found_somewhere && sav_in_tokz_esc )
+	{
+	    tokenization_error(INFO,
+		"%s is defined %s%s", stat_name, where_pt1, where_pt2 );
+	    show_node_start();
+	}
+    }
+
+    in_tokz_esc = sav_in_tokz_esc;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  unresolved_instance
+ *      Synopsis:       Print the "unresolved instance" message
+ *
+ *      Inputs:
+ *         Parameters:
+ *             severity                    Severity of the Message
+ *         Local Static Variables:
+ *             instance_filename           File where "instance" invoked
+ *             instance_lineno             Line number where "instance" invoked
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Printout:          Message.
+ *
+ *      Error Detection:
+ *          Error was detected by the calling routine...
+ *
+ **************************************************************************** */
+
+static void unresolved_instance( int severity)
+{
+    tokenization_error( severity, "Unresolved "INSTANCE"" );
+    just_where_started( instance_filename, instance_lineno );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  modified_by_instance
+ *      Synopsis:       Print the "[not] modified by instance" message
+ *
+ *      Inputs:
+ *         Parameters:
+ *             definer                     Internal token for the defining-word
+ *             was_modded                  FALSE if "not modified..."
+ *         Local Static Variables:
+ *             instance_filename           File where "instance" invoked
+ *             instance_lineno             Line number where "instance" invoked
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Printout:          WARNING message.
+ *
+ *      Error Detection:
+ *          Error was detected by the calling routine...
+ *
+ **************************************************************************** */
+
+static void modified_by_instance( fwtoken definer, bool was_modded)
+{
+    char *was_not = was_modded ? "was" : "not" ;
+    char *defn_type_name;
+
+    /*  No need to check the return value  */
+    definer_name(definer, &defn_type_name);
+
+    tokenization_error ( WARNING,
+	"%s definition %s modified by "INSTANCE"",
+	    defn_type_name, was_not );
+    just_where_started( instance_filename, instance_lineno );
+ }
+
+/* **************************************************************************
+ *
+ *      Function name:  validate_instance
+ *      Synopsis:       If "instance" is in effect, check whether it is
+ *                          appropriate to the defining-word being called.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             definer                   Internal token for the defining-word
+ *         Local Static Variables:
+ *             is_instance               TRUE if "instance" is in effect.
+ *             instance_definer_gap      TRUE if invalid definer(s) invoked
+ *                                           since "instance" went into effect.
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Local Static Variables:
+ *             is_instance               Reset to FALSE if definer was valid.
+ *             instance_definer_gap      TRUE if definer was not valid;
+ *                                           FALSE if definer was valid.
+ *
+ *      Error Detection:
+ *          If "instance" is in effect, the only defining-words that are
+ *              valid are:  value  variable  defer  or  buffer:  Attempts
+ *              to use any other defining-word will be reported with a
+ *              WARNING, but "instance" will remain in effect.
+ *          If an invalid defining-word was invoked since "instance" went
+ *              into effect, then, when it is finally applied to a valid
+ *              definer, issue a WARNING.
+ *
+ *      Process Explanation:
+ *          Implicit in the Standard is the notion that, once INSTANCE has
+ *              been executed, it remains in effect until a valid defining-
+ *              word is encountered.  We will do the same.
+ *
+ **************************************************************************** */
+
+static void validate_instance(fwtoken definer)
+{
+    if ( is_instance )
+    {
+	bool is_error = TRUE ;
+
+	switch ( definer)
+	{
+	    case VALUE:
+	    case VARIABLE:
+	    case DEFER:
+	    case BUFFER:
+		is_error = FALSE;
+	    /*  No default needed, likewise, no breaks;      */
+	    /*  but some compilers get upset without 'em...  */
+	    default:
+		break;
+	}
+
+	if( is_error )
+	{
+	    modified_by_instance(definer, FALSE );
+	    instance_definer_gap = TRUE;
+	}else{
+	    if ( instance_definer_gap )
+	    {
+		modified_by_instance(definer, TRUE );
+	    }
+	    is_instance = FALSE;
+	    instance_definer_gap = FALSE;
+	}
+    }
+}
+    
+
+/* **************************************************************************
+ *
+ *      Function name:  trace_creation
+ *      Synopsis:       If the word being created is on the Trace List,
+ *                          display the appropriate message
+ *
+ *      Inputs:
+ *         Parameters:
+ *             definer                 Internal token for the defining-word
+ *             nu_name                 The word being created
+ *         Global Variables:
+ *             verbose                 No point in doing all this if we're
+ *                                         not showing the message anyway...
+ *             in_tokz_esc             TRUE if we are in Tokenizer-Escape mode
+ *             scope_is_global         TRUE if "global" scope is in effect
+ *             current_device_node     Current dev-node data-struct
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Printout:
+ *             Advisory Message, if the word is on the Trace List.
+ *
+ *      Process Explanation:
+ *          The order of scope-checking is important:
+ *              A Local has no scope beyond the definition in which it occurs.
+ *              Tokenizer-Escape mode supercedes "Normal" mode, and renders
+ *                  moot the differences between Global and Device scope.
+ *              Global scope is mutually exclusive with Device scope.
+ *              Device scope needs to identify where the Current device-node
+ *                  began.
+ *
+ **************************************************************************** */
+
+void trace_creation( fwtoken definer, char *nu_name)
+{
+    if ( verbose )
+    {
+	if ( is_on_trace_list( nu_name) )
+	{
+            char  as_what[96] = "";
+	    bool show_last_colon = BOOLVAL( definer == LOCAL_VAL);
+
+	    as_a_what( definer, as_what);  /*  No need to check return value. */
+
+	    /*  Scope-checking starts here, unless  show_last_colon  is TRUE.
+	     *  Come out of this with  as_what[]  filled up and
+	     *      terminated with a new-line, if appropriate,
+	     */
+	    while ( ! show_last_colon )
+	    {
+		strcat( as_what, " ");
+
+		if ( in_tokz_esc )
+		{
+		    strcat( as_what, in_tkz_esc_mode);
+		    break;
+		}
+
+		if ( scope_is_global )
+		{
+		    strcat( as_what, "with Global scope.\n");
+		}else{
+		    /*  In Device scope.  Show the Current node.   */
+		    strcat( as_what, in_what_node( current_device_node));
+		}
+		break;
+
+	    }   /*  Destination of BREAKs ...   */
+
+	    tokenization_error(INFO, "Creating %s %s", nu_name, as_what);
+
+	    if ( show_last_colon )
+	    {
+		in_last_colon();
+	    }else{
+		show_node_start();
+	    }
+
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  create_word
+ *      Synopsis:       
+ *
+ *      Inputs:
+ *         Parameters:
+ *             definer             Internal token for the defining-word
+ *         Global Variables:
+ *             control_stack_depth Number of "Control Stack" entries in effect
+ *             nextfcode           FCode-number to be assigned to the new name
+ *             statbuf             Symbol last read from the input stream
+ *             pc                  Input-source Scanning pointer
+ *             hdr_flag            State of headered-ness for name-creation
+ *             force_tokens_case   If TRUE, force token-names' case in FCode
+ *             force_lower_case_tokens
+ *                                 If  force_tokens_case  is TRUE, this
+ *                                     determines which case to force
+ *             iname               Input-source file name; for error-reporting
+ *             lineno              Input-source Line number; also for err-rep't
+ *
+ *      Outputs:
+ *         Returned Value:         TRUE if successful
+ *         Global Variables:  
+ *             nextfcode           Incremented  (by bump_fcode() )
+ *             statbuf             Advanced to next symbol; must be re-read
+ *             pc                  Advanced, then restored to previous value
+ *         Memory Allocated
+ *             Copy of the name being defined, by support routine.
+ *             Copy of input-source file name, for error-reporting
+ *         When Freed?
+ *             Copy of name being defined is freed when Current Device Vocab
+ *                 is "finished", or at end of tokenization.
+ *             Copy of input-source file name is freed at end of this routine.
+ *
+ *      Error Detection:
+ *          ERROR if already inside a colon-definition.  Discontinue
+ *              processing and return FALSE.
+ *          ERROR if inside a control-structure.  Continue processing,
+ *              though, to catch other errors, and even return TRUE;
+ *              except:  leave the new token undefined. 
+ *          Warning on duplicate name (subject to command-line control)
+ *          Message if name is excessively long; Warning if headerless.
+ *          FATAL if the value of  nextfcode  is larger than the legal
+ *              maximum for an FCode, (0x0fff).
+ *
+ *      Revision History:
+ *      Updated Thu, 24 Mar 2005 by David L. Paktor
+ *          Optional warning when name about to be created is a
+ *              duplicate of an existing name.
+ *      Updated Wed, 30 Mar 2005 by David L. Paktor
+ *          Warning when name length exceeds ANSI-specified max (31 chars).
+ *      Updated Tue, 05 Apr 2005 by David L. Paktor
+ *          Add "definer" parameter and call to  add_definer() .  Part
+ *              of the mechanism to forbid attempts to use the  TO 
+ *              directive to change values of CONSTANTs in particular
+ *              and of inappropriate targets in general.
+ *      Updated Fri, 06 May 2005 by David L. Paktor
+ *          Error-detection of   DO ...  LOOP  and  BEGIN ...  imbalance
+ *          Error-detection of  nextfcode  exceeding legal maximum (0x0fff).
+ *      Updated Wed, 20 Jul 2005 by David L. Paktor
+ *          Put Duplicate-Name-Test under command-line control...
+ *      Updated Wed, 24 Aug 2005 by David L. Paktor
+ *          Error-detection via  clear_control_structs()  routine.
+ *      Updated Tue, 10 Jan 2006 by David L. Paktor
+ *          Convert to  tic_hdr_t  type vocabulary.
+ *      Updated Thu, 20 Apr 2006 by David L. Paktor
+ *          Allow creation of new definition within body of a flow-control
+ *              structure.  (Remove error-detection via  clear_control_structs)
+ *      Updated Tue, 13 Jun 2006 by David L. Paktor
+ *          Move detection of out-of-bounds  nextfcode  to  assigning_fcode()
+ *              routine, which also detects Overlapping Ranges error.
+ *      Updated Thu, 27 Jun 2006 by David L. Paktor
+ *          Report Error for attempt to create def'n inside control structure.
+ *
+ *      Extraneous Remarks:
+ *          We must not set  incolon  to TRUE (if we are creating a colon
+ *              definition) until *AFTER* this routine has been called, due
+ *              to the initial error-checking.  If we need to detect whether
+ *              we are creating a colon definition, we can do so by testing
+ *              whether the parameter, DEFINER, equals COLON .
+ *
+ **************************************************************************** */
+
+static bool create_word(fwtoken definer)
+{
+    signed long wlen;
+    bool retval = FALSE;
+    char *defn_type_name;
+
+    /*  If already inside a colon, ERROR and discontinueprocessing    */
+    /*  If an alias to a definer is used, show the name of the alias  */
+    if ( test_in_colon(statbuf, FALSE, TKERROR, NULL) ) 
+    {
+	char defn_type_buffr[32] = "";
+	unsigned int old_lineno = lineno;    /*  For error message  */
+	bool define_token = TRUE;
+
+	{   /*  Set up definition-type text for error-message */
+
+	    /*  No need to check the return value  */
+	    definer_name(definer, &defn_type_name);
+
+	    strcat( defn_type_buffr, defn_type_name);
+	    strcat( defn_type_buffr, " definition");
+	}
+	/*  If in a control-structure, ERROR but continue processing  */
+	if ( control_stack_depth != 0 )
+	{
+	    announce_control_structs( TKERROR, defn_type_buffr, 0);
+	    /*  Leave the new token undefined.  */
+	    define_token = FALSE;
+	}
+
+	/*  Get the name of the new token  */
+	wlen = get_word();
+
 #ifdef DEBUG_SCANNER
    printf("%s:%d: debug: defined new word %s, fcode no 0x%x\n",
    		iname, lineno, name, nextfcode);
 #endif
-	add_token(nextfcode, name);
-	if (flags) {
-		if (flags&FLAG_EXTERNAL)
-			emit_token("external-token");
-		else
-			emit_token("named-token");
-		emit_string((u8 *)name, wlen);
-	} else
-		emit_token("new-token");
+	if ( wlen <= 0 )
+	{
+	    warn_unterm( TKERROR, defn_type_buffr, old_lineno);
+	}else{
+	    bool emit_token_name = TRUE;
+
+	    /*  Handle Tracing of new definitions  */
+	    trace_creation( definer, statbuf);
+
+	    /*  Other Error or Warnings as applicable  */
+	    validate_instance( definer);
+	    warn_if_duplicate( statbuf);
+	    check_name_length( wlen);
+
+	    /*  Bump FCode; error-check as applicable  */
+	    assigning_fcode();
+
+	    /*  Define the new token, unless disallowed  */
+	    add_to_current( statbuf, nextfcode, definer, define_token);
+
+	    /*  Emit appropriate FCodes:  Type of def'n,   */
+	    switch ( hdr_flag )
+	    {
+		case FLAG_HEADERS:
+		    emit_token("named-token");
+		    break;
+
+		case FLAG_EXTERNAL:
+		    emit_token("external-token");
+		    break;
+
+		default:  /*   FLAG_HEADERLESS   */
+		    emit_token("new-token");
+		    emit_token_name = FALSE;
+	    }
+
+	    /*  Emit name of token, if applicable  */
+	    if ( emit_token_name )
+	    {
+		if ( force_tokens_case )
+		{
+		    if ( force_lower_case_tokens )
+		    {
+			strlwr( statbuf);
+		    }else{
+			strupr( statbuf);
+		    }
+		}
+		emit_string((u8 *)statbuf, wlen);	
+	    }
+
+	    /*  Emit the new token's FCode   */
+	    emit_fcode(nextfcode);
+
+	    /*  Prepare FCode Assignment Counter for next definition   */
+	    bump_fcode();
+
+	    /*  Declare victory   */
+	    retval = TRUE;
+	}
+    }
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  cannot_apply
+ *      Synopsis:       Print error message of the form:
+ *                     "Cannot apply <func> to <targ>, which is a <def'n>"
+ *
+ *      Inputs:
+ *         Parameters:
+ *             func_nam                    The name of the function
+ *             targ_nam                    The name of the target
+ *             defr                        The numeric-code of the definer-type
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Printout:
+ *             The error message is the entire printout of this routine
+ *
+ *      Error Detection:
+ *          Error was detected by calling routine
+ *
+ *      Process Explanation:
+ *          The calling routine already looked up the definer for its
+ *              own purposes, so we don't need to do that again here.
+ *
+ *      Still to be done:
+ *          If the definer-name is not found, we might still look up
+ *              the target name in the various vocabularies and use
+ *              a phrase for those.  E.g., if it is a valid token,
+ *              we could say it's defined as a "primitive".  (I'm
+ *              not sure what we'd say about an FWord...)
+ *
+ **************************************************************************** */
+
+static void cannot_apply( char *func_nam, char *targ_nam, fwtoken defr)
+{
+    char *defr_name = "" ;
+    const char *defr_phrase = ", which is defined as a " ;
+
+    if ( ! definer_name(defr, &defr_name) )
+    {
+	defr_phrase = "";
+    }
+
+    tokenization_error ( TKERROR , 
+	"Cannot apply  %s  to  %s %s%s.\n",
+	     func_nam, targ_nam, defr_phrase, defr_name );
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_with_definer
+ *      Synopsis:       Return pointer to data-structure of named word,
+ *                      if it's valid in Current context, and supply its
+ *                      definer.  If it's not valid in Current context,
+ *                      see if it might be a Local, and supply that definer.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             stat_name                  Name to look up
+ *             *definr                    Pointer to place to put the definer.
+ *
+ *      Outputs:
+ *         Returned Value:                Pointer to data-structure, or
+ *                                            NULL if not in Current context.
+ *         Supplied Pointers:
+ *             *definr                    Definer; possibly LOCAL_VAL
+ *
+ *      Process Explanation:
+ *          If the name is not found in the Current context, and does not
+ *              exist as a Local, *definr will remain unchanged.
+ *
+ *      Extraneous Remarks:
+ *          This is an odd duck^H^H^H^H^H^H^H^H^H^H^H^H a highly-specialized 
+ *              routine created to meet some corner-case needs engendered by
+ *              the conversion to tic_hdr_t vocabularies all around, combined
+ *              with an obsessive urge to preserve a high level of detail in
+ *              our error-messages.
+ *
+ **************************************************************************** */
+
+static tic_hdr_t *lookup_with_definer( char *stat_name, fwtoken *definr )
+{
+    tic_hdr_t *retval = lookup_current( stat_name);
+    if ( retval != NULL )
+    {
+         *definr = retval->fword_defr;
+    }else{
+        if ( exists_as_local( stat_name) ) *definr = LOCAL_VAL;
+    }
+    return ( retval );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  validate_to_target
+ *      Synopsis:       Print a message if the intended target
+ *                          of the  TO  directive is not valid
+ *      
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             statbuf             Next symbol to be read from the input stream
+ *             pc                  Input-source Scanning pointer
+ *
+ *      Outputs:
+ *         Returned Value:         TRUE = Allow  b(to)  token to be output.
+ *         Global Variables:
+ *             statbuf             Advanced to next symbol; must be re-read
+ *             pc                  Advanced, then restored to previous value
+ *
+ *      Error Detection:
+ *          If next symbol is not a valid target of  TO , issue ERROR    
+ *              message.  Restored  pc  will cause the next symbol to
+ *              be processed by ordinary means.
+ *          Allow  b(to)  token to be output in selected cases.  Even if
+ *              user has set the "Ignore Errors" flag, certain targets are
+ *              still too risky to be allowed to follow a  b(to)  token;
+ *              if "Ignore Errors" is not set, output won't get created
+ *              anyway.
+ *          Issue ERROR in the extremely unlikely case that "to" is the
+ *              last word in the Source.
+ *
+ *      Process Explanation:
+ *          Valid targets for a TO directive are words defined by:
+ *              DEFER, VALUE and arguably VARIABLE.  We will also allow
+ *              CONSTANT, but will still issue an Error message.
+ *          After the check, restore  pc ; this was only a look-ahead.
+ *              Also restore  lineno  and  abs_token_no 
+ *
+ *      Extraneous Remarks:
+ *          Main part of the mechanism to detect attempts to use the  TO 
+ *              directive to change the values of CONSTANTs in particular
+ *              and of inappropriate targets in general.
+ *
+ **************************************************************************** */
+
+static bool validate_to_target( void )
+{
+    signed long wlen;
+    tic_hdr_t *test_entry;
+    u8 *saved_pc = pc;
+    char *cmd_cpy = strupr( strdup( statbuf));    /*  For error message  */
+    unsigned int saved_lineno = lineno;
+    unsigned int saved_abs_token_no = abs_token_no;
+    fwtoken defr = UNSPECIFIED ;
+    bool targ_err = TRUE ;
+    bool retval = FALSE ;
+
+    wlen = get_word();
+    if ( wlen <= 0 )
+    {
+	warn_unterm( TKERROR, cmd_cpy, saved_lineno);
+    }else{
+
+	test_entry = lookup_with_definer( statbuf, &defr);
+	if ( test_entry != NULL )
+	{
+	    switch (defr)
+	    {
+		case VARIABLE:
+		    tokenization_error( WARNING,
+			"Applying %s to a VARIABLE (%s) is "
+			"not recommended; use  !  instead.\n",
+			cmd_cpy, statbuf);
+		case DEFER:
+		case VALUE:
+		    targ_err = FALSE ;
+		case CONST:
+		    retval = TRUE ;
+		/*  No default needed, likewise, no breaks;      */
+		/*  but some compilers get upset without 'em...  */
+		default:
+		    break;
+	    }
+	}
+
+	if ( targ_err )
+	{
+	    cannot_apply(cmd_cpy, strupr(statbuf), defr );
+	}
+
+	pc = saved_pc;
+	lineno = saved_lineno;
+	abs_token_no = saved_abs_token_no;
+    }
+    free( cmd_cpy);
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  you_are_here
+ *      Synopsis:       Display a generic Advisory of the Source command
+ *                          or directive encountered and being processed
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             statbuf                The command being processed 
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Printout:
+ *             Advisory message
+ *
+ **************************************************************************** */
+
+static void you_are_here( void)
+{
+    tokenization_error( INFO,
+	"%s encountered; processing...\n",
+	    strupr(statbuf) );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  fcode_starter
+ *      Synopsis:       Respond to one of the "FCode Starter" words
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             token_name         The FCode-token for this "Starter" word
+ *             spread             The separation between tokens.
+ *             is_offs16          Whether we are using a 16-bit number
+ *                                    for branch- (and suchlike) -offsets,
+ *                                    or the older-style 8-bit offset numbers.
+ *         Global Variables:
+ *            iname               Input-File name, used to set ifile_name 
+ *                                    field of  current_device_node
+ *            lineno              Current Input line number, used to set
+ *                                    line_no field of  current_device_node
+ *         Local Static Variables:
+ *            fcode_started       If this is TRUE, we have an Error.
+ *            first_fc_starter    Control calling  reset_fcode_ranges() ;
+ *                                    only on the first fcode_starter of
+ *                                    a tokenization.
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *         Global Variables:
+ *            offs16              Global "16-bit-offsets" flag
+ *            current_device_node   The ifile_name and line_no fields will be
+ *                                    loaded with the current input file name
+ *                                    and line number.  This node will be the
+ *                                    top-level device-node.
+ *            FCode Ranges will be reset the first time per tokenization
+ *                that this routine is entered.
+ *            A new FCode Range will be started every time after that.
+ *         Local Static Variables:
+ *            fcode_started       Set to TRUE.  We invoke the starter only
+ *                                    once per image-block.
+ *            first_fc_starter    Reset to FALSE if not already
+ *         Memory Allocated
+ *             Duplicate of Input-File name
+ *         When Freed?
+ *             In  fcode_ender()
+ *
+ *      Error Detection:
+ *          Spread of other than 1 -- Warning message.
+ *          "FCode Starter" previously encountered -- Warning and ignore.
+ *
+ *      Question under consideration:
+ *          Do we want directives -- such as definitions of constants --
+ *              supplied before the "FCode Starter", to be considered as
+ *              taking place in "Tokenizer Escape" mode?  That would mean
+ *              the "Starter" functions must be recognized in "Tokenizer
+ *              Escape" mode.  Many ramifications to be thought through...
+ *          I think I'm coming down strongly on the side of "No".  The user
+ *              who wants to do that can very well invoke "Tokenizer Escape"
+ *              mode explicitly.
+ *
+ **************************************************************************** */
+
+static void fcode_starter( const char *token_name, int spread, bool is_offs16)
+{
+    you_are_here();
+    if ( spread != 1 )
+    {
+        tokenization_error( WARNING, "spread of %d not supported.\n", spread);
+    }
+    if ( fcode_started )
+    {
+        tokenization_error( WARNING,
+	    "Only one "FCode Starter" permitted per tokenization.  "
+		"Ignoring...\n");
+    } else {
+
+	emit_fcodehdr(token_name);
+	offs16 = is_offs16;
+	fcode_started = TRUE;
+
+	current_device_node->ifile_name = strdup(iname);
+	current_device_node->line_no = lineno;
+
+	if ( first_fc_starter )
+	{
+	    reset_fcode_ranges();
+	    first_fc_starter = FALSE;
+	}else{
+	    set_next_fcode( nextfcode);
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  fcode_end_err_check
+ *      Synopsis:       Do error-checking at end of tokenization,
+ *                          whether due to FCODE-END or end-of-file,
+ *                          and reset the indicators we check.
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Global Variables:
+ *             Data-Stack depth     Is anything left on the stack?
+ *
+ *      Outputs:
+ *         Returned Value:                NONE
+ *         Global Variables:
+ *             Data-Stack           Reset to empty
+ *
+ *      Error Detection:
+ *          Unresolved control structures detected by clear_control_structs()
+ *          If anything is left on the stack, it indicates some incomplete
+ *              condition; we will treat it as a Warning.
+ *
+ **************************************************************************** */
+
+static void fcode_end_err_check( void)
+{
+    bool stack_imbal = BOOLVAL( stackdepth() != 0 );
+
+	if ( stack_imbal )
+	{
+	    tokenization_error( WARNING,
+		"Stack imbalance before end of tokenization.\n");
+	}
+    clear_stack();
+    clear_control_structs("End of tokenization");
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  fcode_ender
+ *      Synopsis:       Respond to one of the "FCode Ender" words:
+ *                          The FCode-token for "End0" or "End1"
+ *                              has already been written to the
+ *                              FCode Output buffer.
+ *                          Finish the FCode header:  fill in its
+ *                              checksum and length.
+ *                          Reset the token names defined in "normal" mode
+ *                          (Does not reset the FCode-token number)
+ *
+ *      Associated FORTH words:                 END0, END1
+ *      Associated Tokenizer directive:         FCODE-END
+ *
+ *      Inputs:
+ *         Parameters:            NONE
+ *         Global Variables:
+ *             incolon            If TRUE, a colon def'n has not been completed
+ *             last_colon_filename         For error message.
+ *             last_colon_lineno           For error message.
+ *             scope_is_global             For error detection
+ *             is_instance                 For error detection
+ *
+ *      Outputs:
+ *         Returned Value:        NONE
+ *         Global Variables:
+ *             haveend            Set to TRUE
+ *             fcode_started      Reset to FALSE.  Be ready to start anew.
+ *             FCode-defined tokens, aliases and macros -- i.e., those
+ *                 *NOT* defined in tokenizer-escape mode -- are reset.
+ *                 (Also, command-line-defined symbols are preserved).
+ *             Vocabularies will be reset
+ *             Device-node data structures will be deleted
+ *             Top-level device-node ifile_name and line_no fields
+ *                 will be reset.
+ *         Memory Freed
+ *             Duplicate of Input-File name, in top-level device-node.
+ *         Printout:
+ *             Advisory message giving current value of nextfcode
+ *                 (the "FCode-token Assignment Counter")
+ *
+ *      Error Detection:
+ *          ERROR if a Colon definition has not been completed.
+ *          ERROR if "instance" is still in effect
+ *          WARNING if Global-Scope has not been terminated; compensate.
+ *
+ *      Extraneous Remarks:
+ *          In order to accommodate odd cases, such as multiple FCode blocks
+ *          within a single PCI header, this routine does not automatically
+ *          reset nextfcode  to h# 0800
+ *
+ **************************************************************************** */
+
+void fcode_ender(void)
+{
+    if ( incolon )
+    {
+	char *tmp_iname = iname;
+	iname = last_colon_filename;
+	unterm_is_colon = TRUE;
+	warn_unterm( TKERROR, "Colon Definition", last_colon_lineno);
+	iname = tmp_iname;    
+    }
+    
+    haveend = TRUE;
+
+    if ( is_instance )
+    {
+	unresolved_instance( TKERROR);
+    }
+
+    if ( scope_is_global )
+    {
+        tokenization_error( WARNING ,
+	    "No DEVICE-DEFINITIONS directive encountered before end.  "
+		"Compensating...\n");
+	resume_device_scope();
+    }
+    fcode_end_err_check();
+    reset_normal_vocabs();
+    finish_fcodehdr();
+    fcode_started = FALSE;
+
+    if ( current_device_node->ifile_name != default_top_dev_ifile_name )
+    {
+	free( current_device_node->ifile_name );
+	current_device_node->ifile_name = default_top_dev_ifile_name;
+	current_device_node->line_no = 0;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  get_token
+ *      Synopsis:       Read the next word in the input stream and retrieve
+ *                          its FCode-token number.  If it's not a symbol to
+ *                          which a single token is assigned (e.g., if it's
+ *                          a macro), report an error.
+ *
+ *      Associated FORTH words:                   [']  '
+ *      Associated Tokenizer directive:          F[']
+ *
+ *      Inputs:
+ *         Parameters:
+ *             *tok_entry             Place to put the pointer to token entry
+ *         Global Variables:
+ *             statbuf                The command being processed 
+ *             pc                     Input stream character pointer
+ *
+ *      Outputs:
+ *         Returned Value:            TRUE if successful (i.e., no error)
+ *         Supplied Pointers:
+ *             *tok_entry             The token entry, if no error
+ *         Global Variables:
+ *             statbuf                The next word in the input stream
+ *             pc                     Restored to previous value if error
+ *
+ *      Error Detection:
+ *          The next word in the input stream is expected to be on the
+ *              same line as the directive.  The  get_word_in_line()
+ *              routine will check for that.
+ *          If the next word in the input stream is not a symbol
+ *              for which a single-token FCode number is assigned,
+ *              report an ERROR and restore PC to its previous value.
+ *
+ **************************************************************************** */
+
+static bool get_token(tic_hdr_t **tok_entry)
+{
+    bool retval = FALSE;
+    u8 *save_pc;
+
+    /*  Copy of command being processed, for error message  */
+    char cmnd_cpy[FUNC_CPY_BUF_SIZE+1];
+    strncpy( cmnd_cpy, statbuf, FUNC_CPY_BUF_SIZE);
+    cmnd_cpy[FUNC_CPY_BUF_SIZE] = 0;   /*  Guarantee null terminator. */
+
+    save_pc = pc;
+
+    if ( get_word_in_line( statbuf) )
+    {
+	fwtoken defr = UNSPECIFIED;
+
+	/*  We need to scan the newest definitions first; they
+	 *      might supercede standard ones.  We need, though,
+	 *      to bypass built-in FWords that need to trigger
+	 *      some tokenizer internals before emitting their
+	 *      synonymous FCode Tokens, (e.g., version1 , end0 ,
+	 *      and start{0-4}); if we find one of those, we will
+	 *      need to search again, specifically within the list
+	 *      of FCode Tokens.
+	 */
+	*tok_entry = lookup_with_definer( statbuf, &defr);
+	if ( *tok_entry != NULL )
+	{
+	    /*  Built-in FWords can be uniquely identified by their
+	     *      definer,  BI_FWRD_DEFN .  The definer for "shared"
+	     *      FWords is  COMMON_FWORD  but there are none of
+	     *      those that might be synonymous with legitimate
+	     *      FCode Tokens, nor are any likely ever to be...
+	     */
+	    if ( defr == BI_FWRD_DEFN )
+	    {
+	        *tok_entry = lookup_token( statbuf);
+		retval = BOOLVAL( *tok_entry != NULL );
+	    }else{
+		retval = entry_is_token( *tok_entry);
+	    }
+	}
+
+	if ( INVERSE( retval) )
+	{
+	    cannot_apply( cmnd_cpy, strupr(statbuf), defr );
+	    pc = save_pc;
+	}
+    }
+
+    return ( retval );
+}
+
+
+static void base_change ( int new_base )
+{
+    if ( incolon && ( INVERSE( in_tokz_esc) ) )
+    {
+        emit_literal(new_base );
+	emit_token("base");
+	emit_token("!");
+    } else {
+        base = new_base;
+    }
+}
+
+static void base_val (int new_base)
+{
+    u8  *old_pc;
+
+    char base_cmnd[FUNC_CPY_BUF_SIZE+1];
+    strncpy( base_cmnd, statbuf, FUNC_CPY_BUF_SIZE);
+    base_cmnd[FUNC_CPY_BUF_SIZE] = 0;  /* Guarantee NULL terminator */
+
+    old_pc=pc;
+    if ( get_word_in_line( statbuf) )
+    {
+	u8 basecpy=base;
+
+	base = new_base;
+	if ( ! handle_number() )
+	{
+	    /*  We did get a word on the line, but it's not a valid number */
+	    tokenization_error( WARNING ,
+		 "Applying %s to non-numeric value.  Ignoring.\n",
+		      strupr(base_cmnd) );
+	    pc = old_pc;
+	}
+	base=basecpy;
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  eval_string
+ *      Synopsis:       Prepare to tokenize a string, artificially generated
+ *                          by this program or created as a user-defined
+ *                          Macro.   When done, resume at existing source.
+ *                      Keep the file-name and line-number unchanged.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             inp_bufr          String (or buffer) to evaluate
+ *
+ *      Outputs:
+ *         Returned Value:       NONE
+ *         Global Variables, changed by call to init_inbuf():
+ *             start             Points to given string
+ *             pc                         ditto
+ *             end               Points to end of given string
+ *
+ *      Revision History:
+ *          Updated Thu, 23 Feb 2006 by David L. Paktor
+ *              This routine no longer calls its own instance of  tokenize()
+ *              It has become the gateway to the mechanism that makes a
+ *                  smooth transition between the body of the Macro, User-
+ *                  defined Symbol or internally-generated string and the
+ *                  resumption of processing the source file. 
+ *              A similar (but more complicated) transition when processing
+ *                  an FLOADed file will be handled elsewhere.
+ *          Updated Fri, 24 Feb 2006 by David L. Paktor
+ *              In order to support Macro-recursion protection, this routine
+ *                  is no longer the gateway for Macros; they will have to
+ *                  call push_source() directly.
+ *
+ **************************************************************************** */
+
+void eval_string( char *inp_bufr)
+{
+    push_source( NULL, NULL, FALSE);
+    init_inbuf( inp_bufr, strlen(inp_bufr));
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  finish_or_new_device
+ *      Synopsis:       Handle the shared logic for the NEW-DEVICE and
+ *                          FINISH-DEVICE commands.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             finishing_device		   TRUE for FINISH-DEVICE,
+ *					       FALSE for NEW-DEVICE
+ *         Global Variables:
+ *             incolon			     TRUE if inside a colon definition
+ *             noerrors 		     TRUE if ignoring errors
+ *             scope_is_global		     TRUE if "global scope" in effect
+ *         Local Static Variables:
+ *             is_instance		     TRUE if "instance" is in effect
+ *             dev_change_instance_warning   TRUE if warning hasn't been issued
+ *
+ *      Outputs:
+ *         Returned Value: 		     NONE
+ *         Local Static Variables:
+ *             dev_change_instance_warning   FALSE if warning is issued
+ *             instance_definer_gap	     TRUE if "instance" is in effect
+ *
+ *      Error Detection:
+ *          NEW-DEVICE and FINISH-DEVICE should not be used outside of
+ *              a colon-definition if global-scope is in effect.  Error
+ *              message; no further action unless we are ignoring errors.
+ *          Issue a WARNING if INSTANCE wasn't resolved before the current
+ *              device-node is changed.  Try not to be too repetitive...
+ *
+ *      Process Explanation:
+ *          The words NEW-DEVICE and FINISH-DEVICE may be incorporated into
+ *              a colon-definition, whether the word is defined in global-
+ *              or device- -scope.  Such an incorporation does not effect
+ *              a change in the device-node vocabulary; simply emit the token.
+ *          If we are in interpretation mode, though, we need to check for
+ *              errors before changing the device-node vocabulary:
+ *          If global-scope is in effect, we need to check whether we are
+ *              ignoring errors; if so, we will compensate by switching to  
+ *              device-scope.
+ *          If "instance" is in effect, it's "dangling".  It will remain
+ *              in effect through a device-node change, but this is very
+ *              bad style and deserves a WARNING, but only one for each
+ *              occurrence.  It would be unaesthetic, to say the least,
+ *              to have multiple messages for the same dangling "instance"
+ *              in a "finish-device   new-device" sequence.
+ *           We must be careful about the order we do things, because of
+ *              the messages printed as a side-effect of the node change...
+ *
+ *      Extraneous Remarks:
+ *          I will violate strict structure here.
+ *
+ **************************************************************************** */
+
+static void finish_or_new_device( bool finishing_device )
+{
+    if ( INVERSE( incolon ) )
+    {
+	if ( INVERSE( is_instance) )
+	{
+	    /*  Arm warning for next time:         */
+	    dev_change_instance_warning = TRUE;
+	}else{
+	    /*  Dangling "instance"                */
+	    instance_definer_gap = TRUE;
+	    /*   Warn only once.                   */
+	    if ( dev_change_instance_warning )
+	    {
+		unresolved_instance( WARNING);
+		dev_change_instance_warning = FALSE;
+	    }
+	}
+
+	/*  Note:  "Instance" cannot be in effect during "global" scope  */ 
+	if ( scope_is_global )
+	{
+	    glob_not_allowed( TKERROR, noerrors );
+	    if ( noerrors )
+	    {
+		 resume_device_scope();
+	    }else{
+		 return;
+	    }
+	}
+
+	if ( finishing_device )
+	{
+	     finish_device_vocab();
+	}else{
+	     new_device_vocab();
+	}
+    }
+    emit_token( finishing_device ? "finish-device" : "new-device" );
+	}
    
-	emit_fcode(nextfcode);
-	nextfcode++;
+	
+/* **************************************************************************
+ *
+ *      Function name:  abort_quote
+ *      Synopsis:       Optionally implement the   ABORT"  function as
+ *                      though it were a macro.  Control whether to allow
+ *                      it, and which style to support, via switches set
+ *                      on the command-line at run-time.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tok                       Numeric-code associated with the
+ *                                           FORTH word that was just read.
+ *         Global Variables:
+ *             enable_abort_quote        Whether to allow ABORT"
+ *             sun_style_abort_quote     SUN-style versus Apple-style
+ *             abort_quote_throw         Whether to use -2 THROW vs ABORT
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE if it was handled
+ *         Global Variables:
+ *             report_multiline              Reset to FALSE.
+ *         Printout:
+ *             ADVISORY:   ABORT" in fcode is not defined by IEEE 1275-1994
+ *
+ *      Error Detection:
+ *          Performed by other routines.  If user selected not to
+ *              allow  ABORT" , it will simply be treated as an
+ *              unknown word.
+ *          The string following it, however, will still be consumed.
+ *
+ *      Process Explanation:
+ *          If the supplied  tok  was not  ABORTTXT , then return FALSE.
+ *          If the  enable_abort_quote  flag is FALSE, consume the
+ *              string following the Abort" token, but be careful to
+ *              leave the  Abort" token in statbuf, as it will be used
+ *              for the error message.
+ *          Otherwise, create and prepare for processing the appropriate Macro:
+ *              For Apple Style, we push the specified string onto the stack
+ *                  and do -2 THROW (and hope the stack unwinds correctly).
+ *              For Sun Style, we test the condition on top of the stack,
+ *                  and if it's true, print the specified string before we
+ *                  do the -2 THROW.
+ *          We perform the underlying operations directly:  placing an "IF"
+ *              (if Sun Style), then placing the string.  This bypasses
+ *              any issues of double-parsing, as well as of doubly checking
+ *              for a multi-line string.
+ *          Finally, we perform the operational equivalents of the remainder
+ *              of the command sequence.
+ *
+ *      Extraneous Remarks:
+ *          I would have preferred not to have to directly perform the under-
+ *              lying operations, and instead simply prepare the entire command
+ *              sequence in a buffer, but I needed to handle the case where
+ *              quote-escaped quotes are included in the string:  If the string
+ *              were simply to be reproduced into the buffer, the quote-escaped
+ *              quotes would appear as plain quote-marks and terminate the
+ *              string parsing prematurely, leaving the rest of the string
+ *              to be treated as code instead of text...
+ *          Also, the introduction of the variability of whether to do the
+ *               -2 THROW  or to compile-in the token for  ABORT  makes the
+ *              buffer-interpretation scheme somewhat too messy for my tastes.
+ *
+ **************************************************************************** */
+	
+static bool abort_quote( fwtoken tok)
+{
+    bool retval = FALSE;
+    if ( tok == ABORTTXT )
+    {
+	if ( ! enable_abort_quote )
+	{
+	    /* ABORT" is not enabled; we'd better consume the string  */
+	    char *save_statbuf;
+	    signed long wlen;
+	    save_statbuf = strdup( (char *)statbuf);
+	    wlen = get_string( FALSE);
+	    strcpy( statbuf, save_statbuf);
+	    free( save_statbuf);
+	}else{
+	    /* ABORT" is not to be used in FCODE drivers
+	     * but Apple drivers do use it. Therefore we
+	     * allow it. We push the specified string to
+	     * the stack, do -2 THROW and hope that THROW
+	     * will correctly unwind the stack.
+	     * Presumably, Apple Source supplies its own
+	     *  IF ... THEN
+	     */
+	    char *abort_string;
+	    signed long wlen;

-	return 0;
+	    retval = TRUE;
+	    tokenization_error (INFO, "ABORT" in fcode not "
+			    "defined by IEEE 1275-1994\n");
+	    test_in_colon("ABORT"", TRUE, TKERROR, NULL);
+	    wlen=get_string( TRUE);
+
+	    if ( sun_style_abort_quote )  emit_if();
+
+	    emit_token("b(")");
+	    emit_string(statbuf, wlen);
+	
+	    if ( sun_style_abort_quote )  emit_token("type");
+
+	    if ( abort_quote_throw )
+	    {
+		emit_literal( -2);
+		emit_token("throw");
+	    }else{
+		emit_token("abort");
+	}
+		
+	    if ( sun_style_abort_quote )  emit_then();
+	        /*  Sun Style  */
+		abort_string = " type -2 THROW THEN:" ;
 }
+	}
+    return( retval );
+}

-static void encode_file( const char *filename )
+
+/* **************************************************************************
+ *
+ *      Function name:  create_alias
+ *      Synopsis:       Create an alias, as specified by the user
+ *
+ *      Associated FORTH word:                 ALIAS
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             incolon                Colon-def'n-in-progress indicator
+ *             in_tokz_esc            "Tokenizer Escape" mode indicator
+ *         Input Stream
+ *             Two words will be read.
+ *
+ *      Outputs:
+ *         Returned Value:            TRUE if succeeded.
+ *         Global Variables:    
+ *             statbuf                New name will be copied back into here.
+ *         Memory Allocated
+ *             The two words will be copied into freshly-allocated memory 
+ *                 that will be passed to the create_..._alias()  routine.
+ *         When Freed?
+ *             When Current Device Vocabulary is "finished", or at end
+ *                 of tokenization, or upon termination of program.
+ *             If not able to create alias, the copies will be freed here.
+ *
+ *      Error Detection:
+ *          If the ALIAS command was given during colon-definition, that
+ *              can be handled by this tokenizer, but it is not supported
+ *              by IEEE 1275-1994.  Issue a WARNING.
+ *          If the new name is a copy of an existing word-name, issue a warning.
+ *          If the word to which an alias is to be created does not exist
+ *              in the appropriate mode -- relative to "Tokenizer-Escape" --
+ *              that is an ERROR.
+ *          If "instance" is in effect, the ALIAS command is an ERROR.
+ *
+ *      Process Explanation:
+ *          Get two words -- the new name and the "old" word -- from the
+ *              same line of input as the ALIAS command.
+ *          Copy the new name back into statbuf for use in trace_creation.
+ *          Determine whether or not we are in "Tokenizer-Escape" mode.
+ *              Subsequent searches will take place in that same mode.
+ *          If the "new" name already exists, issue a warning.
+ *          In each vocabulary applicable to the current mode -- i.e., 
+ *                  "Tokenizer-Escape" or "Normal" -- (except:  cannot
+ *                  make aliases to "Locals"):
+ *              Try using the  create_..._alias()  routine.
+ *              If it succeeds, we are done.
+ *          IMPORTANT:  The order in which we try the vocabularies MUST
+ *              match the order in which  tokenize_one_word()  searches them. 
+ *          If all the attempts failed, the "old" word does not exist;
+ *              declare an ERROR and free up the memory that was allocated.
+ *
+ *      Extraneous Remarks:
+ *          With the separation of the  tokenizer[  state, this
+ *              function has become too complicated to keep as a
+ *              simple  CASE  in the big  SWITCH  statement anymore...
+ *
+ *          I had earlier thought that it was sufficient to create a
+ *              macro linking the "new" name to the "old" word.  There
+ *              were too many cases, though, where that didn't work.
+ *              This is cleaner.
+ *
+ *          I will not be adhering to the strict rules of structure in
+ *              this routine, as it would get me nested too deeply...
+ *
+ *      Revision History:
+ *          Updated Tue, 10 Jan 2006 by David L. Paktor
+ *              Convert to  tic_hdr_t  type vocabularies.
+ *
+ **************************************************************************** */
+
+static bool create_alias( void )
 {
-	FILE *f;
-	size_t s;
-	int i=0;
+    char *new_alias ;
+
+    validate_instance(ALIAS);
+    if ( incolon )
+    {
+	 tokenization_error ( WARNING,
+	    "ALIAS during colon-definition "
+		"is not supported by IEEE 1275-1994\n");
+}
+    if ( get_word_in_line( "ALIAS") )
+    {
+
+	new_alias = strdup((char *)statbuf);
+
+	if (get_word_in_line( "ALIAS") )
+{
+	    char *old_name = strdup((char *)statbuf) ;
+
+	    /*  Copy the "new" alias name back into statbuf.
+	     *      This is a HACK ^H^H^H^H awkward way to retrofit
+	     *      support for the  trace_creation()  function.
+	     */
+	    strcpy( statbuf, new_alias);
+
+	    /*  We don't call  trace_creation()  here because we don't
+	     *      know if the creation succeeded.  However, we want
+	     *      to issue a "Duplicate" warning based on the attempt,
+	     *      even if it doesn't succeed.
+	     *  We would prefer to have the "Trace" message precede the 
+	     *      "Duplicate" warning, but we don't think it's worth
+	     *      the effort.  When it becomes worthwhile, the way to
+	     *      do it would be to factor out the block that handles
+	     *      normal-tokenization versus "Tokenizer-Escape" mode;
+	     *      condition the "Trace" message on its success-return, 
+	     *      show the "Duplicate" warning in any case, then show
+	     *      the error-message and do the cleanup conditioned on
+	     *      a failure-return.
+	     *  That will also obviate the need for a return value from
+	     *      this routine and for the copy-back into statbuf.
+	     */
+	    warn_if_duplicate(new_alias);
+
+	    /*
+	     *  Here is where we begin trying the  create_..._alias() 
+	     *      routines for the vocabularies.
+	     */
+
+	    /*
+	     *  Distinguish between "Normal" tokenization mode
+	     *  and "Tokenizer Escape" mode
+	     */
+	    if ( in_tokz_esc )
+	    {
+		if ( create_tokz_esc_alias( new_alias, old_name) )
+		    return(TRUE);
    
-	if( !(f=fopen(filename,"rb")) ) {
-		printf("%s:%d: opening '%s':\n", iname, lineno, filename );
-		ERROR;
-		return;
+		/*
+		 *  Handle the classes of operatives that are common between
+		 *      "Tokenizer Escape" mode and "Normal" tokenization mode.
+		 *  Those classes include selected non-fcode forth constructs
+		 *     and Conditional-Compilation Operators.
+		 */
+		{
+		    tic_hdr_t *found = lookup_shared_word( old_name);
+		    if ( found != NULL )
+		    {
+			if ( create_core_alias( new_alias, old_name) )
+			    return(TRUE);
+		    }
    }
-	while( (s=fread(statbuf, 1, 255, f)) ) {
-		emit_token("b(")");
-		emit_string(statbuf, s);
-		emit_token("encode-bytes");
-		if( i++ )
-			emit_token("encode+");
+	    }else{
+        	/*  "Normal" tokenization mode  */
+	
+		/*  Can create aliases for "Locals", why not?  */
+		if ( create_local_alias( new_alias, old_name) )
+		    return(TRUE);
+
+		/*
+		 *  All other classes of operatives -- non-fcode forth
+		 *      constructs, Standard and user-defined fcode
+		 *      tokens, Macros, and Conditional-Compilation
+		 *      Operators, -- are included in the "currently
+		 *      active" vocabulary.
+		 */
+
+		if ( create_current_alias( new_alias, old_name) )
+		    return(TRUE);
+	
+	    }    /*  End of separate handling for normal-tokenization mode
+        	  *      versus  "Tokenizer-Escape" mode
+		  */
+
+	    /*  It's not a word, a macro or any of that other stuff.  */
+	    tokenized_word_error(old_name);
+	    free(old_name);
    }
-	fclose( f );
+	free (new_alias);
+    }
+    return(FALSE);
 }

+	
+/* **************************************************************************
+ *
+ *      Function name:  string_err_check
+ *      Synopsis:       Error-check after processing or Ignoring
+ *                          simple strings
+ *
+ *      Inputs:
+ *         Parameters:
+ *             is_paren           TRUE if string is Dot-Paren  .( 
+ *                                    FALSE if Ess-Quote  ( s"  )
+ *             sav_lineno         Saved Line Number, for Unterminated Error
+ *             strt_lineno        Start Line Number, for Multiline Warning
+ *         Global Variables:
+ *             noerrors 	  TRUE if ignoring errors
+ *         Local Static Variables:
+ *             got_until_eof      TRUE if reached end of buffer before delim.
+ *
+ *      Outputs:
+ *         Returned Value:        TRUE if did not reach end of buffer, or,
+ *                                    if ignoring errors, TRUE anyway.
+ *
+ *      Error Detection:
+ *          Multi-line warning, "Unterminated" Error messages, as apppropriate
+ *
+ **************************************************************************** */

-static void handle_internal(u16 tok)
+static  bool string_err_check( bool is_paren,
+                                  unsigned int sav_lineno,
+				      unsigned int strt_lineno )
 {
-	unsigned long wlen;
-	long offs1,offs2;
-	u16 itok;
+    bool retval = noerrors ;
+    char *item_typ = is_paren ?
+	"Dot-Paren" : "Ess-Quote" ;
+    if ( got_until_eof )   /*  Crude retrofit... */
+    {
+	warn_unterm( TKERROR, item_typ, sav_lineno );
+    }else{
+	retval = TRUE;
+	warn_if_multiline( item_typ, strt_lineno );
+	}
+    return( retval);
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_internal
+ *      Synopsis:       Perform the functions associated with FORTH words
+ *                      that do not map directly to a single token.  This
+ *                      is the functions that will go into the FUNCT field
+ *                      of entries in the "FWords" and "Shared Words" lists.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             pfield               Param-field of the  tic_hdr_t  -type entry
+ *                                      associated with the FORTH-Word (FWord)
+ *                                      just read that is being "handled".
+ *         Global Variables:
+ *             statbuf              The word that was just read.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Global Variables:
+ *             statbuf              More words may be read.
+ *
+ *      Error Detection:
+ *          Too numerous to list here...
+ *
+ *      Process Explanation:
+ *          Recast the type of the param-field of a  tic_hdr_t -type
+ *              entry and rename it "tok".
+ *          The "tok" will be used as the control-expression for a
+ *              SWITCH statement with a large number of CASE labels.
+ *              Both "FWords" and "shared_words" list entries will
+ *              be processed by this routine.
+ *      
+ *      Revision History:
+ *      Updated Wed, 20 Jul 2005 by David L. Paktor
+ *          Put handling of  ABORT"  under control of a run-time
+ *              command-line switch.
+ *          Put decision to support IBM-style Locals under control
+ *              of a run-time command-line switch.
+ *      Updated Tue, 17 Jan 2006 by David L. Paktor
+ *          Convert to handler for  tic_hdr_t  type vocab entries.
+ *
+ *      Extraneous Remarks:
+ *          We would prefer to keep this function private, so we will
+ *              declare its prototype here and in the one other file
+ *              where we need it, namely, dictionary.c, rather than
+ *              exporting it widely in a  .h  file.
+ *
+ **************************************************************************** */
+
+void handle_internal( tic_param_t pfield);
+void handle_internal( tic_param_t pfield)
+{
+	fwtoken tok = pfield.fw_token;
+
+	signed long wlen;
+	unsigned int sav_lineno = lineno;    /*  For error message  */
+
+	bool handy_toggle = TRUE ;   /*  Various uses...   */
    
 #ifdef DEBUG_SCANNER
    printf("%s:%d: debug: tokenizing control word '%s'\n",
@@ -443,727 +4450,1087 @@
 #endif
    switch (tok) {
    case BEGIN:
-		emit_token("b(<mark)");
-		dpush((unsigned long)opc);
+		emit_begin();
    	break;

case BUFFER:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(buffer:)");
+		}
    	break;

case CONST:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(constant)");
+		}
    	break;

case COLON:
-		create_word();
+		{
+		    /*  Collect error- -detection or -reporting items,
+		     *      but don't commit until we're sure the
+		     *      creation was a success.
+		     */
+		    u16 maybe_last_colon_fcode = nextfcode ;
+		    unsigned int maybe_last_colon_lineno = lineno;
+		    unsigned int maybe_last_colon_abs_token_no = abs_token_no;
+		    unsigned int maybe_last_colon_do_depth = do_loop_depth;
+		    /*  last_colon_defname
+		     *     has to wait until after call to  create_word()
+		     */
+
+		    if ( create_word(tok) )
+		    {
+			last_colon_fcode = maybe_last_colon_fcode;
+			last_colon_lineno = maybe_last_colon_lineno;
+			last_colon_abs_token_no = maybe_last_colon_abs_token_no;
+			last_colon_do_depth = maybe_last_colon_do_depth;
+			collect_input_filename( &last_colon_filename);
+			/*  Now we can get  last_colon_defname  */
+			if ( last_colon_defname != NULL )
+			{
+			    free( last_colon_defname);
+			}
+			last_colon_defname = strdup(statbuf);
+
    	emit_token("b(:)");
    	incolon=TRUE;
+			hide_last_colon();
+			lastcolon = opc;
+		    }
+		}
    	break;
    
    case SEMICOLON:
+		if ( test_in_colon("SEMICOLON", TRUE, TKERROR, NULL) )
+		{
+		    ret_stk_balance_rpt( "termination,", TRUE);
+		    /*  Clear Control Structures just back to where
+		     *      the current Colon-definition began.
+		     */
+		    clear_control_structs_to_limit(
+			"End of colon-definition", last_colon_abs_token_no);
+
+		    if ( ibm_locals )
+		    {
+			finish_locals();
+			forget_locals();
+		    }
+
    	emit_token("b(;)");
    	incolon=FALSE;
+		    reveal_last_colon();
+		}
    	break;

case CREATE:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(create)");
+		}
    	break;

case DEFER:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(defer)");
+		}
    	break;

+	case ALLOW_MULTI_LINE:
+		report_multiline = FALSE;
+		break;
+
+	case OVERLOAD:
+		if ( test_in_colon(statbuf, FALSE, WARNING, NULL) )
+		{
+		    do_not_overload = FALSE;
+		}
+		break;
+
+	case DEFINED:
+		if (get_word_in_line( statbuf) )
+		{
+		    eval_user_symbol(statbuf);
+		}
+		break;
+
+	case CL_FLAG:
+		if (get_word_in_line( statbuf) )
+		{
+		     set_cl_flag( statbuf, TRUE);
+		}
+		break;
+
+	case SHOW_CL_FLAGS:
+		show_all_cl_flag_settings( TRUE);
+		break;
+
    case FIELD:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(field)");
+		}
    	break;

case VALUE:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(value)");
+		}
    	break;
    	
    case VARIABLE:
-		create_word();
+		if ( create_word(tok) )
+		{
    	emit_token("b(variable)");
+		}
    	break;

-	case EXTERNAL:
-		flags=FLAG_EXTERNAL;
-		break;
-
-	case TOKENIZE:
-		emit_token("b(')");
-		break;
-
    case AGAIN:
-		emit_token("bbranch");
-		offs1=dpop()-(unsigned long)opc;
-		emit_offset(offs1);
+		emit_again();
    	break;

case ALIAS:
-		wlen=get_word();
-		name=strdup((char *)statbuf);
-		wlen=get_word();
-		alias=strdup((char *)statbuf);
-		if(lookup_macro(name))
-			printf("%s:%d: warning: duplicate alias\n", 
-							iname, lineno);
-		add_macro(name,alias);
+		if ( create_alias() )
+		{
+		    trace_creation( ALIAS, statbuf);
+		}
    	break;

case CONTROL:
-		wlen=get_word();
-		emit_token("b(lit)");
-		emit_num32(statbuf[0]&0x1f);
+		if ( get_word_in_line( statbuf) )
+		{
+		    emit_literal(statbuf[0]&0x1f);
+		}
    	break;

case DO:
    	emit_token("b(do)");
-		dpush((unsigned long)opc);
-		emit_offset(0);
-		dpush((unsigned long)opc);
+		mark_do();
    	break;

case CDO:
    	emit_token("b(?do)");
-		dpush((unsigned long)opc);
-		emit_offset(0);
-		dpush((unsigned long)opc);
+		mark_do();
    	break;

case ELSE:
-		offs2=dpop();
-		emit_token("bbranch");
-		dpush((unsigned long)opc);
-		emit_offset(0);
-		emit_token("b(>resolve)");
-		offs1=(unsigned long)opc;
-		opc=(u8 *)offs2;
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		opc=(u8 *)offs1;
+		emit_else();
    	break;

case CASE:
-		emit_token("b(case)");
-		dpush(0);
+		emit_case();
    	break;

case ENDCASE:
-		/* first emit endcase, then calculate offsets. */
-		emit_token("b(endcase)");
-		
-		offs1=(unsigned long)opc;
-		
-		offs2=dpop();
-		while (offs2) {
-			u16 tmp;
-			
-			opc=(u8 *)(ostart+offs2);
+		emit_endcase();
+		break;

-			/* we're using an fcode offset to temporarily
-			 * store our linked list. At this point we take
-			 * the signed fcode offset as unsigned as we know
-			 * that the offset will always be >0. This code
-			 * is still broken for any case statement that
-			 * occurs after the fcode bytecode grows larger 
-			 * than 64kB
-			 */
-			tmp=receive_offset();
+	case NEW_DEVICE:
+		handy_toggle = FALSE;
+	case FINISH_DEVICE:
+		finish_or_new_device( handy_toggle );
+		break;

-			offs2=(unsigned long)offs1-(unsigned long)opc;
-
-#if defined(DEBUG_SCANNER)
-			printf ("%s:%d: debug: endcase offset 0x%lx\n",
-				iname,lineno, (unsigned long)offs2);
-#endif
-			emit_offset(offs2);
-			offs2=tmp;
+	case FLITERAL:
+		{
+		    u32 val;
+		    val = dpop();
+		    emit_literal(val);
    	}
-		opc=(u8 *)offs1;
    	break;

case OF:
-		emit_token("b(of)");
-		dpush((unsigned long)(opc-ostart));
-		emit_offset(0);
+		emit_of();
    	break;

case ENDOF:
-		offs1=dpop();
-		emit_token("b(endof)");
+		emit_endof();
+		break;
    	
-		offs2=dpop();
-		dpush((unsigned long)(opc-ostart));
-		emit_offset(offs2);
-		
-		offs2=(unsigned long)(opc-ostart);
-		opc=(u8 *)(ostart+offs1);
-		offs1=offs2-offs1;
-		emit_offset(offs1);
-		
-		opc=(u8 *)(ostart+offs2);
+	case EXTERNAL:
+		set_hdr_flag( FLAG_EXTERNAL );
    	break;
    	
    case HEADERLESS:
-		flags=0;
+		set_hdr_flag( FLAG_HEADERLESS );
    	break;
    
    case HEADERS:
-		flags=FLAG_HEADERS;
+		set_hdr_flag( FLAG_HEADERS );
    	break;

case DECIMAL:
    	/* in a definition this is expanded as macro "10 base !" */
-		if (incolon) {
-			emit_token("b(lit)");
-			emit_num32(0x0a);
-			emit_token("base");
-			emit_token("!");
-		} else
-			base=10;
+		base_change ( 0x0a );
    	break;
    	
    case HEX:
-		if (incolon) {
-			emit_token("b(lit)");
-			emit_num32(0x10);
-			emit_token("base");
-			emit_token("!");
-		} else
-			base=16;
+		base_change ( 0x10 );
    	break;

case OCTAL:
-		if (incolon) {
-			emit_token("b(lit)");
-			emit_num32(0x08);
-			emit_token("base");
-			emit_token("!");
-		} else
-			base=8;
+		base_change ( 0x08 );
    	break;

case OFFSET16:
    	if (!offs16)
-			printf("%s:%d: switching to "
-				"16bit offsets.\n", iname, lineno);
+		{
+		    tokenization_error(INFO, "Switching to 16-bit offsets.\n");
+		}else{
+		    tokenization_error(WARNING,
+			"Call of OFFSET16 is redundant.\n");
+		}
    	emit_token("offset16");
    	offs16=TRUE;
    	break;

case IF:
-		emit_token("b?branch");
-		dpush((unsigned long)opc);
-		emit_offset(0);
+		emit_if();
    	break;

-	case CLEAVE:
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Correct analysis of Return-Stack usage within Do-Loops
+ *              or before Loop Elements like I and J or UNLOOP or LEAVE.
+ *
+ **************************************************************************** */
+	case UNLOOP:
+		emit_token("unloop");
+		must_be_deep_in_do(1);
+		break;
+
    case LEAVE:
    	emit_token("b(leave)");
+		must_be_deep_in_do(1);
    	break;
+
+	case LOOP_I:
+		emit_token("i");
+		must_be_deep_in_do(1);
+		break;
+
+	case LOOP_J:
+		emit_token("j");
+		must_be_deep_in_do(2);
+		break;
    	
    case LOOP:
    	emit_token("b(loop)");
-		offs1=dpop();
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		offs1=(unsigned long)opc;
-		opc=(u8 *)dpop();
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		opc=(u8 *)offs1;
+		resolve_loop();
    	break;
    	
-	case CLOOP:
+	case PLUS_LOOP:
    	emit_token("b(+loop)");
-		offs1=dpop();
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		offs1=(unsigned long)opc;
-		opc=(u8 *)dpop();
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		opc=(u8 *)offs1;
+		resolve_loop();
    	break;

-	case GETTOKEN:
-		emit_token("b(')");
-		wlen=get_word();
-		itok=lookup_token((char *)statbuf);
-		if (itok==0xffff) {
-			printf("%s:%d: error: no such word '%s' in [']\n",
-					iname, lineno, statbuf);
-			ERROR;
+
+	case INSTANCE:
+		{
+		    bool set_instance_state = FALSE;
+		    bool emit_instance = TRUE;
+		    /*  We will treat "instance" in a colon-definition as
+		     *      an error, but allow it to be emitted if we're
+		     *      ignoring errors; if we're not ignoring errors,
+		     *      there's no output anyway...
+		     */
+		    if ( test_in_colon(statbuf, FALSE, TKERROR, NULL) )
+		    {   /*   We are in interpretation (not colon) state.  */
+			/*  "Instance" not allowed during "global" scope  */ 
+			if ( scope_is_global )
+			{
+			    glob_not_allowed( WARNING, FALSE );
+			    emit_instance = FALSE;
+			}else{
+			    set_instance_state = TRUE;
+			}
+		    }
+		    if ( emit_instance )
+		    {
+			if ( set_instance_state )
+			{
+			    /*  "Instance" isn't cumulative....  */
+			    if ( is_instance )
+			    {
+				unresolved_instance( WARNING);
+			    }
+			    collect_input_filename( &instance_filename);
+			    instance_lineno = lineno;
+			    is_instance = TRUE;
+			    dev_change_instance_warning = TRUE;
+			}
+			emit_token("instance");
+		    }
    	}
-		/* FIXME check size, u16 or token */
-		emit_fcode(itok);
    	break;
    	
-	case ASCII:
-		wlen=get_word();
-		emit_token("b(lit)");
-		emit_num32(statbuf[0]);
+	case GLOB_SCOPE:
+		if ( test_in_colon(statbuf, FALSE, TKERROR, NULL) )
+		{
+		    if ( INVERSE( is_instance) )
+		    {
+			enter_global_scope();
+		    }else{
+			tokenization_error( TKERROR,
+			    "Global Scope not allowed.  "
+			    ""Instance" is in effect; issued" );
+			just_where_started( instance_filename,
+					        instance_lineno );
+		    }
+		}
    	break;

-	case CHAR:
-		if (incolon)
-			printf("%s:%d: warning: CHAR cannot be used inside "
-				"of a colon definition.\n", iname, lineno);
-		wlen=get_word();
-		emit_token("b(lit)");
-		emit_num32(statbuf[0]);
+	case DEV_SCOPE:
+		if ( test_in_colon(statbuf, FALSE, TKERROR, NULL) )
+		{
+		    resume_device_scope();
+		}
    	break;

+	case TICK:             /*    '    */
+		test_in_colon(statbuf, FALSE, WARNING, "[']");
+	case BRACK_TICK:       /*   [']   */
+		{
+		    tic_hdr_t *token_entry;
+		    if ( get_token( &token_entry) )
+		    {
+			emit_token("b(')");
+			/* Emit the token; warning or whatever comes gratis */
+			token_entry->funct( token_entry->pfield);
+		    }
+		}
+		break;
+
+	case F_BRACK_TICK:     /*  F['] <name>
+	        		*     emits the token-number for <name>
+				*  Mainly useful to compute the argument
+				*     to   get-token   or  set-token
+				*/
+		{
+		    tic_hdr_t *token_entry;
+		    if ( get_token( &token_entry) )
+		    {
+			/*  "Obsolete" warning doesn't come gratis here...  */
+			token_entry_warning( token_entry);
+			/*  In Tokenizer-Escape mode, push the token  */
+			if ( in_tokz_esc )
+			{
+			    dpush( token_entry->pfield.deflt_elem);
+			}else{
+			    emit_literal( token_entry->pfield.deflt_elem);
+			}
+		    }
+		}
+		break;
+
+	case CHAR:
+		handy_toggle = FALSE;
    case CCHAR:
-		wlen=get_word();
-		emit_token("b(lit)");
-		emit_num32(statbuf[0]);
+		test_in_colon(statbuf, handy_toggle, WARNING,
+		    handy_toggle ? "CHAR" : "[CHAR]" );
+	case ASCII:
+		if ( get_word_in_line( statbuf) )
+		{
+		    emit_literal(statbuf[0]);
+		}
    	break;
    	
    case UNTIL:
-		emit_token("b?branch");
-		emit_offset(dpop()-(unsigned long)opc);
+		emit_until();
    	break;

case WHILE:
-		emit_token("b?branch");
-		dpush((unsigned long)opc);
-		emit_offset(0);
+		emit_while();
    	break;
    	
    case REPEAT:
-		emit_token("bbranch");
-		offs2=dpop();
-		offs1=dpop();
-		offs1-=(unsigned long)opc;
-		emit_offset(offs1);
-		
-		emit_token("b(>resolve)");
-		offs1=(unsigned long)opc;
-		opc=(u8 *)offs2;
-		emit_offset(offs1-offs2);
-		opc=(u8 *)offs1;
+		emit_repeat();
    	break;
    	
    case THEN:
-		emit_token("b(>resolve)");
-		offs1=(unsigned long)opc;
-		opc=(u8 *)dpop();
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		opc=(u8 *)offs1;
+		emit_then();
    	break;

+	case IS:
+		tokenization_error ( INFO,
+		     "Substituting  TO  for deprecated  IS\n");
    case TO:
+		if ( validate_to_target() )
+		{
    	emit_token("b(to)");
+		}
    	break;

case FLOAD:
+		if ( get_word_in_line( statbuf) )
    	{
-			u8 *oldstart, *oldpc, *oldend;
-			char *oldiname;
-			int oldlineno;
-
-			wlen=get_word();
+		    bool stream_ok ;
    		
-			oldstart=start; oldpc=pc; oldend=end; 
-			oldiname=iname; oldlineno=lineno;
+		    push_source( close_stream, NULL, TRUE) ;
    		
-			init_stream((char *)statbuf);
-			tokenize();
-			close_stream();
+		    tokenization_error( INFO, "FLOADing %s\n", statbuf );
    		
-			iname=oldiname; lineno=oldlineno;
-			end=oldend; pc=oldpc; start=oldstart;
+		    stream_ok = init_stream( statbuf );
+		    if ( INVERSE( stream_ok) )
+		    {
+			drop_source();
+		    }
    	}
    	break;

-	case STRING:
-		if (*pc++=='\n') lineno++;
-		wlen=get_string();
+	case STRING:         /*  Double-Quote ( " ) string  */
+		handy_toggle = FALSE;
+	case PSTRING:        /*  Dot-Quote  ( ." ) string   */
+		wlen=get_string( TRUE);
    	emit_token("b(")");
-		emit_string(statbuf,wlen-1);
+		emit_string(statbuf, wlen);
+		if ( handy_toggle )
+		{
+		    emit_token("type");
+		}
    	break;

-	case PSTRING:
+	case SSTRING:        /*  Ess-Quote  ( s"  ) string  */
+		handy_toggle = FALSE;
+	case PBSTRING:       /*  Dot-Paren  .(   string  */
    	if (*pc++=='\n') lineno++;
-		wlen=get_string();
+		{
+		    unsigned int strt_lineno = lineno;
+		    wlen = get_until( handy_toggle ? ')' : '"' );
+		    if ( string_err_check( handy_toggle,
+		             sav_lineno, strt_lineno) )
+		    {
    	emit_token("b(")");
-		emit_string(statbuf,wlen-1);
+			emit_string(statbuf, wlen);
+			if ( handy_toggle )
+			{
    	emit_token("type");
+			}
+		    }
+		}
    	break;

-	case PBSTRING:
-		if (*pc++=='\n') lineno++;
-		get_until(')');
+	case FUNC_NAME:
+		if ( test_in_colon( statbuf, TRUE, TKERROR, NULL) )
+		{
+		    if ( in_tokz_esc )
+		    {
+			tokenization_error( P_MESSAGE, "Currently" );
+			in_last_colon();
+		    }else{
    	emit_token("b(")");
-		emit_string(statbuf,strlen((char *)statbuf)-1);
-		emit_token("type");
+			emit_string( last_colon_defname,
+		            strlen( last_colon_defname) );
+			/*  if ( hdr_flag == FLAG_HEADERLESS ) { WARNING } */
+		    }
+		}
    	break;

-	case SSTRING:
-		if (*pc++=='\n') lineno++;
-		get_until('"');
+	case IFILE_NAME:
    	emit_token("b(")");
-		emit_string(statbuf,strlen((char *)statbuf)-1);
-		pc++; /* skip " */
+		emit_string( iname, strlen( iname) );
    	break;

+	case ILINE_NUM:
+		emit_literal( lineno);
+		break;
+			
    case HEXVAL:
-		{
-			u8 basecpy=base;
-			long val;
-			
-			base=0x10;
-			wlen=get_word();
-			if (!get_number(&val)) {
-				emit_token("b(lit)");
-				emit_num32(val);
-			} else {
-				printf("%s:%d: warning: illegal value in h#"
-						" ignored\n", iname, lineno);
-			}
-			base=basecpy;
-		}
+		base_val (0x10);
    	break;
    	
    case DECVAL:
-		{
-			u8 basecpy=base;
-			long val;
-
-			base=0x0a;
-			wlen=get_word();
-			if (!get_number(&val)) {
-				emit_token("b(lit)");
-				emit_num32(val);
-			} else {
-				printf("%s:%d: warning: illegal value in d#"
-						" ignored\n", iname, lineno);
-			}
-			
-			base=basecpy;
-		}
+		base_val (0x0a);
    	break;
    	
    case OCTVAL:
+		base_val (8);
+		break;
+
+	case ASC_LEFT_NUM:
+		handy_toggle = FALSE;
+	case ASC_NUM:
+		if (get_word_in_line( statbuf) )
    	{
-			u8 basecpy=base;
-			long val;
-
-			base=0x08;
-			wlen=get_word();
-			if (!get_number(&val)) {
-				emit_token("b(lit)");
-				emit_num32(val);
+		    if ( handy_toggle )
+		    {
+			ascii_right_number( statbuf);
    		} else {
-				printf("%s:%d: warning: illegal value in o#"
-						" ignored\n", iname, lineno);
+			ascii_left_number( statbuf);
    		}
-
-			base=basecpy;
    	}
    	break;
-	case BEGINTOK:
-		intok=TRUE;
+
+	case CONDL_ENDER:   /*  Conditional directives out of context  */
+	case CONDL_ELSE:
+		tokenization_error ( TKERROR,
+		    "No conditional preceding %s directive\n",
+			strupr(statbuf) );
    	break;

-	case EMITBYTE:
-		if (intok)
-			emit_byte(dpop());
-		else 
-			printf ("%s:%d: warning: emit-byte outside tokenizer"
-					" scope\n", iname, lineno);
+	case PUSH_FCODE:
+		tokenization_error( INFO,
+		    "FCode-token Assignment Counter of 0x%x "
+		    "has been saved on stack.\n", nextfcode );
+		dpush( (long)nextfcode );
    	break;

-	case NEXTFCODE:
-		if (intok)
-			nextfcode=dpop();
-		else
-			printf("%s:%d: warning: next-fcode outside tokenizer"
-					" scope\n", iname, lineno);
+	case POP_FCODE:
+		pop_next_fcode();
    	break;
+
+	case RESET_FCODE:
+		tokenization_error( INFO,
+		    "Encountered %s.  Resetting FCode-token "
+			"Assignment Counter.  ", strupr(statbuf) );
+		list_fcode_ranges( FALSE);
+		reset_fcode_ranges();
+		break;
    	
-	case ENDTOK:
-		intok=FALSE;
+	case EXIT:
+		if ( test_in_colon( statbuf, TRUE, TKERROR, NULL)
+		     || noerrors )
+		{
+		    ret_stk_balance_rpt( NULL, FALSE);
+		    if ( ibm_locals )
+		    {
+			finish_locals ();
+		    }
+		    emit_token("exit");
+		}
    	break;
+
+	case ESCAPETOK:
+		enter_tokz_esc();
+		break;
    
    case VERSION1:
    case FCODE_V1:
-		printf("%s:%d: using version1 header\n", iname, lineno);
-		emit_token("version1");
-		fcode_hdr=opc;
-		emit_fcodehdr();
-		offs16=FALSE;
+		tokenization_error( INFO, "Using version1 header "
+		    "(8-bit offsets).\n");
+		fcode_starter( "version1", 1, FALSE) ;
    	break;
    
    case START1:
    case FCODE_V2:
    case FCODE_V3: /* Full IEEE 1275 */
-		emit_token("start1");
-		fcode_hdr=opc;
-		emit_fcodehdr();
-		offs16=TRUE;
+		fcode_starter( "start1", 1, TRUE);
    	break;
    	
    case START0:
-		printf ("%s:%d: warning: spread of 0 not supported.", 
-				iname, lineno);
-		emit_token("start0");
-		fcode_hdr=opc;
-		emit_fcodehdr();
-		offs16=TRUE;
+		fcode_starter( "start0", 0, TRUE);
    	break;
    	
    case START2:
-		printf ("%s:%d: warning: spread of 2 not supported.", 
-			iname, lineno);
-		emit_token("start2");
-		fcode_hdr=opc;
-		emit_fcodehdr();
-		offs16=TRUE;
+		fcode_starter( "start2", 2, TRUE);
    	break;
    	
    case START4:
-		printf ("%s:%d: warning: spread of 4 not supported.", 
-			iname, lineno);
-		emit_token("start4");
-		fcode_hdr=opc;
-		emit_fcodehdr();
-		offs16=TRUE;
+		fcode_starter( "start4", 4, TRUE);
    	break;
    	
+	case END1:
+		tokenization_error( WARNING, 
+		    "Appearance of END1 in FCode source code "
+			"is not intended by IEEE 1275-1994\n");
+		handy_toggle = FALSE;
    case END0:
    case FCODE_END:
-		haveend=TRUE;
-		emit_token("end0");
+		if ( handy_toggle )
+		{
+		    you_are_here();
+		}
+		emit_token( handy_toggle ? "end0" : "end1" );
+		fcode_ender();
+		FFLUSH_STDOUT
    	break;

-	case END1:
-		haveend=TRUE;
-		emit_token("end1");
+	case RECURSE:
+		if ( test_in_colon(statbuf, TRUE, TKERROR, NULL ) )
+		{
+		    emit_fcode(last_colon_fcode);
+		}
    	break;
    	
+
    case RECURSIVE:
+		if ( test_in_colon(statbuf, TRUE, TKERROR, NULL ) )
+		{
+		    reveal_last_colon();
+		}
    	break;

+	case RET_STK_FETCH:
+		ret_stk_access_rpt();
+		emit_token( "r@");
+		break;
+
+	case RET_STK_FROM:
+		ret_stk_access_rpt();
+		bump_ret_stk_depth( -1);
+		emit_token( "r>");
+		break;
+
+	case RET_STK_TO:
+		bump_ret_stk_depth( 1);
+		emit_token( ">r");
+		break;
+
    case PCIHDR:
-		{
-			u32 classid=dpop();
-			u16 did=dpop();
-			u16 vid=dpop();
-			
-			pci_hdr=opc;
-			emit_pcihdr(vid, did, classid);
-			
-			printf("%s:%d: PCI header vendor id=0x%04x, "
-				"device id=0x%04x, class=%06x\n",
-				iname, lineno, vid, did, classid);
-		}
+		emit_pcihdr();
    	break;
    
    case PCIEND:
-		if (!pci_hdr) {
-			printf("%s:%d: error: pci-header-end/pci-end "
-				"without pci-header\n", iname, lineno);
-			ERROR;
-		}
    	finish_pcihdr();
+		reset_fcode_ranges();
+		FFLUSH_STDOUT
    	break;

case PCIREV:
-		pci_revision=dpop();
-		printf("%s:%d: PCI header revision=0x%04x\n", 
-				iname, lineno, pci_revision);
+		pci_image_rev = dpop();
+		tokenization_error( INFO,
+		    "PCI header revision=0x%04x%s\n", pci_image_rev,
+			big_end_pci_image_rev ?
+			    ".  Will be saved in Big-Endian format."
+			    : ""  );
    	break;

case NOTLAST:
-		pci_is_last_image=FALSE;
-		printf("%s:%d: PCI header not last image!\n",
-				iname, lineno);
+		handy_toggle = FALSE;
+	case ISLAST:
+		dpush(handy_toggle);
+	case SETLAST:
+		{
+		    u32 val = dpop();
+		    bool new_pili = BOOLVAL( (val != 0) );
+		    if ( pci_is_last_image != new_pili )
+		    {
+			tokenization_error( INFO,
+			    new_pili ?
+				"Last image for PCI header.\n" :
+				"PCI header not last image.\n" );
+			pci_is_last_image = new_pili;
+		    }
+		}
    	break;
    	
-	case ABORTTXT:
-		/* ABORT" is not to be used in FCODE drivers
-		 * but Apple drivers do use it. Therefore we
-		 * allow it. We push the specified string to
-		 * the stack, do -2 THROW and hope that THROW
-		 * will correctly unwind the stack.
-		 */
-		
-		printf("%s:%d: warning: ABORT" in fcode not defined by "
-				"IEEE 1275-1994.\n", iname, lineno);
-		if (*pc++=='\n') lineno++;
-		wlen=get_string();
-		
-#ifdef BREAK_COMPLIANCE
-		/* IF */
-		emit_token("b?branch");
-		
-		dpush((unsigned long)opc);
-		emit_offset(0);
-#endif
-		emit_token("b(")");
-		emit_string(statbuf,wlen-1);
-#ifdef BREAK_COMPLIANCE
-		emit_token("type");
-#endif
-		emit_token("-2");
-		emit_token("throw");
-#ifdef BREAK_COMPLIANCE
-		/* THEN */
-		emit_token("b(>resolve)");
-		offs1=(unsigned long)opc;
-		opc=(u8 *)dpop();
-		offs2=offs1-(unsigned long)opc;
-		emit_offset(offs2);
-		opc=(u8 *)offs1;
-#endif
+	case SAVEIMG:
+		if (get_word_in_line( statbuf) )
+		{
+		    free(oname);
+		    oname = strdup( statbuf );
+		    tokenization_error( INFO,
+			"Output is redirected to file:  %s\n", oname);
+		}
    	break;

-	case FCODE_DATE:
+	case RESETSYMBS:
+		tokenization_error( INFO,
+		    "Resetting symbols defined in %s mode.\n",
+			in_tokz_esc ? "tokenizer-escape" : ""normal"");
+		if ( in_tokz_esc )
    	{
-			time_t tt;
-			char fcode_date[11];
-			
-			tt=time(NULL);
-			strftime(fcode_date, 11, "%m/%d.%Y", 
-					localtime(&tt));
-			emit_token("b(")");
-			emit_string((u8 *)fcode_date,10);
+		    reset_tokz_esc();
+		}else{
+		    reset_normal_vocabs();
    	}	
    	break;

+	case FCODE_DATE:
+		handy_toggle = FALSE;
    case FCODE_TIME:
    	{
    		time_t tt;
-			char fcode_time[9];
+		    char temp_buffr[32];
    		
    		tt=time(NULL);
-			strftime(fcode_time, 9, "%H:%M:%S", 
-					localtime(&tt));
+		    if ( handy_toggle )
+		    {
+			strftime(temp_buffr, 32, "%T %Z", localtime(&tt));
+		    }else{
+			strftime(temp_buffr, 32, "%m/%d/%Y", localtime(&tt));
+		    }
+		    if ( in_tokz_esc )
+		    {
+			tokenization_error( MESSAGE, temp_buffr);
+		    }else{
    		emit_token("b(")");
-			emit_string((u8 *)fcode_time,8);
+			emit_string((u8 *)temp_buffr, strlen(temp_buffr) );
+		    }
    	}
    	break;

case ENCODEFILE:
-		wlen=get_word();
+		if (get_word_in_line( statbuf) )
+		{
    	encode_file( (char*)statbuf );
+		}
    	break;

default:
-		printf("%s:%d: error: Unimplemented control word '%s'\n",
-				iname, lineno, statbuf);
-		ERROR;
+	    /*  IBM-style Locals, under control of a switch  */
+	    if ( ibm_locals )
+	    {
+		bool found_it = TRUE;
+		switch (tok) {
+		    case CURLY_BRACE:
+			declare_locals( FALSE);
+			break;
+		    case DASH_ARROW:
+			assign_local();
+			break;
+		    default:
+			found_it = FALSE;
    }
+		if ( found_it ) break;
 }

-void tokenize(void)
+	    /*  Down here, we have our last chance to recognize a token.
+	     *      If  abort_quote  is disallowed, we will still consume
+	     *      the string.  In case the string spans more than one
+	     *      line, we want to make sure the line number displayed
+	     *      in the error-message is the one on which the disallowed
+	     *       abort_quote  token appeared, not the one where the
+	     *      string ended; therefore, we might need to be able to
+	     *      "fake-out" the line number...
+	     */
 {
-	unsigned long wlen;
-	u16 tok;
-	u8 *mac;
-	long val;
+		bool fake_out_lineno = FALSE;
+		unsigned int save_lineno = lineno;
+		unsigned int true_lineno;
+		if ( abort_quote( tok) )
+		{   break;
+		}else{
+		    if ( tok == ABORTTXT )  fake_out_lineno = TRUE;
+		}
+		true_lineno = lineno;
+
+		if ( fake_out_lineno )  lineno = save_lineno;
+		tokenization_error ( TKERROR,
+		    "Unimplemented control word '%s'\n", strupr(statbuf) );
+		if ( fake_out_lineno )  lineno = true_lineno;
+	    }
+	}
+}
    
-	while ((wlen=get_word())!=0) {
+/* **************************************************************************
+ *
+ *      Function name:  skip_string
+ *      Synopsis:       When Ignoring, skip various kinds of strings.  Maps
+ *                          to string-handlers in handle_internal()...
+ *
+ *      Associated FORTH words:                 Double-Quote ( " ) string
+ *                                              Dot-Quote  ( ." ) string
+ *                                              Ess-Quote  ( s"  ) string
+ *                                              Dot-Paren  .(   string
+ *                                              ABORT" (even if not enabled)
+ *             { (Local-Values declaration) and -> (Local-Values assignment)
+ *                  are also handled if  ibm_locals  is enabled.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             pfield               Param-field of the entry associated with
+ *                                      the FWord that is being Ignored.
+ *         Global Variables:
+ *             statbuf              The word that was just read.
+ *             pc                   Input-stream pointer
+ *             lineno               Line-number, used for errors and warnings
+ *             ibm_locals           TRUE if IBM-style Locals are enabled
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *
+ *      Error Detection:
+ *          Multi-line warnings, "Unterminated" Errors
+ *              handled by called routines
+ *
+ *      Extraneous Remarks:
+ *          We would prefer to keep this function private, too, so we
+ *              will declare its prototype here and in the one other
+ *              file where we need it, namely, dictionary.c, rather
+ *              than exporting it widely in a  .h  file.
+ *
+ **************************************************************************** */

-		/* Filter comments */
-		switch (statbuf[0]) {
-		case '\':
-			pc-=wlen;
-			get_until('\n');
-			continue;
+void skip_string( tic_param_t pfield);
+void skip_string( tic_param_t pfield)
+{
+    fwtoken tok = pfield.fw_token;
+    unsigned int sav_lineno = lineno;
+    bool handy_toggle = TRUE ;   /*  Various uses...   */
    		
-		case '(':
-			/* only a ws encapsulated '(' is a stack comment */
-			if (statbuf[1])
+    switch (tok) {
+    case STRING:         /*  Double-Quote ( " ) string    */
+    case PSTRING:        /*  Dot-Quote  ( ." ) string     */
+    case ABORTTXT:       /*  ABORT", even if not enabled  */
+	get_string( FALSE);   /*  Don't truncate; ignoring anyway  */
+	/*  Will handle multi-line warnings, etc.   */
    			break;
    		
-			pc-=wlen;
-			get_until(')');
+    case SSTRING:        /*  Ess-Quote  ( s"  ) string  */
+	handy_toggle = FALSE;
+    case PBSTRING:       /*  Dot-Paren  .(   string  */
    		if (*pc++=='\n') lineno++;
+	{
+	    unsigned int strt_lineno = lineno;
+	    get_until( handy_toggle ? ')' : '"' );
+	    string_err_check( handy_toggle, sav_lineno, strt_lineno );
+	}
+	break;
+
+    default:
+	/*  IBM-style Locals, under control of a switch  */
+	if ( ibm_locals )
+	{
+	    bool found_it = TRUE;
+	    switch (tok) {
+		case CURLY_BRACE:
+		    declare_locals( TRUE);
+		    break;
+		case DASH_ARROW:
+		    get_word();
+		    break;
+		default:
+		    found_it = FALSE;
+	    }
+	    if ( found_it ) break;
+	}
+
+	tokenization_error ( FATAL,  "Program Error.  "
+	    "Unimplemented skip-string word '%s'\n", strupr(statbuf) );
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  process_remark
+ *      Synopsis:       The active function for remarks (backslash-space)
+ *                          and comments (enclosed within parens)
+ *
+ *      Associated FORTH word(s):        \   (         
+ *
+ *      Inputs:
+ *         Parameters:
+ *             TIC entry "parameter field", init'd to delimiter character.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *
+ *      Error Detection:
+ *          Warning if end-of-file encountered before delimiter.
+ *          Warning if multi-line parentheses-delimited comment.
+ *
+ *      Process Explanation:
+ *          Skip until the delimiter.
+ *          If end-of-file was encountered, issue Warning.
+ *          Otherwise, and if delimiter was not new-line,
+ *              check for multi-line with Warning.
+ *
+ **************************************************************************** */
+
+void process_remark( tic_param_t pfield )
+{
+    char until_char = (char)pfield.deflt_elem ;
+    unsigned int start_lineno = lineno;
+
 #ifdef DEBUG_SCANNER
+
+    get_until(until_char);
    		printf ("%s:%d: debug: stack diagram: %s)\n",
    					iname, lineno, statbuf);
-#endif
-			continue;
-		}
+#else

-		/* Check whether a macro with given name exists */
+    if ( skip_until( until_char) )
+    {
+	if ( until_char == '\n' )
+	{
+	    /*  Don't need any saved line number here ...  */
+	    tokenization_error ( WARNING,
+		"Unterminated remark.\n");
+	}else{
+	    warn_unterm( WARNING, "comment", start_lineno);
+	}
+    }else{
+	if ( until_char != '\n' )
+	{
+	    pc++;
+	    warn_if_multiline( "comment", start_lineno);
+	}
+    }
+#endif  /*  DEBUG_SCANNER  */
+}
    	
-		mac=(u8 *)lookup_macro((char *)statbuf);
-		if(mac) {
-			u8 *oldstart, *oldpc, *oldend;
-#ifdef DEBUG_SCANNER
-			printf("%s:%d: debug: macro %s folds out to sequence"
-				" '%s'\n", iname, lineno, statbuf, mac);
-#endif	
-			oldstart=start; oldpc=pc; oldend=end;
-			start=pc=end=mac;
-			end+=strlen((char *)mac);
    		
-			tokenize();
+/* **************************************************************************
+ *
+ *      Function name:  filter_comments
+ *      Synopsis:       Process remarks and comments in special conditions
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             inword             Current word just parsed
+ *
+ *      Outputs:
+ *         Returned Value:        TRUE if Current word is a Comment-starter.
+ *                                    Comment will be processed
+ *
+ *      Process Explanation:
+ *          We want to be able to recognize any alias the user may have
+ *              defined to a comment-delimiter, in whatever applicable
+ *              vocabulary it might be.
+ *          The active-function of any such alias will, of necessity, be
+ *              the  process_remark()  routine, defined just above.
+ *          We will search for the TIC-entry of the given word; if we don't    
+ *              find it, it's not a comment-delimiter.  If we do find it, 
+ *              and it is one, we invoke its active-function and return TRUE.
+ *          We also want to permit the "allow-multiline-comments" directive   
+ *              to be processed in the context that calls this routine, so
+ *              we will check for that condition, too.
+ *
+ **************************************************************************** */

-			end=oldend; pc=oldpc; start=oldstart;
+bool filter_comments( u8 *inword)
+{
+    bool retval = FALSE;
+    tic_hdr_t *found = lookup_word( inword, NULL, NULL );
    		
-			continue;
+    if ( found != NULL )
+    {
+	if ( found->funct == process_remark )
+	{
+	    found->funct( found->pfield);
+	    retval = TRUE;
+	}else{
+	    /*  Permit the "allow-multiline-comments" directive  */
+	    if ( found->funct == handle_internal )
+	    {
+	        if ( found->pfield.fw_token == ALLOW_MULTI_LINE )
+		{
+		    /*   Make sure any intended side-effects occur...  */
+		    found->funct( found->pfield);
+		    retval = TRUE;
    	}
+	    }
+	}
+    }
+    return ( retval );
+		}

-		/* Check whether it's a non-fcode forth construct */
    	
-		tok=lookup_fword((char *)statbuf);
-		if (tok!=0xffff) {
-#ifdef DEBUG_SCANNER
-			printf("%s:%d: debug: matched internal opcode 0x%04x\n",
-					iname, lineno, tok);
-#endif
-			handle_internal(tok);
-#ifdef DEBUG_SCANNER
-			if (verbose)
-				printf("%s:%d: debug: '%s' done.\n",
-					iname,lineno,statbuf);
-#endif
-			continue;
-		}
+/* **************************************************************************
+ *
+ *      Function name:  tokenize_one_word
+ *      Synopsis:       Tokenize the currently-obtained word
+ *                          along with whatever it consumes.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             wlen       Length of symbol just retrieved from the input stream
+ *                              This is not really used here any more; it's
+ *                              left over from an earlier implementation.
+ *         Global Variables:        
+ *             statbuf      The symbol (word) just retrieved from input stream.
+ *             in_tokz_esc  TRUE if "Tokenizer-Escape" mode is in effect; a
+ *                              different set of vocabularies from "Normal"
+ *                              mode will be checked (along with those that
+ *                              are common to both modes).  
+ *             ibm_locals   Controls whether to check for IBM-style Locals;
+ *                              set by means of a command-line switch.
+ *
+ *      Outputs:
+ *         Returned Value:      NONE
+ *         Global Variables:         
+ *             statbuf          May be incremented    
+ *             in_tokz_esc      May be set if the word just retrieved is
+ *                                  the  tokenizer[   directive. 
+ *             tic_found        
+ *
+ *      Error Detection:
+ *           If the word could neither be identified nor processed as a number,
+ *               that is an ERROR; pass it to  tokenized_word_error  for a
+ *               message.
+ *
+ *      Process Explanation:
+ *          Look for the word in each of the various lists and vocabularies
+ *              in which it might be found, as appropriate to the current
+ *              state of activity.
+ *          If found, process it accordingly.
+ *          If not found, try to process it as a number.
+ *          If cannot process it as a number, declare an error.
+ *
+ *      Revision History:
+ *          Updated Tue, 10 Jan 2006 by David L. Paktor
+ *              Convert to  tic_hdr_t  type vocabularies.
+ *          Updated Mon, 03 Apr 2006 by David L. Paktor
+ *             Replaced bulky "Normal"-vs-"Escape" block with a call
+ *                 to  lookup_word .  Attend to a small but important
+ *                 side-effect of the "handle_<vocab>" routines that
+ *                 feeds directly into the protection against self-
+ *                 -recursion in a user-defined Macro:  Set the global
+ *                 variable  tic_found  to the entry, just before we
+ *                 execute it, and we're good to go... 
+ *
+ *      Extraneous Remarks:
+ *          We trade off the strict rules of structure for simplicity
+ *              of coding.
+ *
+ **************************************************************************** */
    	
-		/* Check whether it's one of the defined fcode words */
+void tokenize_one_word( signed long wlen )
+{
    	
-		tok=lookup_token((char *)statbuf);
-		if (tok!=0xffff) {
-#ifdef DEBUG_SCANNER
-			printf("%s:%d: debug: matched fcode no 0x%04x\n",
-					iname, lineno, tok);
-#endif
-			emit_fcode(tok);
-			continue;
-		}
+    /*  The shared lookup routine now handles everything.   */
+    tic_hdr_t *found = lookup_word( statbuf, NULL, NULL );
    	
-		/* It's not a word or macro - is it a number? */
+    if ( found != NULL )
+    {
+	tic_found = found;
+	found->funct( found->pfield);
+	return ;
+    }
    	
-		if (!get_number(&val)) {
-			if (intok)
-				dpush(val);
-			else {
-				emit_fcode(lookup_token("b(lit)"));
-				emit_num32(val);
+    /*  It's not a word in any of our current contexts.
+     *      Is it a number?
+     */
+    if ( handle_number() )
+    {
+	return ;
    		}
-			continue;
+
+    /*  Could not identify - give a shout. */
+    tokenized_word_error( statbuf );
    	}

-		/* could not identify - bail out. */
+/* **************************************************************************
+ *
+ *      Function name:  tokenize
+ *      Synopsis:       Tokenize the current input stream.
+ *                          May be called recursively for macros and such.
+ *
+ *      Revision History:
+ *      Updated Thu, 24 Mar 2005 by David L. Paktor
+ *          Factor-out comment-filtration; apply to  gather_locals
+ *          Factor-out tokenizing a single word (for conditionals)
+ *          Separate actions of "Tokenizer-Escape" mode.
+ *
+ **************************************************************************** */
+
+void tokenize(void)
+{
+    signed long wlen = 0;
    	
-		printf("%s:%d: error: word '%s' is not in dictionary.\n",
-				iname, lineno, statbuf);
-		ERROR;
+    while ( wlen >= 0 )
+    {
+	wlen = get_word();
+	if ( wlen > 0 )
+	{
+	    tokenize_one_word( wlen );
    }
+	}
 }

Added: fcode-utils/toke/scanner.h
===================================================================
--- fcode-utils/toke/scanner.h	                        (rev 0)
+++ fcode-utils/toke/scanner.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,123 @@
+#ifndef _TOKE_SCANNER_H
+#define _TOKE_SCANNER_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      External/Prototype definitions for Scanning functions in Tokenizer
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+
+#include "types.h"
+#include "ticvocab.h"
+
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */
+
+extern u8  *statbuf;           /*  The word just read from the input stream  */
+extern u8   base;	       /*  The numeric-interpretation base           */
+
+
+/* pci data */
+extern bool pci_is_last_image;
+extern u16  pci_image_rev;        /* Vendor's Image, NOT PCI Data Struct Rev */
+extern u16  pci_vpd;
+
+
+/*  Having to do with the state of the tokenization  */
+extern bool offs16;	     /*  Using 16-bit branch- (etc) -offsets  */
+extern bool in_tokz_esc;     /*  TRUE if in "Tokenizer Escape" mode   */
+extern bool incolon;	     /*  TRUE if inside a colon definition    */
+extern bool haveend;	     /*  TRUE if the "end" code was read.     */
+extern int do_loop_depth;    /*  How deep we are inside DO ... LOOP variants */
+
+/*  For special-case error detection or reporting */
+extern int lastcolon;	/*  Loc'n in output stream of latest colon-def'n.  */
+			/*  Used for error-checking of IBM-style Locals    */
+extern char *last_colon_defname;       /*  Name of last colon-definition     */
+extern char *last_colon_filename;      /*  File where last colon-def'n made  */
+extern unsigned int last_colon_lineno; /*  Line number of last colon-def'n   */
+extern bool report_multiline;          /*  False to suspend multiline warning */
+extern unsigned int last_colon_abs_token_no;
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+void	init_scanner( void );
+void	exit_scanner( void );
+void init_scan_state( void );
+void	 fcode_ender( void );
+
+bool skip_until( char lim_ch);
+void push_source( void (*res_func)(), _PTR res_parm, bool is_f_chg );
+signed long get_word( void);
+bool get_word_in_line( char *func_nam);
+bool get_rest_of_line( void);
+void warn_unterm( int severity,
+                            char *something,
+			        unsigned int saved_lineno);
+void warn_if_multiline( char *something, unsigned int start_lineno );
+void user_message( tic_param_t pfield );
+void skip_user_message( tic_param_t pfield );
+bool get_number( long *result);
+void eval_string( char *inp_bufr);
+
+void process_remark( tic_param_t pfield );
+bool filter_comments( u8 *inword);
+bool as_a_what( fwtoken definer, char *as_what);
+tic_hdr_t *lookup_word( char *stat_name, char **where_pt1, char **where_pt2 );
+bool word_exists( char *stat_name, char **where_pt1, char **where_pt2 );
+void warn_if_duplicate ( char *stat_name);
+void trace_creation( fwtoken definer, char *nu_name);
+void tokenize_one_word( signed long wlen );
+void check_name_length( signed long wlen );
+
+void	tokenize( void );
+
+/* **************************************************************************
+ *
+ *          Macro Name:   FUNC_CPY_BUF_SIZE
+ *                        Size of a temporary buffer to retain a copy of
+ *                        a function name taken from statbuf, when statbuf
+ *                        will be used to return a value, but the function
+ *                        name might still be needed for an error message.
+ *
+ **************************************************************************** */
+
+#define FUNC_CPY_BUF_SIZE  40
+
+
+#endif   /*  _TOKE_SCANNER_H    */

Modified: fcode-utils/toke/stack.c
===================================================================
--- fcode-utils/toke/stack.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/stack.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,93 +24,203 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>

-#include "toke.h"
 #include "stack.h"
+#include "scanner.h"
+#include "errhandler.h"

+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              statbuf          The word just read from the input stream
+ *
+ **************************************************************************** */

-#define GUARD_STACK
-#define EXIT_STACKERR

-#ifdef GLOBALSTACK
-#define STATIC static
-#else
-#define STATIC
-#endif
-STATIC long *dstack,*startdstack,*enddstack;
-#undef STATIC
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *              dstack         Pointer to current item on top of Data Stack
+ *
+ **************************************************************************** */

+long *dstack;
+
+/* **************************************************************************
+ *
+ *      Local/Static Pointers  .....      to  ...         -> Points to ...
+ *          startdstack         Start of data-stack area  -> last possible item
+ *          enddstack           End of data-stack area    -> past first item
+ *
+ *************************************************************************** */
+
+static long *startdstack;
+static long *enddstack;
+
+void clear_stack(void)
+{
+    dstack = enddstack;
+}
+
 /* internal stack functions */

-int init_stack(void)
+void init_stack(void)
 {
-	startdstack=enddstack=malloc(MAX_ELEMENTS*sizeof(long));
-	enddstack+=MAX_ELEMENTS;
+	startdstack = safe_malloc(MAX_ELEMENTS*sizeof(long), "initting stack");
+	enddstack = startdstack + MAX_ELEMENTS;
    dstack=enddstack;
-	return 0;
 }

-#ifdef GLOBALSTACK 
+/*  Input Param:  stat   TRUE = Underflow, FALSE = Overflow   */
+static void stackerror(bool stat)
+{
+    /*
+     *   Because all of our stack operations are protected and
+     *       we have no risk of the  dstack  pointer going into
+     *       invalid territory, this error needs not be  FATAL.
+     */
+    tokenization_error ( TKERROR , "stack %sflow at or near  %s \n",
+	 (stat)?"under":"over" , statbuf );
+}

-#ifdef GUARD_STACK
-static void stackerror(int stat)
+/*
+ *  Return TRUE if the stack depth is equal to or greater than
+ *      the supplied minimum requirement.  If not, print an error.
+ */
+bool min_stack_depth(int mindep)
 {
-  	printf ("FATAL: stack %sflow\n",
-		(stat)?"under":"over" );
-#ifdef EXIT_STACKERR
-	exit(-1);
-#endif
+    bool retval = TRUE ;
+    long *stack_result;
+
+    stack_result = dstack + mindep;
+    /*
+     *  The above appears counter-intuitive.  However, C compensates
+     *      for the size of the object of a pointer when it handles
+     *      address arithmetic.  A more explicit expression that would
+     *      yield the same result, might look something like this:
+     *
+     *        (long *)((int)dstack + (mindep * sizeof(long)))
+     *
+     *  I doubt that that form would yield tighter code, or otherwise
+     *      represent any material advantage...
+     */
+
+    if ( stack_result > enddstack )
+    {
+	retval = FALSE;
+	stackerror(TRUE);
+    }
+
+    return ( retval );
 }
-#endif

+/*
+ *  Return TRUE if the stack has room for the supplied # of items
+ */
+static bool room_on_stack_for(int newdep)
+{
+    bool retval = TRUE ;
+    long *stack_result;
+
+    stack_result = dstack - newdep;
+    /*  See above note about "counter-intuitive" pointer address arithmetic  */
+
+    if ( stack_result < startdstack )
+    {
+	retval = FALSE;
+	stackerror(FALSE);
+    }
+
+    return ( retval );
+}
+
 void dpush(long data)
 {
 #ifdef DEBUG_DSTACK
    printf("dpush: sp=%p, data=0x%lx, ", dstack, data);
 #endif
-	*(--dstack)=data;
-#ifdef GUARD_STACK
-	if (dstack<startdstack) stackerror(0);
-#endif
+	if ( room_on_stack_for(1) )
+	{
+	    --dstack;
+	    *(dstack)=data;
+	}
 }

long dpop(void)
 {
-  	long val;
+  	long val = 0;
 #ifdef DEBUG_DSTACK
    printf("dpop: sp=%p, data=0x%lx, ",dstack, *dstack);
 #endif
-	val=*(dstack++);
-#ifdef GUARD_STACK
-	if (dstack>enddstack) stackerror(1);
-#endif
+	if ( min_stack_depth(1) )
+	{
+	    val=*(dstack);
+	    dstack++;
+	}
    return val;
 }

long dget(void)
 {
-	return *(dstack);
+  	long val = 0;
+	if ( min_stack_depth(1) )
+	{
+	    val = *(dstack);
+	}
+	return val;
 }
-#endif

-/* Stack helper functions */
-
-u8 *get_stackstring(void)
+long stackdepth(void)
 {
-	long size;
-	u8   *fstring, *retstring;
+    long depth;

-	size=dpop();
-	fstring=(u8 *)dpop();
+    depth = enddstack - dstack;
+    /*
+     *  Again, C's address arithmetic compensation comes into play.
+     *      See the note at  min_stack_depth()
+     *
+     *  A more explicit equivalent expression might look like this:
+     *
+     *        (((long int)enddstack - (long int)dstack) / sizeof(long))
+     *
+     *  I doubt any material advantage with that one either...
+     */

-	retstring=malloc(size+1);
-	strncpy((char *)retstring, (const char *)fstring, size);
-	retstring[size]=0;
+    return (depth);
+}

-	return retstring;
+void swap(void)
+{
+    long nos_temp;    /*  Next-On-Stack temp  */
+    if ( min_stack_depth(2) )
+    {
+	nos_temp = dstack[1];
+	dstack[1]= dstack[0];
+	dstack[0]= nos_temp;
+    }
 }

+void two_swap(void)
+{
+    long two_deep, three_deep;
+    if ( min_stack_depth(4) )
+    {
+	two_deep   = dstack[2];
+	three_deep = dstack[3];
+	dstack[2]  = dstack[0];
+	dstack[3]  = dstack[1];
+	dstack[1]  = three_deep;
+	dstack[0]  = two_deep;
+    }
+}

+

Modified: fcode-utils/toke/stack.h
===================================================================
--- fcode-utils/toke/stack.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/stack.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _TOKE_STACK_H
+#define _TOKE_STACK_H
+
 /*
  *                     OpenBIOS - free your system! 
  *                         ( FCode tokenizer )
@@ -24,23 +27,47 @@
  *
  */

-#ifdef ANSI_ONLY
-#define GLOBALSTACK
-#endif
+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */

+#include "types.h"
+
+/* ************************************************************************** *
+ *
+ *      Macros:
+ *          MAX_ELEMENTS          Size of stack area, in "elements"
+ *
+ **************************************************************************** */
+
 #define MAX_ELEMENTS 1024

-#ifdef GLOBALSTACK
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */
+
+extern long *dstack;
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
 void dpush(long data);
 long dpop(void);
 long dget(void);
-#else
-extern long *dstack,*startdstack;
-static inline void dpush(long data) { *(--dstack)=data; }
-static inline long dpop(void) { return (long)*(dstack++); }
-static inline long dget(void) { return *(dstack); }
-#endif

-int init_stack(void);
-u8 *get_stackstring(void);
+void clear_stack(void);
+void init_stack(void);

+bool min_stack_depth(int mindep);   /*  TRUE if no error  */
+long stackdepth(void);
+void swap(void);
+void two_swap(void);
+
+#endif   /*  _TOKE_STACK_H    */

Modified: fcode-utils/toke/stream.c
===================================================================
--- fcode-utils/toke/stream.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/stream.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -24,6 +24,12 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #ifdef __GLIBC__
@@ -32,134 +38,1577 @@
 #include <string.h>
 #include <sys/stat.h>

+#include "emit.h"
+#include "stream.h"
+#include "errhandler.h"
 #include "toke.h"
-#include "stream.h"

-#define OUTPUT_SIZE	131072
+/* **************************************************************************
+ *
+ *      Revision History:
+ *      Updated Tue, 31 Jan 2006 by David L. Paktor
+ *          Add support for embedded Environment-Variables in path name
+ *      Updated Thu, 16 Feb 2006 David L. Paktor
+ *          Collect missing (inaccessible) filenames
+ *      Updated Thu, 16 Mar 2006 David L. Paktor
+ *          Add support for Include-Lists
+ *
+ **************************************************************************** */

-extern bool offs16;
-extern u16 nextfcode;
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *              start                 Start of input-source buffer
+ *              end                   End of input-source buffer
+ *              pc                    Input-source Scanning pointer
+ *              iname                 Current Input File name
+ *              lineno                Current Line Number in Input File
+ *              ostart                Start of Output Buffer
+ *              opc                   FCode Output Buffer Position Counter
+ *              oname                 Output File name
+ *
+ **************************************************************************** */

-u8 *start, *pc, *end;
-char *iname;

+/* Input pointers, Position Counters and Length counters */
+u8 *start = NULL;
+u8 *pc;
+u8 *end;
+char *iname = NULL;
+unsigned int lineno = 0;
+unsigned int abs_token_no = 0;  /*  Absolute Token Number in all Source Input
+                                 *      Will be used to identify position
+                                 *      where colon-definition begins and
+                                 *      to limit clearing of control-structs.
+                                 */
+static unsigned int ilen;   /*  Length of Input Buffer   */
+
 /* output pointers */
-u8 *ostart, *opc, *oend, *oname;
-static unsigned int ilen;
+u8 *ostart;
+char *oname = NULL;
+unsigned int opc;           /*  Output Position Counter  */

-unsigned int lineno;
+/* We want to limit exposure of this v'ble, so don't put it in  .h  file  */
+unsigned int olen;          /*  Length of Output Buffer  */
+/* We want to limit exposure of this Imported Function, likewise.  */
+void init_emit( void);

-int init_stream( const char *name)
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *     load_list_name       Name of the Load List File
+ *     load_list_file       (Pointer to) File-Structure for the Load List File
+ *     depncy_list_name     Name of the Dependency List File
+ *     depncy_file     (Pointer to) File-Structure for the Dependency List File
+ *     missing_list_name    Name of the Missing-Files-List File
+ *     missing_list_file    (Pointer to) File-Structure for Missing-List File
+ *     no_files_missing     TRUE if able to load all files
+ *
+ **************************************************************************** */
+
+static char *load_list_name;
+static FILE *load_list_file;
+static char *depncy_list_name;
+static FILE *depncy_file;
+static char *missing_list_name;
+static FILE *missing_list_file;
+static bool no_files_missing = TRUE;
+
+/* **************************************************************************
+ *
+ *         Private data-structure for Include-List support
+ *
+ *     Components are simply a string-pointer and a link pointer
+ *
+ **************************************************************************** */
+
+typedef struct incl_list
 {
-	FILE *infile;
-	unsigned int i;
+        char             *dir_path;
+	struct incl_list *next;
+    } incl_list_t;
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables associated with Include-List support
+ *     include_list_start        Start of the Include-List
+ *     include_list_next         Next entry in Include-List to add or read
+ *     max_dir_path_len          Size of longest entry in the Include-List
+ *     include_list_full_path    Full-Path (i.e., expanded File-name with
+ *                                   Include-List Entry) that was last opened.
+ *
+ **************************************************************************** */
+static incl_list_t *include_list_start = NULL;
+static incl_list_t *include_list_next = NULL;
+static unsigned int max_dir_path_len = 0;
+static char *include_list_full_path = NULL;
+
+
+/* **************************************************************************
+ *
+ *      Function name:  add_to_include_list
+ *      Synopsis:       Add an entry to the Include-List
+ *
+ *      Inputs:
+ *         Parameters:
+ *             dir_compt               Directory Component to add to Inc-List
+ *         Local Static Variables:
+ *             include_list_start      First entry in the Include-List
+ *             include_list_next       Next entry in Include-List to add
+ *             max_dir_path_len        Previous max Dir-Component Length 
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Local Static Variables:
+ *             include_list_start      Assigned a value, first time through
+ *             include_list_next       "Next" field updated with new entry,
+ *                                         then pointer updated to new entry.
+ *             max_dir_path_len        Maximum Length 
+ *         Memory Allocated
+ *             For the list-entry, and for the directory/path name to be added
+ *         When Freed?
+ *             Remains in effect through the life of the program.
+ *
+ *      Process Explanation:
+ *          Unlike most of our linked-lists, this one will be linked forward,
+ *              i.e., in the order elements are added, and will be searched
+ *              in a forward order.
+ *          This means extra code to handle the first entry.
+ *          Allocate and initialize the New Include-List Entry.
+ *          If this is the first entry, point the List-Starter at it.
+ *          Otherwise, the Last-Entry-on-the-List pointer is already valid;
+ *              point its "next" field at the New Entry.
+ *          Point the Last-Entry-on-the-List pointer at the New Entry.
+ *
+ **************************************************************************** */
+
+void add_to_include_list( char *dir_compt)
+{
+    unsigned int new_path_len = strlen( dir_compt);
+    incl_list_t *new_i_l_e = safe_malloc( sizeof( incl_list_t),
+        "adding to include-list" );
+
+    new_i_l_e->dir_path = strdup( dir_compt);
+    new_i_l_e->next = NULL;
+
+    if ( include_list_start == NULL )
+    {
+	include_list_start = new_i_l_e;
+    }else{
+       include_list_next->next = new_i_l_e;
+    }
+    
+    include_list_next = new_i_l_e;
+    if ( new_path_len > max_dir_path_len  ) max_dir_path_len = new_path_len;
+}
+
+#define   DISPLAY_WIDTH  80
+/* **************************************************************************
+ *
+ *      Function name:  display_include_list
+ *      Synopsis:       Display the Include-List, once it's completed,
+ *                          if "verbose" mode is in effect.
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Local Static Variables:
+ *             include_list_start         First entry in the Include-List
+ *             include_list_next          Next entry, as we step through.
+ *         Macro:
+ *             DISPLAY_WIDTH              Width limit of the display
+ *
+ *      Outputs:
+ *         Returned Value:                NONE
+ *         Local Static Variables:
+ *             include_list_next          NULL, when reaches end of Incl-List
+ *         Printout:
+ *             The elements of the Include-List, separated by a space, on
+ *                 a line up to the DISPLAY_WIDTH, or on a line by itself
+ *                 if the element is wider.
+ *
+ *      Process Explanation:
+ *          The calling routine will check for the  verbose  flag.
+ *          Nothing to be done here if Include-List is NULL.
+ *          Print a header, then the list.
+ *
+ **************************************************************************** */
+
+void display_include_list( void)
+{
+    if ( include_list_start != NULL )
+    {
+        int curr_wid = DISPLAY_WIDTH;     /*  Current width; force new line  */
+	printf("\nInclude-List:");
+	include_list_next = include_list_start ;
+	while ( include_list_next != NULL )
+	{
+	    int this_wid = strlen( include_list_next->dir_path) + 1;
+	    char *separator = " ";
+	    if ( curr_wid + this_wid > DISPLAY_WIDTH )
+	    {
+	        separator = "\n\t";
+		curr_wid = 7;  /*  Allow 1 for the theoretical space  */
+	    }
+	    printf("%s%s", separator, include_list_next->dir_path);
+	    curr_wid += this_wid;
+	    include_list_next = include_list_next->next ;
+	}
+	printf("\n");
+    }
+}
+
+
+
+/* **************************************************************************
+ *
+ *          We cannot accommodate the structures of the different
+ *              routines that open files with a single function, so
+ *              we will have to divide the action up into pieces:
+ *              one routine to initialize the include-list search, a
+ *              second to return successive candidates.  The calling
+ *              routine will do the operation (stat or fopen) until
+ *              it succeeds or the list is exhausted.  Then it will
+ *              call a final routine to clean up and display messages.
+ *
+ *          I'm sure I don't need to mention that, when no include-list
+ *              is defined, these functions will still support correct
+ *              operation...
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Function name:  init_incl_list_scan
+ *      Synopsis:       Initialize the search through the Include-List
+ *
+ *      Inputs:
+ *         Parameters:
+ *             base_name               Expanded user-supplied file-name
+ *         Local Static Variables:
+ *             include_list_start      First entry in the Include-List
+ *             max_dir_path_len        Maximum Directory Component Length 
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Local Static Variables:
+ *             include_list_next       Next entry in Include-List to read
+ *             include_list_full_path  Full-Path Buffer pointer
+ *         Memory Allocated
+ *             Full-Path Buffer        (If Include-List was defined)
+ *         When Freed?
+ *             In  finish_incl_list_scan()
+ *
+ *      Process Explanation:
+ *          The  base_name  passed to the routine is expected to have
+ *              any embedded Environment-Variables already expanded.
+ *          The Full-Path Buffer is presumed to be unallocated, and
+ *              its pointer to be NULL.
+ *          The Next-Entry-to-read pointer is also presumed to be NULL.
+ *          If an Include-List has been defined, we will allocate memory
+ *              for the Full-Path Buffer and point the Next-Entry pointer
+ *              at the Start of the List.  If not, no need to do anything.
+ *
+ **************************************************************************** */
+
+static void init_incl_list_scan( char *base_name)
+{
+    if ( include_list_start != NULL )
+    {
+	/*  Allocate memory for the file-name buffer.
+	 *      maximum path-element length plus base-name length
+	 *      plus one for the slash plus one for the ending NULL
+	 */
+	unsigned int new_path_len = max_dir_path_len + strlen( base_name) + 2;
+	include_list_full_path = safe_malloc( new_path_len,
+	    "scanning include-list" );
+	include_list_next = include_list_start;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  scan_incl_list
+ *      Synopsis:       Prepare the next candidate in the include-list search
+ *                          Indicate when the end has been reached.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             base_name               Expanded user-supplied file-name
+ *         Local Static Variables:
+ *             include_list_full_path  Full-Path Buffer pointer
+ *             include_list_next       Next entry in Include-List to read
+ *
+ *      Outputs:
+ *         Returned Value:             TRUE if valid candidate; FALSE when done
+ *         Local Static Variables:
+ *             include_list_full_path  Next Full file-name Path to use
+ *             include_list_next       Updated to next entry in the List
+ *
+ *      Process Explanation:
+ *          Include-List Components are presumed not to require any expansion;
+ *              the Shell is expected to resolve any Environment-Variables
+ *              supplied in command-line arguments before they are passed
+ *              to the (this) application-program.
+ *          We will, therefore, not attempt to expand any Components of
+ *              the Include-List.  
+ *          If the Full-Path Buffer pointer is NULL, it indicates that no
+ *              entries have been made to the Include-List and this is our
+ *              first time through this routine in this search; we will
+ *              use the base-name as supplied, presumably relative to the
+ *              current directory.  Point the buffer-pointer at the base-
+ *              -name and return "Not Done".
+ *          Otherwise, we will look at the Next-Entry pointer:
+ *              If it is NULL, we have come to the end of the Include-List;
+ *                  whether because no Include-List was defined or because
+ *                  we have reached the end, we will return "Done".
+ *              Otherwise, we will load the Full-Path Buffer with the entry
+ *                  currently indicated by the Next-Entry pointer, advance
+ *                  it to the next entry in the List and return "Not Done".
+ *          We will supply a slash as the directory-element separator in
+ *              between the Include-List entry and the  base_name 
+ *
+ *      Extraneous Remarks:
+ *          The slash as directory-element separator works in UNIX-related
+ *              environments, but is not guaranteed in others.  I would
+ *              have preferred to specify that Include-List Components are
+ *              required to end with the appropriate separator, but that
+ *              was neither acceptable nor compatible with existing practice
+ *              in other utilities, and the effort to programmatically
+ *              determine the separator used by the Host O/S was too much
+ *              for such a small return.  And besides, we already have so
+ *              many other UNIX-centric assumptions hard-coded into the
+ *              routines in this file (dollar-sign to signify Environment
+ *              Variables, period for file-name-extension separator, etc.)
+ *              that it's just too much of an uphill battle anymore...
+ *
+ **************************************************************************** */
+
+static bool scan_incl_list( char *base_name)
+{
+    bool retval = FALSE;   /*  default to "Done"  */
    
+    if ( include_list_full_path == NULL )
+    {
+	include_list_full_path = base_name;
+	retval = TRUE;
+    }else{
+	if ( include_list_next != NULL )
+	{
+
+	    /*  Special case:  If the next Directory Component is
+	     *      an empty string, do not prepend a slash; that
+	     *      would either become a root-based absolute path,
+	     *      or, if the base-name is itself an absolute path,
+	     *      it would be a path that begins with two slashes,
+	     *      and *some* Host Operating Systems ***REALLY***
+	     *      DO NOT LIKE that!
+	     */
+	    if ( strlen( include_list_next->dir_path) == 0 )
+	    {
+		sprintf( include_list_full_path, "%s", base_name);
+	    }else{
+		sprintf( include_list_full_path, "%s/%s",
+	            include_list_next->dir_path, base_name);
+	    }
+	    include_list_next = include_list_next->next;
+	    retval = TRUE;
+	}
+    }
+
+    return( retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  finish_incl_list_scan
+ *      Synopsis:       Clean up after a search through the Include-List
+ *                          Display appropriate messages.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             op_succeeded            TRUE if intended operation was ok.
+ *             
+ *         Local Static Variables:
+ *             include_list_start      Non-NULL if Include-List was defined
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Local Static Variables:
+ *             include_list_full_path  Reset to NULL
+ *             include_list_next       Reset to NULL
+ *         Memory Freed
+ *             Full-Path Buffer        (If Include-List was defined)
+ *         Printout:
+ *             If file was found in Include-List, Advisory showing where.
+ *
+ **************************************************************************** */
+
+static void finish_incl_list_scan( bool op_succeeded)
+{
+    if ( include_list_start != NULL )
+    {
+	if ( op_succeeded )
+	{
+	    tokenization_error( INFO,
+		"File was found in %s\n" ,include_list_full_path );
+	}
+	free( include_list_full_path);
+    }
+    include_list_full_path = NULL;
+    include_list_next = NULL;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  open_incl_list_file
+ *      Synopsis:       Look in the Include-List, if one is defined, for
+ *                          the file whose name is given and open it.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             base_name               Expanded user-supplied file-name
+ *             mode                    Mode-string to use; usually "r" or "rb"
+ *         Local Static Variables:
+ *             include_list_full_path  Full Path to use in fopen atttempt
+ *
+ *      Outputs:
+ *         Returned Value:             File structure pointer; NULL if failed
+ *         Local Static Variables:
+ *             include_list_full_path  Full Path used, if succeeded
+ *
+ *      Error Detection:
+ *          Calling routine will detect and report Errors.
+ *
+ *      Process Explanation:
+ *          This routine will initialize and step through Include-List.
+ *              Calling routine will be responsible for "finishing" the
+ *              Include-List search, as well as any Advisory messages.
+ *
+ **************************************************************************** */
+
+static FILE *open_incl_list_file( char *base_name, char *mode)
+{
+    FILE *retval = NULL;
+
+    init_incl_list_scan( base_name);
+    while ( scan_incl_list( base_name) )
+    {
+        retval = fopen( include_list_full_path, mode);
+	if ( retval != NULL )
+	{
+	    break; 
+	}
+    }
+
+    return (retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  stat_incl_list_file
+ *      Synopsis:       Look in the Include-List, if defined, for given file,
+ *                          and collect its statistics.
+ *                      
+ *
+ *      Inputs:
+ *         Parameters:
+ *             base_name               Expanded user-supplied file-name
+ *             file_info               Pointer to STAT structure
+ *         Local Static Variables:
+ *             include_list_full_path  Full Path to use in file-status atttempt
+ *
+ *      Outputs:
+ *         Returned Value:             TRUE if succeeded.
+ *         Local Static Variables:
+ *             include_list_full_path  Full Path used, if succeeded
+ *         Supplied Pointers:
+ *             *file_info              File-statistics structure from STAT call
+ *
+ *      Error Detection:
+ *          Calling routine will detect and report Errors.
+ *          
+ *      Process Explanation:
+ *          This routine will initialize and step through Include-List.
+ *              Calling routine will be responsible for "finishing" the
+ *              Include-List search, as well as any Advisory messages.
+ *
+ **************************************************************************** */
+
+static bool stat_incl_list_file( char *base_name, struct stat *file_info)
+{
+    bool retval = FALSE;
+    int stat_reslt = -1;    /*  Success = 0   */
+
+    init_incl_list_scan( base_name);
+    while ( scan_incl_list( base_name) )
+    {
+        stat_reslt = stat( include_list_full_path, file_info);
+	if ( stat_reslt == 0 )
+	{
+	    retval = TRUE;
+	    break; 
+	}
+    }
+
+    return (retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  init_inbuf
+ *      Synopsis:       Set the given buffer as the current input source
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             inbuf         Pointer to start of new input buffer
+ *             buflen        Length of new input buffer
+ *
+ *      Outputs:
+ *         Returned Value:    NONE
+ *         Global Variables:
+ *             start             Points to given buffer
+ *             end               Points to end of new input buffer
+ *             pc                Re-initialized      
+ *
+ **************************************************************************** */
+
+void init_inbuf(char *inbuf, unsigned int buflen)
+{
+    start = inbuf;
+    pc = start;
+    end = pc + buflen;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  could_not_open
+ *      Synopsis:       Report the "Could not open" Message for various Files.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             severity                  Severity of message, WARNING or ERROR
+ *             fle_nam                   Name of file
+ *             for_what                  Phrase after "... for "
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Printout:
+ *             Message of indicated severity.
+ *
+ *      Error Detection:
+ *          Error already detected; reported here.
+ *
+ **************************************************************************** */
+
+static void could_not_open(int severity, char *fle_nam, char *for_what)
+{
+    tokenization_error( severity, "Could not open file %s for %s.\n",
+	fle_nam, for_what);
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  file_is_missing
+ *      Synopsis:       Add the given File to the Missing-Files-List.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             fle_nam                   Name of file
+ *         Local Static Variables:
+ *             missing_list_file         Missing-Files-List File Structure
+ *                                           May be NULL if none was opened
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Local Static Variables:
+ *             no_files_missing         Set FALSE
+ *         File Output:
+ *             Write File name to Missing-Files-List (if one was opened)
+ *
+ *      Error Detection:
+ *          Error already detected; reported here.
+ *
+ **************************************************************************** */
+
+static void file_is_missing( char *fle_nam)
+{
+    if ( missing_list_file != NULL )
+    {
+	fprintf( missing_list_file, "%s\n", fle_nam);
+	no_files_missing = FALSE;
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  add_to_load_lists
+ *      Synopsis:       Add the given input file-name to the Load-List File,
+ *                      and the Full File path to the Dependency-List File.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             in_name                  Name of given input Source File
+ *         Local Static Variables:
+ *             load_list_file           Load List File Structure pointer
+ *             depncy_file              Dependency-List File Structure ptr
+ *                            Either may be NULL if the file was not opened.
+ *             include_list_full_path   Full Path to where the file was found.
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         File Output:
+ *             Write given file-name to Load-List file (if one was opened)
+ *             Write File Path to Dependency-List file (if one was opened)
+ *
+ *      Process Explanation:
+ *          Write into the Load-List file the input Source Filename in the
+ *              same form -- i.e., unexpanded -- as was supplied by the User.
+ *          Write into the Dependency-List file the full expanded path, as
+ *              supplied by the program to the Host Operating System.
+ *
+ **************************************************************************** */
+
+static void add_to_load_lists( const char *in_name)
+{
+    if ( load_list_file != NULL )
+    {
+	fprintf( load_list_file, "%s\n", in_name);
+    }
+    if ( depncy_file != NULL )
+    {
+	fprintf( depncy_file, "%s\n", include_list_full_path);
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      In the functions that support accessing files whose path-names
+ *          contain embedded Environment-Variables, the commentaries
+ *          will refer to this process, or to inputs that require it,
+ *          using variants of the term "expand".
+ *
+ *      We will also keep some of the relevant information as Local
+ *          Static Variables.
+ *
+ **************************************************************************** */
+
+static char expansion_buffer[ 2*GET_BUF_MAX];
+static bool was_expanded;
+static int expansion_msg_severity = INFO;
+
+/* **************************************************************************
+ *
+ *      Function name:  expanded_name
+ *      Synopsis:       Advisory message to display filename expansion.
+ *
+ *      Inputs:
+ *         Parameters:
+ *         Local Static Variables:
+ *             was_expanded              TRUE if expansion happened
+ *             expansion_buffer          Buffer with result of expansion
+ *             expansion_msg_severity    Whether it's an ordinary Advisory
+ *                                           or we force a message.
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *             
+ *         Printout:
+ *             Advisory message showing expansion, if expansion happened
+ *             Otherwise, nothing.
+ *
+ **************************************************************************** */
+
+static void expanded_name( void )
+{
+    if ( was_expanded )
+    {
+	tokenization_error( expansion_msg_severity,
+	    "File name expanded to:  %s\n", expansion_buffer);
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  expansion_error
+ *      Synopsis:       Supplemental message to display filename expansion.
+ *                      Called after an expanded-filename failure was reported.
+ *
+ *      Inputs:
+ *         Parameters:
+ *         Global Variables:
+ *             verbose                   Set by "-v" switch
+ *
+ *      Outputs:
+ *         Returned Value:               NONE
+ *         Printout:
+ *             Advisory message showing expansion, if applicable
+ *                 and if wasn't already shown.
+ *
+ *      Error Detection:
+ *          Called after Error was reported.
+ *
+ *      Process Explanation:
+ *          Presumptions are that:
+ *              An Error message, showing the user-supplied form of the
+ *                  pathname is also being displayed
+ *              An advisory message showing the pathname expansion may
+ *                  have been displayed during the expansion process,
+ *                  if  verbose  was TRUE.
+ *          The purpose of this routine is to display the expansion if
+ *              it had not already just been displayed, i.e., if  verbose
+ *              is not set to TRUE:  Temporarily force the display of an
+ *              Advisory message.
+ *
+ *      Extraneous Remarks:
+ *          If this routine is called before the Error message is displayed,
+ *              the verbose and non-verbose versions of the Log-File will
+ *              match up nicely...
+ *
+ **************************************************************************** */
+
+static void expansion_error( void )
+{
+    if ( INVERSE( verbose) )
+    {
+	expansion_msg_severity |= FORCE_MSG;
+	expanded_name();
+	expansion_msg_severity ^= FORCE_MSG;
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  expand_pathname
+ *      Synopsis:       Perform the expansion of a path-name that may contain
+ *                          embedded Environment-Variables
+ *
+ *      Inputs:
+ *         Parameters:
+ *             input_pathname           The user-supplied filename
+ *         Global/Static MACRO:
+ *             GET_BUF_MAX              Size of expansion buffer is twice this.
+ *
+ *      Outputs:
+ *         Returned Value:              Pointer to expanded name, or to
+ *                                          input if no expansion needed.
+ *                                      NULL if error.
+ *         Local Static Variables:
+ *             was_expanded             TRUE if expansion needed and succeeded
+ *             expansion_buffer         Result of expansion
+ *         Printout:
+ *             Advisory message showing expansion
+ *             Presumption is that an Advisory giving the user-supplied
+ *                 pathname was already printed.
+ *
+ *      Error Detection:
+ *          Syntax error.  System might print something; it might not be
+ *              captured, even to a log-file.  System failure return might
+ *              be the only program-detectable indication.  Display ERROR
+ *              message and return NULL pointer.  Calling routine will
+ *              display the user-supplied pathname in its Error message
+ *              indicating failure to open the file.
+ *
+ *      Process Explanation:
+ *          Generally speaking, we will let the Shell expand the Environment
+ *              Variables embedded in the user-supplied pathname.
+ *          First, though, we will see if the expansion is necessary:  Look
+ *              for the telltale character, '$', in the input string.  If
+ *              it's not there, there are no Env't-V'bles, and no expansion
+ *              is necessary.  Return the pointer to the input string and
+ *              we're done.  Otherwise.....
+ *          Acquire a temporary file-name.  Construct a string of the form:
+ *                      echo input_string > temp_file_name
+ *              and then issue that string as a command to the Shell.
+ *          If that string generates a system-call failure, report an ERROR.
+ *          Open the temporary file and read its contents.  That will be
+ *              the expansion of the input string.  If its length exceeds
+ *              the capacity of the expansion buffer, it's another ERROR.
+ *          (Of course, don't forget to delete the temporary file.)
+ *          Place the null-byte marker at the end of the expanded name,
+ *              trimming off the terminating new-line.
+ *          Success.  Display the expanded name (as an Advisory message)
+ *              Return the pointer to the expansion buffer and set the flag.
+ *              (Did I mention don't forget to delete the temporary file?)
+ *
+ *      Extraneous Remarks:
+ *          This implementation approach turned out to be the simplest and
+ *              cleanest way to accomplish our purpose.  It also boasts the
+ *              HUGE advantage of not requiring re-invention of a well-used
+ *              (proverbial) wheel.  Plus, any variations allowed by the
+ *              shell (e.g.,: $PWD:h ) are automatically converted, too,
+ *              depending on the System shell (e.g., not for Bourne shell).
+ *          In order to spare you, the maintenance programmer, unnecessary
+ *              agony, I will list a few other approaches I tested, with a
+ *              brief note about the results of each:
+ *          (1)
+ *          I actually tried parsing the input line and passing each component
+ *              V'ble to the getenv() function, accumulating the results into
+ *              a conversion buffer.  I needed to check for every possible
+ *              delimiter, and handle curly-brace enclosures.  The resultant
+ *              code was *UGLY* ... you'd be appalled!  The only good spot was
+ *              that I was able to compensate for an open-curly-brace without
+ *              a corresponding close-curly-brace (if close-c-b wasn't found,
+ *              resume the search for other delimiters...) which, apparently,
+ *              the System does not or will not do.  It was, however, too
+ *              small a compensation for all the awfulness entailed overall.
+ *
+ *          I tried various approaches to using the Environment-Variables to
+ *              convert and retrieve the input string:
+ *          (2)
+ *          Create a command-string that would set an Env't V'ble to the
+ *              input-string, and pass the command-string to the system() call,
+ *              then retrieve the Env't V'ble thus set via getenv().  No dice;
+ *              the system() call operated in a separate sub-shell and could
+ *              not export its Env't upwards.
+ *          (3)
+ *          Use the setenv() command to set an Env't V'ble to the input-string
+ *              and retrieve it via getenv().  The returned string matched the
+ *              input-string without converting it.
+ *          (4)
+ *          Use the setenv() command to set an Env't V'ble to a string like:
+ *                      `echo input_string`
+ *              Again, the string retrieved via getenv() exactly matched the
+ *              unconverted command-string, back-quotes and all.
+ *
+ *          Of course, the equivalents of (2), (3) and (4) worked as desired
+ *              when tested as direct commands to the Shell.  UNIX can be
+ *              funny that way...
+ *
+ *          Oh!  Also:  we will slightly stretch the rules of well-structured
+ *              code.
+ *
+ **************************************************************************** */
+
+static char *expand_pathname( const char *input_pathname)
+{
+    static const int buffer_max = GET_BUF_MAX * 2;
+
+    char *retval = (char *)input_pathname;
+    was_expanded = FALSE;
+
+    /*  If no '$' is found, expansion is unnecessary.  */
+    if ( strchr( input_pathname, '$') != NULL )
+    {
+	FILE *temp_file;
+	int syst_stat;
+	const char *temp_file_name = tmpnam( NULL);
+
+	/*  Use the expansion buffer for our temporary command string  */
+	sprintf( expansion_buffer,
+	    "echo %s>%s\n", input_pathname, temp_file_name);
+	syst_stat = system( expansion_buffer);
+	if ( syst_stat != 0 )
+	{
+	    tokenization_error( TKERROR,
+		"Expansion Syntax.\n");
+	    /*  The "File-Opening" error message will show the input string */
+	    return( NULL);
+	}
+
+	temp_file = fopen( temp_file_name, "r");  /*  Cannot fail.   */
+	syst_stat = fread( expansion_buffer, 1, buffer_max, temp_file);
+	/*  Error test.  Length of what we read is not a good indicator;
+	 *      it's limited anyway by buffer_max.
+	 *  Valid test is if last character read was the new-line.
+	 */
+	if ( expansion_buffer[syst_stat-1] != '\n' )
+	{
+	    tokenization_error( TKERROR,
+		"Expansion buffer overflow.  Max length is %d.\n",
+		    buffer_max);
+	    retval = NULL;
+	}else{
+	    expansion_buffer[syst_stat-1] =0;
+	    was_expanded = TRUE;
+	    retval = expansion_buffer;
+	    expanded_name();
+	}
+
+	fclose( temp_file);
+	remove( temp_file_name);
+    }
+
+    return( retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  open_expanded_file
+ *      Synopsis:       Open a file, expanding Environment-Variables that
+ *                          may be embedded in the given path-name.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             path_name              The user-supplied path-name
+ *             mode                   Mode-string to use; usually "r" or "rb"
+ *             for_what               Phrase to use in Messages
+ *
+ *      Outputs:
+ *         Returned Value:            Pointer to FILE structure; NULL if failed
+ *         Local Static Variables:
+ *             was_expanded          TRUE if expansion happened
+ *             expansion_buffer      Result of expansion
+ *         Printout:
+ *             Advisory message showing expansion
+ *
+ *      Error Detection:
+ *          If expansion or system-call for file-open failed,
+ *              report Error and return NULL.
+ *
+ **************************************************************************** */
+
+FILE *open_expanded_file( const char *path_name, char *mode, char *for_what)
+{
+
+    FILE *retval = NULL;
+
+    char *infile_name = expand_pathname( path_name);
+    if ( infile_name != NULL )
+    {
+        retval = open_incl_list_file( infile_name, mode);
+    }
+
+    if ( retval == NULL )
+    {
+        expansion_error();
+	tokenization_error ( TKERROR,
+	    "Failed to open file %s for %s\n", path_name, for_what );
+    }
+
+    finish_incl_list_scan( BOOLVAL( retval != NULL) );
+
+    return( retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  init_stream
+ *      Synopsis:       Open a file and make it the current source.
+ *                      This is called, not only at the start of tokenization,
+ *                      but also when a subsidiary file is FLOADed.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             name                     Name of the new Input File to open
+ *                                          May be path-name containing
+ *                                          embedded Environment-Variables.
+ *         Global Variables:
+ *             oname                    NULL if opening Primary Input File
+ *         Local Static Variables:
+ *             include_list_full_path   Full Path to where the file was found
+ *
+ *      Outputs:
+ *         Returned Value:    TRUE = opened and read file successfully
+ *         Global Variables     (Only changed if successful):
+ *             iname                    Set to new Input File name
+ *             lineno                   Re-initialized to 1
+ *         Local Static Variables:
+ *             no_files_missing         Set FALSE if couldn't read input file
+ *             include_list_full_path   Retains full-path if file opened was
+ *                      the Primary Input File (as contrasted with an FLoaded
+ *                      Source file), in which case a call to  init_output()
+ *                      is expected; the Full-Path Buffer will be freed there.
+ *         Memory Allocated
+ *             Duplicate of Input File name (becomes  iname  )
+ *             A fresh input buffer; input file is copied to it.
+ *                 Becomes  start  by action of call to  init_inbuf().
+ *         When Freed?
+ *             By  close_stream()
+ *         File Output:
+ *             Write new Input File name to Load-List file.
+ *             Writing to Missing-Files-List File if failure,
+ *                 or Full File path to Dependency-List File,
+ *                 is handled by called routine.
+ *         Other Exotic Effects:
+ *             Force a flush of  stdout  before printing ERROR messages
+ *             
+ *      Error Detection:
+ *          Failure to open or read Input file:  ERROR; suppress output;
+ *              write Input File name to Missing-Files-List File.
+ *
+ *      Process Explanation:
+ *          Free local buffer on failure.
+ *          Caller should only invoke  close_stream()  if this call succeeded.
+ *          Some filesystems use zeros for new-line; we need to convert
+ *              those zeros to line-feeds.
+ *          Similarly for files that have carr-ret/line-feed; the carr-ret
+ *              will cause havoc; replace it w/ a space.
+ *      
+ *      Revision History:
+ *      Updated Thu, 07 Apr 2005 by David L. Paktor
+ *          Restructured.  If opened file, close it, even if can't read it.
+ *          Return TRUE on success.
+ *          Caller examines return value.    
+ *      Updated Wed, 13 Jul 2005 by David L. Paktor
+ *          Replace carr-rets with spaces.   
+ *      Updated Sun, 27 Nov 2005 by David L. Paktor
+ *          Write new Input File name to Load-List file. 
+ *      Updated Tue, 31 Jan 2006 by David L. Paktor
+ *          Add support for embedded Environment-Variables in path name
+ *      Updated Thu, 16 Feb 2006 David L. Paktor
+ *          Collect missing (inaccessible) filenames
+ *      Updated Fri, 17 Mar 2006 David L. O'Paktor
+ *          Add support for Include-List search
+ *
+ *      Still to be done:
+ *          Set a flag when carr-ret has been replaced by space;
+ *              when a string crosses a line, if this flag is set,
+ *              issue a warning that an extra space has been inserted.
+ *
+ **************************************************************************** */
+
+bool init_stream( const char *name)
+{
+    FILE *infile;
+    u8 *newbuf;
    struct stat finfo;
+    bool stat_succ = FALSE;
+    bool tried_stat = FALSE;
+    bool retval = FALSE;
+    bool inp_fil_acc_err = FALSE;
+    bool inp_fil_open_err = FALSE;
+    bool inp_fil_read_err = FALSE;
+
+    char *infile_name = expand_pathname( name);
+
+    if ( (infile_name != NULL) )
+    {
+	tried_stat = TRUE;
+	stat_succ = stat_incl_list_file( infile_name, &finfo);
+    }
+
+    if ( INVERSE( stat_succ) )
+    {
+	inp_fil_acc_err = TRUE;
+    }else{
    
-	if (stat(name,&finfo))
-		return -1;
+	infile = fopen( include_list_full_path, "r");
+	if ( infile == NULL )
+	{
+	    inp_fil_open_err = TRUE;
+	}else{
    
    ilen=finfo.st_size;
-	start=malloc(ilen+1);
-	if (!start)
-		return -1;
+	    newbuf = safe_malloc(ilen+1, "initting stream");

-        infile=fopen(name,"r");
-        if (!infile)
-                return -1;
+	    if ( fread( newbuf, ilen, 1, infile) != 1 )
+	    {
+		inp_fil_read_err = TRUE;
+		free( newbuf );
+	    } else {
+		unsigned int i;

-	if (fread(start, ilen, 1, infile)!=1) {
-		free(start);
-		return -1;
+		retval = TRUE ;
+		/*  Replace zeroes in the file with LineFeeds. */
+		/*  Replace carr-rets with spaces.  */
+		for (i=0; i<ilen; i++)
+		{
+		    char test_c = newbuf[i];
+		    if ( test_c == 0    ) newbuf[i] = 0x0a;
+		    if ( test_c == 0x0d ) newbuf[i] = ' ';
    }
+		newbuf[ilen]=0;

+		init_inbuf(newbuf, ilen);
+
+		/*   If the -l option was specified, write the name of the
+		 *       new input-file to the Load-List file...  UNLESS
+		 *       this is the first time through and we haven't yet
+		 *       opened the Load-List file, in which case we'll
+		 *       just open it here and wait until we create the
+		 *       output-file name (since the Load-List file name
+		 *       depends on the output-file name anyway) before
+		 *       we write the initial input-file name to it.
+		 */
+		/*   Looking for the option-flag _and_ for a non-NULL value
+		 *       of the file-structure pointer is redundandundant:
+		 *       The non-NULL pointer is sufficient, once the List
+		 *       File has been created...
+		 */
+		/*   Same thing applies if the -P option was specified,
+		 *       for the Dependency-List file, except there we'll
+		 *       write the full path to where the file was found.
+		 */
+		/*   We have a routine to do both of those.   */
+		add_to_load_lists( name);
+		/*
+		 *   And... there's one slight complication:  If this is
+		 *       the first time through, (i.e., we're opening the
+		 *       Primary Input File) then we haven't yet opened the
+		 *       Dependency-List file, and we need to preserve the
+		 *       Full file-name Buffer until the call to  init_output()
+		 *       where the include-list scan will be "finish"ed.
+		 *   Actually, we want to postpone "finish"ing the inc-l scan
+		 *       for several reasons beyond the Dependency-List file, 
+		 *       such as completing the File Name Announcement first.
+		 *   A NULL output-name buffer is our indicator.
+		 */
+		if ( oname == NULL )
+		{
+		    /*  Quick way to suppress "finish"ing the i-l scan */
+		    tried_stat = FALSE; 
+		}
+	    }
    fclose(infile);
+	}
+    }
    
-	/* no zeroes within the file. */
-	for (i=0; i<ilen; i++) {
-		/* start[i]=start[i]?start[i]:0x0a; */
-		start[i] |= (((signed char)start[i] - 1) >> 7) & 0x0a;
+    FFLUSH_STDOUT	/*   Do this first  */
+    /*  Now we can deliver our postponed error and advisory messages  */
+    if ( INVERSE( retval) )
+    {
+	file_is_missing( (char *)name);
+	if ( inp_fil_acc_err )
+	{
+	    expansion_error();
+	    tokenization_error( TKERROR, 
+		"Could not access input file %s\n", name);
+	}else{
+	    if ( inp_fil_open_err )
+	    {
+		expansion_error();
+		could_not_open( TKERROR, (char *)name, "input");
+	    }else{
+		if ( inp_fil_read_err )
+		{
+		    expansion_error();
+		    tokenization_error( TKERROR, 
+			"Could not read input file %s\n", name);
+		}
+	    }
    }
-	start[ilen]=0;
+	}
    
-	pc=start; 
-	end=pc+ilen;
+    if ( tried_stat )
+    {
+        finish_incl_list_scan( stat_succ);
+    }

+    /*  Don't change the input file name and line-number until after
+     *      the Advisory showing where the file was found.
+     */
+    if ( retval )
+    {
    iname=strdup(name);
-	
    lineno=1;
+    }
    
-	return 0;
+    return ( retval );
    
 }

-int init_output( const char *in_name, const char *out_name )
+/* **************************************************************************
+ *
+ *      Function name:  extend_filename 
+ *      Synopsis:       Change the filename to the given extension
+ *
+ *      Inputs:
+ *         Parameters:
+ *             base_name                  Name of the Input Base File
+ *             new_ext                    New ext'n (with leading period)
+ *
+ *      Outputs:
+ *         Returned Value:                Result filename
+ *         Memory Allocated
+ *             Buffer for result filename
+ *         When Freed?
+ *             At end of Tokenization, by  close_output().
+ *
+ *      Process Explanation:
+ *          If the Input Base File Name has an extension, it will be replaced
+ *              with the given new extension.  If not, the new extension will
+ *              simply be appended.
+ *          If the Input Base File Name has an extension that matches the
+ *              new extension, a duplicate of the extension will be appended.
+ *
+ *      Extraneous Remarks:
+ *          I only recently added protection against the situation where the
+ *              Input Base File Name has no extension, but the Directory Path
+ *              leading to it has a period in one of the directory names.
+ *              Granted, this is a rare case, but not altogether impossible;
+ *              I would have done it earlier except for the fact that the
+ *              separator between directories may vary with different Host
+ *              Operating Systems.
+ *          However, at this point we have UNIX-centric assumptions hard-
+ *              -coded in to so many other places that we might as well
+ *              go with the slash here too.
+ *
+ **************************************************************************** */
+
+static char *extend_filename( const char *base_name, const char *new_ext)
 {
-	const char *ext;
+    char *retval;
+    char *ext;
    unsigned int len; /* should this be size_t? */
+    const char *root;

+    root = strrchr(base_name, '/');
+    if ( root == NULL )  root = base_name;
+
+    ext = strrchr(root, '.');
+    if ( ext != NULL )
+    {
+        if ( strcasecmp(ext, new_ext) == 0 )
+	{
+	    ext = NULL;
+	}
+    }
+
+    len = ext ? (ext - base_name) : (unsigned int)strlen(base_name) ;
+    retval = safe_malloc(len+strlen(new_ext)+1, "extending file-name");
+    memcpy( retval, base_name, len);
+    retval[len] = 0;
+    strcat(retval, new_ext);
+
+    return( retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  init_output
+ *      Synopsis:       After the Input Source File has been opened, assign 
+ *                          the name for the Binary Output File; initialize
+ *                          the FCode Output Buffer; assign names for the
+ *                          FLoad List, Dependency-List, and Missing-Files
+ *                          List files; open them and write their first
+ *                          entries to them.
+ *                     Announce the Input and various output file names.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             in_name                  Name of the Input Source File
+ *             out_name                 Name of the Binary Output File, if
+ *                                          specified on the Command Line,
+ *                                          or NULL if not.
+ *         Global Variables:
+ *             fload_list               Whether to create an FLoad-List file
+ *             dependency_list          Whether to create a Dependency-List file
+ *         Local Static Variables:
+ *             include_list_full_path   Full Path to the Input Source File;
+ *                                          should still be valid from opening
+ *                                          of Primary Source Input file, for
+ *                                          first entry to Dependency-List file.
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Global Variables:
+ *             oname                    Binary Output File Name
+ *             ostart                   Start of FCode Output Buffer
+ *             opc                      FCode Output Buffer Position Counter
+ *             abs_token_no             Initialized to 1
+ *         Local Static Variables:
+ *             load_list_name           Name of the Load List File
+ *             load_list_file           FLoad List File Structure pointer
+ *             depncy_list_name         Name of the Dependency List File ptr
+ *             depncy_file              Dependency List File Structure
+ *             missing_list_name        Name of the Missing-Files-List File
+ *             missing_list_file        Missing-Files-List File Structure
+ *             no_files_missing         Initialized to TRUE
+ *         Memory Allocated
+ *             Binary Output File Name Buffer
+ *             FCode Output Buffer
+ *             FLoad List File Name Buffer
+ *             Dependency List File Name Buffer
+ *             
+ *         When Freed?
+ *             In  close_output()
+ *         File Output:
+ *             FLoad List or Dependency List files are opened (if specified).
+ *                 Primary Source Input file name and path, respectively,
+ *                 are written as the first entry to each.
+ *         Printout:
+ *             (Announcement of input file name has already been made)
+ *             Announce binary output, fload- and dependency- -list file names
+ *
+ *      Error Detection:
+ *          Failure to open FLoad List or Dependency List file:  ERROR;
+ *              suppress binary output.  Further attempts to write to
+ *              FLoad List or Dependency List files are prevented by
+ *              the respective FILE_structure pointers being NULL.
+ *          Failure to open Missing-Files-List file:  WARNING
+ *
+ *      Process Explanation:
+ *          If no Output File Name was specified on the Command Line, the
+ *              name of the Binary (FCode) Output File will be derived
+ *              from the name of the Input File by replacing its extension
+ *              with  .fc , or, if  the Input File had no extension, by
+ *              merely appending the extension  .fc 
+ *          In the odd case where the Input File name has an extension
+ *              of  .fc, we will merely append another  .fc  extension.
+ *          If  fload_list  is TRUE (i.e., the "-l" option was specified on
+ *              the command-line, the FLoad List File name will be derived
+ *              from the name of the Output File by the same rules, only with
+ *              an extension of  .fl   Open the FLoad List File.  Write the
+ *              name of the initial input file to the FLoad List File.
+ *          Similarly if the "-P" command-line option was specified, the name
+ *              of the Dependency List File will be derived with an extension
+ *              of  .P  Open it and write the Full Path for the initial input
+ *              file to it.  NOTE:  To do that, we need to have preserved the
+ *              Full Path-name Buffer from the call to  init_stream()   We
+ *              will "finish" it here, after we've used it.
+ *          The Missing-Files-List File will be created if either option was
+ *              specified.  Its name will be derived similarly, with an
+ *              extension of  .fl.missing
+ *
+ **************************************************************************** */
+
+void init_output( const char *in_name, const char *out_name )
+{
    /* preparing output */

-	if( out_name )
-		oname = (u8 *)strdup( out_name );
-	else {
-		ext=strrchr(in_name, '.');
-		len=ext ? (unsigned int)(ext-in_name) : (unsigned int)strlen(in_name) ;
-		oname=malloc(len+4);
-		memcpy(oname, in_name, len);
-		oname[len] = 0;
-		strcat((char *)oname, ".fc");
+	if( out_name != NULL )
+	{
+		oname = strdup( out_name );
+	}else{
+		oname = extend_filename( in_name, ".fc"); 
    }
    
    /* output buffer size. this is 128k per default now, but we
     * could reallocate if we run out. KISS for now.
     */
-	ostart=malloc(OUTPUT_SIZE);
-	if (!ostart) {
-		free(oname);
-		free(start);
-		return -1;
+	olen = OUTPUT_SIZE;
+	ostart=safe_malloc(olen, "initting output buffer");
+
+	opc = 0;
+	init_emit();  /* Init'l'zns needed by our companion file, emit.c  */
+
+	printf("Binary output to %s ", oname);
+	if ( fload_list )
+	{
+	    load_list_name = extend_filename( oname, ".fl");
+	    load_list_file = fopen( load_list_name,"w");
+	    printf("  FLoad-list to %s ", load_list_name);
    }
+	if ( dependency_list )
+	{
+	    depncy_list_name = extend_filename( oname, ".P");
+	    depncy_file = fopen( depncy_list_name,"w");
+	    printf("  Dependency-list to %s ", depncy_list_name);
+	}
+	printf("\n");

-	opc=oend=ostart;
-	oend+=OUTPUT_SIZE;
+	add_to_load_lists( in_name);
    
-	/* We're beginning a new output file, so we have to
-	 * start our own fcode numbers at 0x800 again. 
-	 */
-	nextfcode=0x800;
+	/*  Let's avoid collisions between stdout and stderr  */
+	FFLUSH_STDOUT
+
+	/*  Now we can deliver our advisory and error messages  */
    
-	return 0;
+	{
+	    /* Suspend showing filename in advisory and error messages. */
+	    char *temp_iname = iname;
+	    iname = NULL; 
+	
+	    finish_incl_list_scan( TRUE);
+
+	    if ( fload_list && (load_list_file == NULL) )
+	    {
+	    	could_not_open( TKERROR, load_list_name, "Load-List");
+		free( load_list_name);
+	    }
+	    if ( dependency_list && (depncy_file == NULL) )
+	    {
+	    	could_not_open( TKERROR, depncy_list_name,
+		    "Dependency-List");
+		free( depncy_list_name);
 }

-int close_stream(void)
+	    if ( fload_list || dependency_list )
+	    {
+		missing_list_name = extend_filename( oname, ".fl.missing");
+		missing_list_file = fopen( missing_list_name,"w");
+		no_files_missing = TRUE;
+
+		if ( missing_list_file == NULL )
+		{
+		    could_not_open( WARNING, missing_list_name,
+			"Missing-Files List" );
+	            free( missing_list_name);
+		}
+	    }
+	    iname = temp_iname;
+	}
+	abs_token_no = 1;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  increase_output_buffer
+ *      Synopsis:       Reallocate the Output Buffer to double its prior size 
+ *
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Global Variables:
+ *             ostart                   Start-address of current Output Buffer
+ *             olen                     Current size of the Output Buffer
+ *         Local Static Variables:
+ *
+ *      Outputs:
+ *         Returned Value:                 NONE
+ *         Local Static Variables:
+ *             olen                     Doubled from value at input
+ *             ostart                   Start-address of new Output Buffer
+ *         Memory Allocated
+ *             A new FCode Output Buffer, using the  realloc()  facility.
+ *         When Freed?
+ *             In  close_output()
+ *         Memory Freed
+ *             Former FCode Output Buffer, also by means of  realloc() 
+ *
+ *         Printout:
+ *             Advisory message that this is taking place.
+ *
+ *      Error Detection:
+ *          FATAL if  realloc()  fails.
+ *          FATAL if output buffer has been expanded to a size beyond
+ *              what an INT can express.  Unlikely? maybe; impossible? no...
+ *
+ *      Process Explanation:
+ *          Because we are allowing the Output Buffer to be relocated, we
+ *              must take care to limit the exposure to external routines
+ *              of its actual address.  All references to locations within
+ *              the Output Buffer should be made in terms of an _offset_.
+ *
+ *      Extraneous Remarks:
+ *          Unfortunately, it is not feasible to completely isolate the
+ *              actual address of the Output Buffer, but we will keep the
+ *              exposure limited to the routines in  emit.c
+ *          Similarly, it wasn't feasible to keep this routine isolated,
+ *              nor the variable  olen  , but we will limit their exposure.
+ *
+ **************************************************************************** */
+void increase_output_buffer( void );  /*  Keep the prototype local  */
+void increase_output_buffer( void )
 {
+    u8 *newout;
+
+    if ( olen == 0 )
+    {
+	tokenization_error( FATAL,
+		"Output Buffer reallocation overflow.");
+    }else{
+	unsigned int rea_len;
+	olen = olen * 2;
+	rea_len = olen;
+	if ( rea_len == 0 )   rea_len = (unsigned int)-1;
+	tokenization_error( INFO,
+	    "Output Buffer overflow.  "
+		"Relocating and increasing to %d bytes.\n", rea_len);
+
+	newout = realloc(ostart, rea_len);
+	if ( newout == NULL)
+	{
+	    tokenization_error( FATAL,
+		"Could not reallocate %d bytes for Output Buffer", rea_len);
+	}
+
+	ostart = newout;
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  close_stream
+ *      Synopsis:       Free-up the memory used for the current input file
+ *                          whenever it is closed.  Reset pointers and
+ *                          line-counter.  Close files as necessary.
+ *
+ *      The dummy parameter is there to accommodate Macro-recursion protection. 
+ *          It's a long story; don't get me started...
+ *
+ **************************************************************************** */
+
+void close_stream( _PTR dummy)
+{
    free(start);
    free(iname);
-	return 0;	
+	start = NULL;
+	iname = NULL;
+	lineno = 0;
 }

-int close_output(void)
+/* **************************************************************************
+ *
+ *      Function name:  close_output
+ *      Synopsis:       Write the Binary Output file, if appropriate.
+ *                          Return a "Failure" flag.
+ *
+ **************************************************************************** */
+
+
+bool close_output(void)
 {
+    bool retval = TRUE;  /*  "Failure"  */
+    if ( error_summary() )
+    { 
+	if ( opc == 0 )
+{
+	    retval = FALSE;  /*  "Not a problem"  */
+	}else{
    FILE *outfile;
-	unsigned int len;

-	len=(unsigned long)opc-(unsigned long)ostart;
+	    outfile=fopen( oname,"w");
+	    if (!outfile)
+	    {
+		/*  Don't do this as a  tokenization_error( TKERROR
+		 *      because those are all counted, among other reasons...
+		 */ 
+		printf( "Could not open file %s for output.\n", oname);
+	    }else{
    
-	outfile=fopen((char *)oname,"w");
-	if (!outfile) {
-		printf("toke: error opening output file.\n");
-		return -1;
+		if ( fwrite(ostart, opc, 1, outfile) != 1 )
+		{
+		    tokenization_error( FATAL, "While writing output.");
    }
    
-	if(fwrite(ostart, len, 1, outfile)!=1)
-		printf ("toke: error while writing output.\n");
-
    fclose(outfile);

-	printf("toke: wrote %d bytes to bytecode file '%s'\n", len, oname);
+		printf("toke: wrote %d bytes to bytecode file '%s'\n",
+		    opc, oname);
+		retval = FALSE;  /*  "No problem"  */
+	    }
+	}
+    }
    
    free(oname);
-	return 0;
+    free(ostart);
+    oname = NULL;
+    ostart = NULL;
+    opc = 0;
+    olen = OUTPUT_SIZE;
+
+    if ( load_list_file != NULL )
+    {
+	fclose(load_list_file);
+	free(load_list_name);
+    }
+    if ( depncy_file != NULL )
+    {
+	fclose(depncy_file);
+	free(depncy_list_name);
+    }
+    if ( missing_list_file != NULL )
+    {
+	fclose( missing_list_file);
+	if ( no_files_missing )
+	{
+	    remove( missing_list_name);
+	}
+	free( missing_list_name);
+    }
+
+    load_list_file = NULL;
+    load_list_name = NULL;
+    missing_list_file = NULL;
+    missing_list_name = NULL;
+    depncy_file = NULL;
+    depncy_list_name = NULL;
+
+    return ( retval );
 }

Modified: fcode-utils/toke/stream.h
===================================================================
--- fcode-utils/toke/stream.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/stream.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _H_STREAM
+#define _H_STREAM
+
 /*
  *                     OpenBIOS - free your system! 
  *                         ( FCode tokenizer )
@@ -24,15 +27,68 @@
  *
  */

-#ifndef _H_STREAM
-#define _H_STREAM
+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */

-int init_stream( const char *name );
-int close_stream(void);
-int init_output( const char *inname, const char *outname );
-int close_output(void);
+#include "types.h"

-extern unsigned int 	lineno;
+/* **************************************************************************
+ *
+ *          Exported Global Variables
+ *
+ **************************************************************************** */
+
+/* input pointers */
+extern u8 *start;
+extern u8 *pc;
+extern u8 *end;
 extern char		*iname;
+extern unsigned int lineno;         /* Line Number within current input file  */
+extern unsigned int abs_token_no;   /* Absolute Token Number in Source Input  */

+/* output pointers */
+extern char *oname;         /* output file name  */
+
+extern unsigned int opc;    /* output buffer position counter  */
+
+/* **************************************************************************
+ *
+ *    Note that the variables  ostart  and  olen , as well as the routine
+ *         increase_output_buffer  are not listed here.
+ *
+ *    We would have preferred to isolate them completely, but we would have
+ *        to disrupt the organization of  emit.c  (which we'd rather not);
+ *        in order to avoid exporting them any more widely than necessary,
+ *        we will declare them  extern  only in the file where they are
+ *        unavoidably needed.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *          Macro Name:    OUTPUT_SIZE
+ *                        Initial size of the Output Buffer
+ *
+ **************************************************************************** */
+
+#define OUTPUT_SIZE	131072
+
+
+/* **************************************************************************
+ *
+ *          Exported Functions
+ *
+ **************************************************************************** */
+
+void add_to_include_list( char *dir_compt);
+void display_include_list( void);
+FILE *open_expanded_file( const char *path_name, char *mode, char *for_what);
+bool init_stream( const char *name );
+void close_stream( _PTR dummy);
+void init_output( const char *inname, const char *outname );
+bool close_output(void);
+void init_inbuf(char *inbuf, unsigned int buflen);
+
 #endif   /* _H_STREAM */

Added: fcode-utils/toke/strsubvocab.c
===================================================================
--- fcode-utils/toke/strsubvocab.c	                        (rev 0)
+++ fcode-utils/toke/strsubvocab.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,337 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      General-purpose support functions for
+ *          String-Substitution-type vocabularies
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      A String-Substitution vocabulary, as the name implies, is one in
+ *          in which each an entry consists of two strings;  one that is
+ *          sought, and one that is returned as a substitute.  Macros and
+ *          aliases are implemented this way, as are also user-supplied
+ *          command-line symbol definitions.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          init_str_sub_vocab     Initialize a String-Substitution vocab
+ *          add_str_sub_entry      Add an entry to a Str-Subst vocab
+ *          lookup_str_sub         Look for a name in a String-Substitution
+ *                                     vocab, return the substitution string.
+ *          exists_in_str_sub      Confirm whether a given name exists in a
+ *                                     String-Substitution vocabulary
+ *          create_str_sub_alias   Duplicate the behavior of one name with
+ *                                     another name.  Return a "success" flag.
+ *          reset_str_sub_vocab    Reset a given Str-Subst vocab to its initial
+ *                                    "Built-In" position.
+ *
+ *
+ **************************************************************************** */
+
+#include <stdio.h>
+#include <stdlib.h>
+#if defined(__linux__) && ! defined(__USE_BSD)
+#define __USE_BSD
+#endif
+#include <string.h>
+
+#include "errhandler.h"
+#include "strsubvocab.h"
+
+
+/* **************************************************************************
+ *
+ *      Function name:  init_str_sub_vocab
+ *      Synopsis:       Dynamically initialize the link-pointers
+ *                          of the.given String-Substitution vocabulary
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             str_sub_vocab_tbl   Pointer to the initial Str-Subst vocab array
+ *             max_indx            Maximum Index of the initial array.
+ *
+ *      Outputs:
+ *         Returned Value:          None
+ *         Global Variables:
+ *              The link-fields of the initial Str-Subs vocab array entries
+ *                  will be filled in.
+ *
+ **************************************************************************** */
+
+void init_str_sub_vocab( str_sub_vocab_t *str_sub_vocab_tbl, int max_indx)
+{
+    int indx;
+    for ( indx = 1 ; indx < max_indx ; indx++ )
+    {
+        str_sub_vocab_tbl[indx].next = &str_sub_vocab_tbl[indx-1];
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  add_str_sub_entry
+ *      Synopsis:       Add an entry to the given Str-Subst vocab
+ *      
+ *      Inputs:
+ *         Parameters:         Pointer to:
+ *             ename               space containing the name of the entry
+ *             subst_str           space containing the substitution string
+ *             *str_sub_vocab      the "tail" of the Str-Subst vocab-list 
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Supplied Pointers:
+ *             *str_sub_vocab       Will point to new entry
+ *         Memory Allocated:
+ *             Memory for the new entry will be allocated.
+ *         When Freed?
+ *             When reset_str_sub_vocab() is applied to the same vocab-list.
+ *                 In some instances, the new entry will be freed upon end
+ *                 of tokenization; in others, only on termination of program.
+ *
+ *      Error Detection:
+ *          Failure to allocate memory is a Fatal Error.
+ *
+ *      Process Explanation:
+ *          The name and substitution-string pointers are presumed to already
+ *              point to stable memory-spaces.  Memory will be allocated
+ *              for the entry itself; its pointers will be entered and the
+ *              given pointer-to-the-tail-of-the-vocabulary will be updated.
+ *
+ *      Extraneous Remarks:
+ *          This might have been where we would have checked for re-aliasing,
+ *              but the introduction of the approach to aliasing embodied in
+ *              the various  create_..._alias()  routines neatly bypasses it.
+ *
+ **************************************************************************** */
+
+void add_str_sub_entry( char *ename,
+                            char *subst_str,
+			        str_sub_vocab_t **str_sub_vocab )
+{
+    str_sub_vocab_t *new_entry;
+
+    new_entry = safe_malloc(sizeof(str_sub_vocab_t), "adding str_sub_entry");
+    new_entry->name   =  ename;
+    new_entry->alias  =  subst_str;
+    new_entry->next   = *str_sub_vocab;
+
+    *str_sub_vocab = new_entry;
+
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_str_sub
+ *      Synopsis:       Look for a name in the given Str-Subst vocabulary
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             tname                The "target" name for which to look
+ *             str_sub_vocab        The Str-Subst vocab-list
+ *
+ *      Outputs:
+ *         Returned Value:          Pointer to the substitution string, or
+ *                                      NULL pointer if name not found.
+ *                                  May be NULL if subst'n string is NULL.
+ *
+ **************************************************************************** */
+
+char *lookup_str_sub( char *tname, str_sub_vocab_t *str_sub_vocab )
+{
+    str_sub_vocab_t *curr;
+    char *retval = NULL;
+
+    for (curr = str_sub_vocab ; curr != NULL ; curr=curr->next)
+    {
+        if ( strcasecmp(tname, curr->name) == 0 )
+	{
+	    retval = curr->alias;
+	    break;
+	}
+    }
+    return ( retval ) ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_in_str_sub
+ *      Synopsis:       Confirm whether a given name exists in a given
+ *                          String-Substitution vocabulary
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             tname                The "target" name for which to look
+ *             str_sub_vocab        Pointer to the Str-Subst vocab-list
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE if the name is found
+ *
+ *      Extraneous Remarks:
+ *          Because the Returned Value of lookup_str_sub() may be NULL for 
+ *              other reasons than that the name was not found, we cannot
+ *              rely on that routine, and must replicate the outer-shell
+ *              of its structure.
+ *
+ **************************************************************************** */
+
+bool exists_in_str_sub( char *tname, str_sub_vocab_t *str_sub_vocab )
+{
+    str_sub_vocab_t *curr;
+    bool retval = FALSE;
+
+    for (curr = str_sub_vocab ; curr != NULL ; curr=curr->next)
+    {
+        if ( strcasecmp(tname, curr->name) == 0 )
+	{
+	    retval = TRUE;
+	    break;
+	}
+    }
+    return ( retval );
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  create_str_sub_alias
+ *      Synopsis:       Create an Alias in a String-Substitution vocabulary
+ *                          Return a "success" flag.
+ *
+ *      Associated FORTH word:                 ALIAS
+ *
+ *      Inputs:
+ *         Parameters:
+ *             old_name             Name of existing entry
+ *             new_name             New name for which to create an entry
+ *             *str_sub_vocab       Pointer to "tail" of Str-Subst vocab-list
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if  old_name  found in str_sub_vocab
+ *         Supplied Pointers:
+ *             *str_sub_vocab       Will be updated to point to the new entry
+ *         Memory Allocated:
+ *             A copy of the "old" name's substitution string.
+ *         When Freed?
+ *             When reset_str_sub_vocab() is applied to the same vocab-list.
+ *                 In some instances, the new entry will be freed when the
+ *                 device-node in which it was created is "finish"ed; in
+ *                 others, only on termination of the program.
+ *
+ *      Process Explanation:
+ *          The "new" name is presumed to point to a stable memory-space.
+ *          If the given "old" name exists in the given Str-Subst vocab-list,
+ *              duplicate the substitution string into newly-allocated memory
+ *              and pass the duplicated string and the "new" name along to
+ *              the  add_str_sub_entry()  routine and return TRUE.
+ *          If the given "old" name does not exist in the given vocab-list,
+ *              return FALSE.
+ *
+ *      Extraneous Remarks:
+ *          This neatly bypasses the question of re-aliasing...  ;-)
+ *
+ *          We can rely on testing for a returned NULL from lookup_str_sub()
+ *              because we won't be applying this routine to any vocabulary
+ *              that permits a NULL in its substitution string.
+ *
+ **************************************************************************** */
+
+bool create_str_sub_alias(char *new_name,
+                              char *old_name,
+			          str_sub_vocab_t **str_sub_vocab )
+{
+    bool retval = FALSE;
+    char *old_subst_str = lookup_str_sub( old_name, *str_sub_vocab );
+    if ( old_subst_str != NULL )
+    {
+        char *new_subst_str = strdup(old_subst_str );
+	add_str_sub_entry(new_name, new_subst_str, str_sub_vocab );
+	retval = TRUE ;
+    }
+
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_str_sub_vocab
+ *      Synopsis:       Reset a given Str-Subst vocab to its initial
+ *                         "Built-In" position.
+ *      
+ *      Inputs:
+ *         Parameters:
+ *            *str_sub_vocab         Pointer to the Str-Subst vocab-list 
+ *             reset_position        Position to which to reset the list
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Supplied Pointers:
+ *             *str_sub_vocab       Reset to given "Built-In" position.
+ *         Memory Freed
+ *             All memory allocated by user-definitions will be freed
+ *
+ *      Process Explanation:
+ *          The "stable memory-spaces" to which the name and substitution
+ *              string pointers point are presumed to have been acquired
+ *              by allocation of memory, which is reasonable for entries
+ *              created by the user as opposed to the built-in entries,
+ *              which we are, in any case, not releasing.
+ *          The substitution-string pointer may be null; watch out when
+ *              we free() it; not all C implementations forgive that.
+ *
+ **************************************************************************** */
+
+void reset_str_sub_vocab(
+            str_sub_vocab_t **str_sub_vocab ,
+	             str_sub_vocab_t *reset_position )
+{
+    str_sub_vocab_t *next_t;
+
+    next_t = *str_sub_vocab;
+    while ( next_t != reset_position  )
+    {
+	next_t = (*str_sub_vocab)->next ;
+
+	free( (*str_sub_vocab)->name );
+	if ( !(*str_sub_vocab)->alias )
+	{
+	    free( (*str_sub_vocab)->alias );
+	}
+	free( *str_sub_vocab );
+	*str_sub_vocab = next_t ;
+    }
+}
+

Added: fcode-utils/toke/strsubvocab.h
===================================================================
--- fcode-utils/toke/strsubvocab.h	                        (rev 0)
+++ fcode-utils/toke/strsubvocab.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,85 @@
+#ifndef _TOKE_STRSUBVOCAB_H
+#define _TOKE_STRSUBVOCAB_H
+
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Headers, general-purpose support structures, function prototypes
+ *          and macros for String-Substitution-type vocabularies.
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Structures:
+ *          str_sub_vocab_t        Entry in a String-Substitution-type vocab 
+ *
+ *      Macros:
+ *          BUILTIN_STR_SUB        Add an entry to the initial Str-Sub vocab.
+ *
+ **************************************************************************** */
+
+#include "types.h"
+
+
+typedef struct str_sub_vocab {
+	u8  *name;
+	u8  *alias;
+	struct str_sub_vocab *next;
+} str_sub_vocab_t;
+
+#if  0     /*  On the way out   */
+/*
+ *          BUILTIN_STR_SUB        Add an entry to the initial Str-Sub vocab.
+ *
+ *      Arguments:  (Both are strings)
+ *          princ         Principal name by which the string is known
+ *          subst         Substitution-string that replaces the Principal.
+ */
+#define BUILTIN_STR_SUB(princ, subst)    \
+     { princ , subst , (str_sub_vocab_t *)NULL  }
+#endif     /*  On the way out   */
+
+
+void init_str_sub_vocab( str_sub_vocab_t *str_sub_vocab_tbl,
+			            int max_indx);
+void add_str_sub_entry( char *ename,
+                        	    char *subst_str,
+			        	str_sub_vocab_t **str_sub_vocab );
+char *lookup_str_sub( char *tname, str_sub_vocab_t *str_sub_vocab );
+bool create_str_sub_alias(char *new_name,
+                        	     char *old_name,
+			        	 str_sub_vocab_t **str_sub_vocab );
+bool exists_in_str_sub( char *tname, str_sub_vocab_t *str_sub_vocab );
+void reset_str_sub_vocab( str_sub_vocab_t **str_sub_vocab ,
+	        		     str_sub_vocab_t *reset_position );
+
+
+#endif   /*  _TOKE_STRSUBVOCAB_H    */

Added: fcode-utils/toke/ticvocab.c
===================================================================
--- fcode-utils/toke/ticvocab.c	                        (rev 0)
+++ fcode-utils/toke/ticvocab.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,426 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      General-purpose support functions for
+ *      Threaded Interpretive Code (T. I. C.)-type vocabularies
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      We are going to implement a strategy that takes better advantage
+ *          of the concept of Threaded Interpretive Code  (well, okay,
+ *          it won't really be interpretive ... )  We will use it to
+ *          implement a small (but expandable) subset of FORTH-like
+ *          commands in Tokenizer-Escape mode, as well as a few other
+ *          things, such as conditional-tokenization.
+ *
+ *      The Threaded Interpretive Code Header data structure is described
+ *          in detail in the accompanying  ticvocab.h  header-file.
+ *
+ *      In most cases, the contents of a beginning portion of the vocabulary
+ *          are known at compile-time, and later parts are added by the
+ *          user at run-time.  (The linked-list structure is needed to allow
+ *          for that.)  We can initialize the known start of the vocabulary
+ *          easily, except for the link-pointers, as an array.
+ *      We can either explicitly state an index for each entry's link-pointer
+ *          to the previous entry (which can become a problem to maintain) or
+ *          have a function to initialize the links dynamically, at run-time.
+ *      I think I will (regretfully, resignedly) choose the latter.
+ *          
+ *      We will define a few general-purpose functions for dealing with
+ *          T. I. C. -type vocabularies.  Eventually, it might be a good
+ *          idea to convert all the vocabularies to this structure...
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *
+ *      Revision History:
+ *          Mon, 19 Dec 2005 by David L. Paktor
+ *          Begin converting most, if not all, of the vocabularies to
+ *              T. I. C. -type structure.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          init_tic_vocab        Initialize a TIC_HDR -type vocabulary
+ *          add_tic_entry         Add an entry to a TIC_HDR -type vocabulary
+ *          lookup_tic_entry      Look for a name in a TIC_HDR -type vocabulary
+ *          handle_tic_vocab      Perform a function in a TIC_HDR -type vocab
+ *          exists_in_tic_vocab   Confirm whether a given name exists in a
+ *                                    TIC_HDR -type vocabulary
+ *          create_tic_alias      Duplicate the behavior of one name with
+ *                                     another name.  Return a "success" flag.
+ *          reset_tic_vocab       Reset a given TIC_HDR -type vocabulary to
+ *                                    its "Built-In" position.
+ *
+ **************************************************************************** */
+
+
+/* **************************************************************************
+ *
+ *          Global Variables Exported
+ *              tic_found       The entry, in a TIC_HDR -type vocabulary,
+ *                                  that has just been found and is being 
+ *                                  "handled".  Needed for protection against
+ *                                  recursion in a User-defined Macro (which
+ *                                  should occur only rarely).
+ *
+ **************************************************************************** */
+
+
+#include <stdlib.h>
+#include <string.h>
+#include "ticvocab.h"
+#include "errhandler.h"
+
+tic_hdr_t *tic_found;
+
+/* **************************************************************************
+ *
+ *      Function name:  init_tic_vocab
+ *      Synopsis:       Dynamically initialize the link-pointers
+ *                          of the.given TIC_HDR -type vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tic_vocab_tbl       Pointer to the initial TIC_HDR vocab array
+ *             max_indx            Maximum Index of the initial array.
+ *             tic_vocab_ptr       Pointer to the vocab "tail"
+ *
+ *      Outputs:
+ *         Returned Value:         None
+ *         Global Variables:
+ *              The link-fields of the initial TIC_HDR vocab array entries
+ *                  will be filled in.
+ *         Supplied Pointers:
+ *             *tic_vocab_ptr       Points to the last element in the array
+ *
+ *      Process Explanation:
+ *          The value that  tic_vocab_ptr  has upon entry to the routine
+ *              (which may point to the end of another array which is to
+ *              precede this one in the voacbulary) gets entered into
+ *              the link-pointer field of the first element of the array.
+ *          For this reason, it is important that all TIC_HDR vocabulary
+ *              pointers that will be pased to this routine have their
+ *              initial values explicitly declared NULL. 
+ *
+ **************************************************************************** */
+
+void init_tic_vocab( tic_hdr_t *tic_vocab_tbl,
+                         int max_indx,
+			     tic_hdr_t **tic_vocab_ptr)
+{
+    int indx;
+    for ( indx = 0 ; indx < max_indx ; indx++ )
+    {
+        tic_vocab_tbl[indx].next = *tic_vocab_ptr;
+	*tic_vocab_ptr = &tic_vocab_tbl[indx];
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  add_tic_entry
+ *      Synopsis:       Add an entry to the given TIC_HDR -type vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname         Pointer to space containing the name of the entry
+ *             tfunct        Pointer to the routine the new entry will call
+ *             tparam        The "parameter field" value (may be a pointer)
+ *             fw_defr       FWord Token of the entry's Definer
+ *             pfldsiz       Size of "param field" (if a pointer to alloc'd mem)
+ *             ign_fnc       Pointer to "ignoring" routine for new entry
+ *             tic_vocab     Pointer to ptr to "tail" of T.I.C.-type vocab-list 
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Supplied Pointers:
+ *             *tic_vocab           Will point to new entry
+ *         Memory Allocated:
+ *             For the new entry.
+ *         When Freed?
+ *             When reset_tic_vocab() is applied to the same vocab-list.
+ *
+ *      Error Detection:
+ *          Failure to allocate memory is a Fatal Error.
+ *
+ *      Process Explanation:
+ *          The name pointer is presumed to already point to a stable,
+ *              newly-allocated memory-space.  If the parameter field is
+ *              actually a pointer, it, too, is presumed to already have
+ *              been allocated.
+ *          Memory will be allocated for the entry itself; its pointers
+ *              will be entered and the given pointer-to-the-tail-of-the-
+ *              -vocabulary will be updated to point to the new entry.
+ *
+ **************************************************************************** */
+
+void add_tic_entry( char *tname,
+                        void (*tfunct)(),
+                             TIC_P_DEFLT_TYPE tparam,
+                                 fwtoken fw_defr,
+				     int pfldsiz,
+                                         void (*ign_fnc)(),
+                                             tic_hdr_t **tic_vocab )
+{
+    tic_hdr_t *new_entry;
+
+    new_entry = safe_malloc(sizeof(tic_hdr_t), "adding tic_entry");
+    new_entry->name              =  tname;
+    new_entry->next              = *tic_vocab;
+    new_entry->funct             =  tfunct;
+    new_entry->pfield.deflt_elem =  tparam;
+    new_entry->fword_defr        =  fw_defr;
+    new_entry->ign_func          =  ign_fnc;
+    new_entry->pfld_size         =  pfldsiz;
+
+    *tic_vocab = new_entry;
+
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_tic_entry
+ *      Synopsis:       Look for a name in the given TIC_HDR -type vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                The "target" name for which to look
+ *             tic_vocab            Pointer to the T. I. C. -type vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:          Pointer to the relevant entry, or
+ *                                      NULL if name not found.
+ *
+ *      Extraneous Remarks:
+ *          We don't set the global  tic_found  here because this routine
+ *              is not always called when the found function is going to
+ *              be executed; sometimes it is called for error-detection,
+ *              for instance...
+ *
+ **************************************************************************** */
+ 
+tic_hdr_t *lookup_tic_entry( char *tname, tic_hdr_t *tic_vocab )
+{
+    tic_hdr_t *curr ;
+
+    for (curr = tic_vocab ; curr != NULL ; curr=curr->next)
+    {
+        if ( strcasecmp(tname, curr->name) == 0 )
+	{
+	    break;
+	}
+    }
+
+    return ( curr ) ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_in_tic_vocab
+ *      Synopsis:       Confirm whether the given name exists in the
+ *                      given TIC_HDR -type vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                The name for which to look
+ *             tic_vocab            Pointer to the T. I. C. -type vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if name is found,
+ *
+ **************************************************************************** */
+
+bool exists_in_tic_vocab( char *tname, tic_hdr_t *tic_vocab )
+{
+    tic_hdr_t *found ;
+    bool retval = FALSE;
+
+    found = lookup_tic_entry( tname, tic_vocab );
+    if ( found != NULL )
+    {
+	retval = TRUE;
+    }
+
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  create_tic_alias
+ *      Synopsis:       Create an Alias in a TIC_HDR -type vocabulary
+ *                          Return a "success" flag.
+ *
+ *      Associated FORTH word:                 ALIAS
+ *
+ *      Inputs:
+ *         Parameters:
+ *             old_name             Name of existing entry
+ *             new_name             New name for which to create an entry
+ *             *tic_vocab           Pointer to the "tail" of the
+ *                                      T. I. C. -type vocab-list 
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if  old_name  found in given vocab
+ *         Supplied Pointers:
+ *             *tic_vocab           Will be updated to point to the new entry
+ *         Memory Allocated:
+ *             For the new entry, by the support routine.
+ *         When Freed?
+ *             When reset_tic_vocab() is applied to the same vocab-list.
+ *
+ *      Process Explanation:
+ *          Both the "old" and "new" names are presumed to already point to
+ *              stable, freshly allocated memory-spaces.
+ *          Even if the "old" entry's  pfld_size  is not zero, meaning its
+ *              param-field is a pointer to allocated memory, we still do
+ *              not need to copy it into a freshly allocated memory-space,
+ *              as long as we make the new alias-entry's  pfld_size  zero:
+ *              the reference to the old space will work, and the old
+ *              entry's param-field memory space will not be freed with
+ *              the alias-entry but only with the "old" entry.
+ *
+ **************************************************************************** */
+
+bool create_tic_alias( char *new_name, char *old_name, tic_hdr_t **tic_vocab )
+{
+    tic_hdr_t *found ;
+    bool retval = FALSE;
+
+    found = lookup_tic_entry( old_name, *tic_vocab );
+    if ( found != NULL )
+    {
+	add_tic_entry( new_name, found->funct,
+			   found->pfield.deflt_elem,
+			       found->fword_defr,
+			           0, found->ign_func, tic_vocab );
+	retval = TRUE;
+    }
+
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_tic_vocab
+ *      Synopsis:       Perform the function associated with the given name
+ *                      in the given TIC_HDR -type vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                The "target" name for which to look
+ *             tic_vocab            Pointer to the T. I. C. -type vocabulary
+ *
+ *      Outputs:
+ *         Returned Value:   TRUE if the given name is valid in the given vocab
+ *         Global Variables:
+ *             tic_found            Points to the TIC entry of the "target"
+ *                                      name, if it was found; else, NULL.
+ *         Global Behavior:
+ *             Whatever the associated function does...
+ *
+ *      Process Explanation:
+ *          Find the name and execute its associated function.
+ *          If the name is not in the given vocabulary, return
+ *              an indication; leave it to the calling routine
+ *              to decide how to proceed.
+ *
+ **************************************************************************** */
+ 
+bool handle_tic_vocab( char *tname, tic_hdr_t *tic_vocab )
+{
+    bool retval = FALSE;
+    
+    tic_found = lookup_tic_entry( tname, tic_vocab );
+    if ( tic_found != NULL )
+    {
+        tic_found->funct( tic_found->pfield);
+	retval = TRUE;
+    }
+
+    return ( retval ) ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_tic_vocab
+ *      Synopsis:       Reset a given TIC_HDR -type vocabulary to
+ *                          its given "Built-In" position.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             *tic_vocab            Pointer to the T. I. C.-type vocab-list
+ *             reset_position        Position to which to reset the list
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Supplied Pointers:
+ *             *tic_vocab          Reset to given "Built-In" position.
+ *         Memory Freed
+ *             All memory allocated by user-definitions will be freed
+ *
+ *      Process Explanation:
+ *          The "stable memory-spaces" to which the name and parameter
+ *              field pointers point are presumed to have been acquired
+ *              by allocation of memory, which is reasonable for entries
+ *              created by the user as opposed to the built-in entries,
+ *              which we are, in any case, not releasing.
+ *          The parameter-field size field tells us whether we need to
+ *              free()  the parameter-field pointer.
+ *
+ **************************************************************************** */
+
+void reset_tic_vocab( tic_hdr_t **tic_vocab, tic_hdr_t *reset_position )
+{
+    tic_hdr_t *next_t;
+
+    next_t = *tic_vocab;
+    while ( next_t != reset_position  )
+    {
+	next_t = (*tic_vocab)->next ;
+
+	free( (*tic_vocab)->name );
+	if ( (*tic_vocab)->pfld_size != 0 )
+	{
+	    free( (*tic_vocab)->pfield.chr_ptr );
+	}
+	free( *tic_vocab );
+	*tic_vocab = next_t ;
+    }
+}

Added: fcode-utils/toke/ticvocab.h
===================================================================
--- fcode-utils/toke/ticvocab.h	                        (rev 0)
+++ fcode-utils/toke/ticvocab.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,478 @@
+#ifndef _TOKE_TICVOCAB_H
+#define _TOKE_TICVOCAB_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      General-purpose support structures and function prototypes for
+ *          Threaded Interpretive Code (T. I. C.)-type vocabularies.
+ *          See ticvocab.c
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Structures:
+ *
+ *      The Threaded Interpretive Code Header data structure consists of:
+ *          (1)  A pointer to the function name as seen in the source
+ *          (2)  A link to the next (well, preceding) element
+ *          (3)  A Pointer to the routine to run as the function's behavior
+ *               during active processing.  It takes the "parameter field"
+ *               item as its argument and has no return-value.
+ *          (4)  The "parameter field" item, which will be passed as an
+ *               argument to the function. It may be an FCode- or FWord-
+ *               -token, or an item of another type, or a pointer to data.
+ *               The function may re-cast it as needed.
+ *          (5)  The "Definer" of the entry; a member of the subset of FWord-
+ *               -tokens that may be Definers.
+ *          (6)  A Pointer to a routine to be run when the word is encountered
+ *               in a Conditional Compilation section that is being ignored.
+ *               Certain functions still need to be recognized in that state,
+ *               and will require special behaviors; those that can be simply
+ *               ignored will have a NULL pointer in this field.  The function
+ *               in this field, also, takes the "parameter field" item as its
+ *               argument and has no return-value.
+ *          (7)  The size of the data pointed to by the "parameter field" item,
+ *               if it is a pointer to allocated data; used to allocate space
+ *               for a copy of the p.f. when making an alias, and to indicate
+ *               whether the space needs to be freed.  Automatic procedures
+ *               are too often fooled, so we don't leave things to chance...
+ *      To accommodate C's insistence on strong-typing, we might need
+ *          to define different "parameter field" structure-types; see
+ *          description of "Parameter Field as a union", below.
+ *
+ *      It's not an exact name, but we'll still call it a T. I. C. Header
+ *
+ *      We will use this structure for most of our vocabularies...
+ *
+ ****************************************************************************
+ *
+ *     The "Parameter Field" is a union of several alternative types,
+ *         for convenience, to avert excessive type-casting.  It may
+ *         either be the Parameter Data itself (if it is small enough)
+ *         or a pointer to Parameter Data.
+ *     In the case where the Parameter Field is the data itself, or is a
+ *         pointer to hard-coded (non-allocated) data, the "size" field
+ *         will be zero, to indicate that no space allocation is required.
+ *     The types are (starting with the default for initialization)
+ *         A long integer (including an FCode Token)
+ *         An FWord Token
+ *         A pointer to a boolean variable
+ *         A pointer to a character or a character string
+ *         If the parameter field value is intended to be a single character,
+ *             it must be recast as a long integer, in order to preserve
+ *             portability across big- and little- -endian platforms.
+ *
+ **************************************************************************** */
+
+/* ************************************************************************** *
+ *
+ *      Macros:
+ *          NO_PARAM_TIC       Create an entry in the initial "Built-In"
+ *                                 portion of a TIC-type vocabulary with
+ *                                 an empty "param field".
+ *          NO_PARAM_IGN       Create an entry with an empty "param field"
+ *                                 whose action- and "Ignore"- -functions
+ *                                 are the same.
+ *          VALPARAM_TIC       " ... with a literal value for the "param field"
+ *          DUALFUNC_TIC       " ... with a literal "param" and both an action-
+ *                                 and an "Ignore"- -function.
+ *          DUFNC_FWT_PARM      A  tic_fwt_hdr_t -type entry with a literal
+ *                                 "param" and both an action-function and an
+ *                                 "Ignore"-function.
+ *          FWORD_TKN_TIC      " ... with an FWord Token for the "param field"
+ *          DUALFUNC_FWT_TIC   " " ... and both action- and "Ignore"- -function
+ *          BUILTIN_MAC_TIC    A  tic_mac_hdr_t -type entry with a Macro-
+ *                                 -Substitution-String "param"
+ *          BUILTIN_BOOL_TIC   A  tic_bool_hdr_t -type entry with a boolean-
+ *                                 -variable-pointer "param"
+ *
+ **************************************************************************** */
+
+#include "types.h"
+#include "dictionary.h"
+
+/* **************************************************************************
+ *
+ *      In order to work around C's limitations regarding data-typing
+ *          during initialization, we need to create some alternative
+ *          data-structures that exactly match the layout of  tic_hdr_t
+ *          as defined above, but have a different default-type for
+ *          the parameter field.
+ *      I apologize in advance for any maintenance awkwardnesses this
+ *          might engender; I would have come up with a more convenient
+ *          way to do this if I could.  At least, the pieces that need
+ *          to be co-ordinated are in close proximity to each other...
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      The differences are all centered in the "parameter field" of
+ *          the TIC header structure.  In order to make sure all the
+ *          alternative types map smoothly into each other, we will
+ *          create a series of "union" types, differing only in what
+ *          their default type is.  We will keep the "parameter field"
+ *          item the size of a Long Integer.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      This form is the default type.
+ *
+ **************************************************************************** */
+
+#define TIC_P_DEFLT_TYPE  long
+typedef union tic_param
+    {
+	TIC_P_DEFLT_TYPE  deflt_elem ;  /*  The "default element" aspect.  */
+	long              long_val ;    /*  Explicitly specifying "long"   */
+	fwtoken           fw_token ;    /*  FWord Token                    */
+	bool             *bool_ptr ;    /*  Pointer to a boolean variable  */
+	char             *chr_ptr  ;    /*  Pointer to a character string  */
+
+	/*  The "character value" aspect behaves differently on big-
+	 *      versus little- -endian platforms (for initialization,
+	 *      anyway), so we cannot actually use it.
+	 *  Keep this commented-out, as a reminder.
+	 */
+     /* char             char_val ;    */
+	/*  We will recast "character value" as an integer.  */
+
+    } tic_param_t ;
+
+typedef struct tic_hdr
+    {
+        char             *name;
+	struct tic_hdr   *next;
+	void            (*funct)();      /*  Function for active processing  */
+	tic_param_t       pfield;
+	fwtoken           fword_defr;    /*  FWord Token of entry's Definer  */
+	void            (*ign_func)();   /*  Function in "Ignored" segment   */
+	int               pfld_size;
+    }  tic_hdr_t ;
+
+/* **************************************************************************
+ *
+ *      This form is for initialization of an FWord Token list.
+ *
+ **************************************************************************** */
+
+#define TIC_FWT_P_DEFLT_TYPE  fwtoken
+typedef union tic_fwt_param
+    {
+	TIC_FWT_P_DEFLT_TYPE  deflt_elem ;  /*  "Default element" aspect.  */
+	long              long_val ;    /*  Long integer                   */
+	fwtoken           fw_token ;    /*  Explicit FWord Token           */
+	bool             *bool_ptr ;    /*  Pointer to a boolean variable  */
+	char             *chr_ptr  ;    /*  Pointer to a character string  */
+    } tic_fwt_param_t ;
+
+typedef struct tic_fwt_hdr
+    {
+        char               *name;
+	struct tic_fwt_hdr *next;
+	void              (*funct)();    /*  Function for active processing  */
+	tic_fwt_param_t     pfield;
+	fwtoken             fword_defr;  /*  FWord Token of entry's Definer  */
+	void              (*ign_func)(); /*  Function in "Ignored" segment   */
+	int                 pfld_size;
+    }  tic_fwt_hdr_t ;
+
+
+/* **************************************************************************
+ *
+ *      The third form is for initialization of Built-in Macros.  Its
+ *          default parameter field type is a pointer to a compiled-in
+ *          (i.e., constant) character string.  Its  pfld_size  can be
+ *          left as zero, because no allocated-memory copies of the
+ *          string will be needed, even for aliases.  (User-created
+ *          macros, however, will need to allocate strings; we will
+ *          deal with that elsewhere...)
+ *
+ **************************************************************************** */
+
+#define TIC_MAC_P_DEFLT_TYPE  char *
+typedef union tic_mac_param
+    {
+	TIC_MAC_P_DEFLT_TYPE  deflt_elem ;  /*  "Default element" aspect.  */
+	long              long_val ;    /*  Long integer                   */
+	fwtoken           fw_token ;    /*  FWord Token                    */
+	bool             *bool_ptr ;    /*  Pointer to a boolean variable  */
+	char            *chr_ptr ;      /*  Explicit Pointer to char str.  */
+    } tic_mac_param_t ;
+
+typedef struct tic_mac_hdr
+    {
+        char               *name;
+	struct tic_mac_hdr *next;
+	void              (*funct)();
+	tic_mac_param_t     pfield;
+	fwtoken             fword_defr;
+	void              (*ign_func)();
+	int                 pfld_size;
+    }  tic_mac_hdr_t ;
+
+/* **************************************************************************
+ *
+ *      The next form is for initialization of Condtionals.  Its default
+ *          parameter field type is a pointer to a boolean variable.
+ *
+ **************************************************************************** */
+
+#define TIC_BOOL_P_DEFLT_TYPE  bool *
+typedef union tic_bool_param
+    {
+	TIC_BOOL_P_DEFLT_TYPE deflt_elem ;  /*  "Default element" aspect.  */
+	long              long_val ;    /*  Long integer                   */
+	fwtoken           fw_token ;    /*  FWord Token                    */
+	bool             *bool_ptr ;    /*  Explicit Ptr to boolean v'ble  */
+	char             *chr_ptr  ;    /*  Pointer to a character string  */
+    } tic_bool_param_t ;
+
+typedef struct tic_bool_hdr
+    {
+        char                *name;
+	struct tic_bool_hdr *next;
+	void               (*funct)();
+	tic_bool_param_t     pfield;
+	fwtoken              fword_defr;
+	void               (*ign_func)();
+	int                  pfld_size;
+    }  tic_bool_hdr_t ;
+
+
+
+/* **************************************************************************
+ *
+ *      Various macros to create an entry in the initial "Built-In" portion
+ *           of a vocabulary-list, specified as an array of one of the
+ *           matching forms of a T. I. C. Header; each macro adjusts type
+ *           casting as needed.
+ *      In all cases, the "size" field will be set to zero, indicating that
+ *         the "param field" item is either the complete data or a pointer
+ *         to a hard-coded (non-allocated) data item.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *          Macro Name:   NO_PARAM_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of a TIC_HDR -type vocabulary with an empty
+ *                            "param field"
+ *                            
+ *   Arguments:
+ *       nam      (string)         Name of the entry as seen in the source
+ *       func     (routine-name)   Name of internal function to call
+ *
+ **************************************************************************** */
+#define NO_PARAM_TIC(nam, func )  \
+  { nam , (tic_hdr_t *)NULL , func , { 0 }, UNSPECIFIED , NULL , 0 }
+
+
+/* **************************************************************************
+ *          Macro Name:   NO_PARAM_IGN
+ *                        Create an entry with an empty "param field"
+ *                            whose action-function and "Ignore"-function
+ *                            are the same.
+ *                            
+ *   Arguments:
+ *       nam      (string)         Name of the entry as seen in the source
+ *       func     (routine-name)   Name of internal function to call for both
+ *
+ *
+ **************************************************************************** */
+#define NO_PARAM_IGN(nam, func )  \
+  { nam , (tic_hdr_t *)NULL , func , { 0 }, UNSPECIFIED , func , 0 }
+
+
+/* **************************************************************************
+ *
+ *      Variations on the above, to compensate for Type-Casting complications
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *          Macro Name:   VALPARAM_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of a TIC_HDR -type vocabulary with a literal
+ *                            value for the "param field" and a specifiable
+ *                            "definer".
+ *   Arguments:
+ *       nam      (string)         Name of the entry as seen in the source
+ *       func     (routine-name)   Name of internal function to call
+ *       pval     (integer)        The "param field" item
+ *       definr   (fword_token)    "Definer" of the entry
+ *
+ *      The "param field" item will be recast to the required default type.
+ *
+ **************************************************************************** */
+
+#define VALPARAM_TIC(nam, func, pval, definr )  \
+    { nam , (tic_hdr_t *)NULL , func ,  \
+        { (TIC_P_DEFLT_TYPE)(pval) }, definr , NULL , 0 }
+
+
+/* **************************************************************************
+ *          Macro Name:   DUALFUNC_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of a TIC_HDR -type vocabulary with both an
+ *                            "active" and an "ignoring" function, a literal
+ *                            value for the "param field" and a specifiable
+ *                            "definer".
+ *                            
+ *   Arguments:
+ *       nam      (string)         Name of the entry as seen in the source
+ *       afunc    (routine-name)   Name of internal "active" function
+ *       pval     (integer)        The "param field" item
+ *       ifunc    (routine-name)   Name of "ignoring" function
+ *       definr   (fword_token)    "Definer" of the entry
+ *
+ *      The "param field" item will be recast to the required default type.
+ *
+ **************************************************************************** */
+#define DUALFUNC_TIC(nam, afunc, pval, ifunc, definr )  \
+    { nam , (tic_hdr_t *)NULL , afunc ,  \
+        { (TIC_P_DEFLT_TYPE)(pval) }, definr , ifunc , 0 }
+
+/*  Similar but a  tic_fwt_hdr_t  type structure  */
+#define DUFNC_FWT_PARM(nam, afunc, pval, ifunc, definr )  \
+    { nam , (tic_fwt_hdr_t *)NULL , afunc ,  \
+        { (TIC_FWT_P_DEFLT_TYPE)(pval) }, definr , ifunc , 0 }
+
+
+/* **************************************************************************
+ *          Macro Name:   FWORD_TKN_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of an FWord Token list of  tic_fwt_hdr_t  type.
+ *                            
+ *   Arguments:
+ *       nam         (string)         Name of the entry as seen in the source
+ *       func        (routine-name)   Name of internal function to call
+ *       fw_tokval   (fword_token)    Value of the FWord Token
+ *       definr      (fword_token)    "Definer" of the entry
+ *
+ *      The "param field" item should not need be recast.
+ *
+ **************************************************************************** */
+
+#define FWORD_TKN_TIC(nam, func, fw_tokval, definr )    \
+    { nam , (tic_fwt_hdr_t *)NULL , func , { fw_tokval }, definr , NULL , 0 }
+
+/* **************************************************************************
+ *          Macro Name:   DUALFUNC_FWT_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of an FWord Token list of  tic_fwt_hdr_t  type
+ *                            with both an action- and an "Ignore"- -function.
+ *                            
+ *   Arguments:
+ *       nam         (string)         Name of the entry as seen in the source
+ *       afunc       (routine-name)   Name of internal "Action" function
+ *       fw_tokval   (fword_token)    Value of the FWord Token
+ *       ifunc       (routine-name)   Name of "ignoring" function
+ *       definr      (fword_token)    "Definer" of the entry
+ *
+ *      The "param field" item should not need be recast.
+ *
+ **************************************************************************** */
+#define DUALFUNC_FWT_TIC(nam, afunc, fw_tokval, ifunc, definr )    \
+    { nam , (tic_fwt_hdr_t *)NULL , afunc , { fw_tokval }, definr , ifunc , 0 }
+
+/* **************************************************************************
+ *          Macro Name:   BUILTIN_MAC_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of a Macros vocabulary of  tic_mac_hdr_t  type.
+ *                            
+ *   Arguments:
+ *       nam         (string)         Name of the entry as seen in the source
+ *       func        (routine-name)   Name of internal function to call
+ *       alias       (string)         Macro-Substitution string
+ *
+ *      The "param field" item should not need be recast.
+ *      The "definer" will be MACRO_DEF
+ *      Builtin Macros do not need to be expanded while Ignoring, so
+ *          the ign_func will be NULL
+ *
+ **************************************************************************** */
+
+#define BUILTIN_MAC_TIC(nam, func, alias )    \
+    { nam , (tic_mac_hdr_t *)NULL , func , { alias }, MACRO_DEF , NULL , 0 }
+
+
+/* **************************************************************************
+ *          Macro Name:   BUILTIN_BOOL_TIC
+ *                        Create an entry in the initial "Built-In" portion
+ *                            of a Condtionals list of  tic_bool_hdr_t  type.
+ *                            
+ *   Arguments:
+ *       nam         (string)          Name of the entry as seen in the source
+ *       func        (routine-name)    Name of internal function to call
+ *       bool_vbl    (boolean v'ble)   Name of the boolean variable.
+ *
+ *      The "param field" item should not need be recast.
+ *      For all of the Condtionals, the "Ignoring" function is the same
+ *          as the "Active" function.
+ *      The "definer" will be COMMON_FWORD
+ *
+ **************************************************************************** */
+
+#define BUILTIN_BOOL_TIC(nam, func, bool_vbl )    \
+    { nam , (tic_bool_hdr_t *)NULL , func , { &bool_vbl },   \
+        COMMON_FWORD , func , 0 }
+
+
+/* **************************************************************************
+ *
+ *     Exported Variables and Function Prototypes the rest of the way...
+ *
+ **************************************************************************** */
+
+extern tic_hdr_t *tic_found;
+
+void init_tic_vocab( tic_hdr_t *tic_vocab_tbl,
+                         int max_indx,
+			     tic_hdr_t **tic_vocab_ptr);
+void add_tic_entry( char *tname,
+                        void (*tfunct)(),
+                             TIC_P_DEFLT_TYPE tparam,
+                                 fwtoken fw_defr,
+				     int pfldsiz,
+                                         void (*ign_fnc)(),
+                                             tic_hdr_t **tic_vocab );
+tic_hdr_t *lookup_tic_entry( char *tname, tic_hdr_t *tic_vocab );
+bool exists_in_tic_vocab( char *tname, tic_hdr_t *tic_vocab );
+bool handle_tic_vocab( char *tname, tic_hdr_t *tic_vocab );
+bool create_tic_alias( char *new_name,
+                                  char *old_name,
+				      tic_hdr_t **tic_vocab );
+void reset_tic_vocab( tic_hdr_t **tic_vocab, tic_hdr_t *reset_position );
+
+#endif   /*  _TOKE_TICVOCAB_H    */

Modified: fcode-utils/toke/toke.c
===================================================================
--- fcode-utils/toke/toke.c	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/toke.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -7,7 +7,7 @@
  *  This program is part of a free implementation of the IEEE 1275-1994 
  *  Standard for Boot (Initialization Configuration) Firmware.
  *
- *  Copyright (C) 2001-2005 by Stefan Reinauer 
+ *  Copyright (C) 2001-2006 by Stefan Reinauer 
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -24,6 +24,12 @@
  *
  */

+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -34,47 +40,214 @@
 #include <getopt.h>
 #endif

+#include "types.h"
 #include "toke.h"
 #include "stream.h"
 #include "stack.h"
 #include "emit.h"

-#define TOKE_VERSION "0.6.10"
+#define TOKE_VERSION "1.0.0"

-int verbose=0;
-int noerrors=0;
+#include "vocabfuncts.h"
+#include "scanner.h"
+#include "errhandler.h"
+#include "usersymbols.h"
+#include "clflags.h"
+#include "tracesyms.h"

+#define CORE_COPYR   "(C) Copyright 2001-2006 Stefan Reinauer.\n" \
+		     "(C) Copyright 2006 coresystems GmbH "
+#define IBM_COPYR    "(C) Copyright 2005 IBM Corporation.  All Rights Reserved."
+
+/*  Temporary hack during development...  See DATE_STAMP  line below... */
+#ifdef DEVEL
+#include "date_stamp.h"
+#endif /*  DEVEL  */
+
+/* **************************************************************************
+ *
+ *     Global Variables Exported:
+ *        verbose          If true, enable optional messages.
+ *        noerrors         If true, create binary even if error(s) encountered.
+ *        fload_list       If true, create an "FLoad-List" file
+ *        dependency_list  If true, create a "Dependencies-List" file
+ *
+ **************************************************************************** */
+
+bool verbose         = FALSE;
+bool noerrors        = FALSE;
+bool fload_list      = FALSE;
+bool dependency_list = FALSE;
+
+/* **************************************************************************
+ *
+ *              Internal Static Variables
+ *         outputname    Name of output file supplied on command-line
+ *                           with the optional  -o  switch.
+ *              Internal System Variable
+ *         optind        Index into argv vector of first param after options,
+ *                           from which input file names will be taken.
+ *
+ **************************************************************************** */
+
+static char *outputname = NULL;
+
+/* **************************************************************************
+ *
+ *    Print the copyright message.
+ *
+ **************************************************************************** */
 static void print_copyright(void)
 {
-        printf( "Welcome to toke - OpenBIOS tokenizer v%s\nCopyright (c)"
-		" 2001-2005 by Stefan Reinauer \n"
+	printf( "Welcome to toke - OpenBIOS tokenizer v" TOKE_VERSION "\n"
+		CORE_COPYR "\n" IBM_COPYR "\n"
    	"This program is free software; you may redistribute it "
    	"under the terms of\nthe GNU General Public License. This "
-		"program has absolutely no warranty.\n\n", TOKE_VERSION);
+		"program has absolutely no warranty.\n\n");
+#ifdef DEVEL
+        /*  Temporary hack during development... */
+	printf( "\tTokenizer Compiled " DATE_STAMP "\n" );
+#endif /*  DEVEL  */
+
 }

+/* **************************************************************************
+ *
+ *      Function name:    usage
+ *      Synopsis:         Print convenient usage-help message
+ *
+ **************************************************************************** */
+
 static void usage(char *name)
 {
-	printf("usage: %s [-v] [-i] [-o target] <forth-file>\n\n",name);
-	printf("  -v|--verbose          print debug messages\n");
-	printf("  -i|--ignore-errors    don't bail out after an error\n");
+	printf("usage: %s [-v] [-i] [-l] [-P] [-o target] <[-d name[=value]]> "
+				"<[-f [no]flagname]> <[-I dir-path]> "
+				"<[-T symbol]> <forth-file>\n\n",name);
+	printf("  -v|--verbose          print Advisory messages\n");
+	printf("  -i|--ignore-errors    don't suppress output after errors\n");
+	printf("  -l|--load-list        create list of FLoaded file names\n");
+	printf("  -P|--dependencies     create dePendency-list file\n");
+	printf("  -o|--output-name      send output to filename given\n");
+	printf("  -d|--define           create user-defined symbol\n");
+	printf("  -f|--flag             set (or clear) Special-Feature flag\n");
+	printf("  -I|--Include          add a directory to the Include-List\n");
+	printf("  -T|--Trace            add a symbol to the Trace List\n");
    printf("  -h|--help             print this help message\n\n");
-
+	printf("  -f|--flag    help     Help for Special-Feature flags\n");
 }

-int main(int argc, char **argv)
+/* **************************************************************************
+ *
+ *      Function name:    get_args
+ *      Synopsis:         Parse the Command-Line option switches
+ *      
+ *      Inputs:
+ *         Parameters:                  NONE
+ *         Global Variables:
+ *                    argc      Counter of command-line arguments
+ *                    argv      Vector pointing to command-line arguments
+ *         Command-Line Items:  The entire command-line will be parsed
+ *
+ *      Outputs:
+ *         Returned Value:              NONE
+ *         Global Variables:
+ *                verbose	     set by "-v" switch
+ *                noerrors           set by "-i" switch
+ *                fload_list         set by "-l" switch
+ *                dependency_list    set by "-P" switch
+ *         Internal Static Variables
+ *                outputname         set by "-o" switch
+ *         Internal System Variable
+ *                optind             Index into argv vector of the position
+ *                                       from which to take input file names.
+ *         Printout:
+ *                (Copyright was already printed by the main body as a
+ *                    matter of course, rather than here depending on
+ *                    the Verbose flag, because  getopt()  prints its
+ *                    own error messages and we want to be sure to show
+ *                    the Copyright notice before any error messages.)
+ *                Rules for Usage and Flags-list or Flags-help display:
+ *                  Usage message on Help-Request or error.
+ *                  Ask for usage help, get Usage plus list of Flag Names.
+ *                  Ask for Flags-help alone, get Flags-help (names plus
+ *                      explanations)
+ *                  Ask for usage help and for Flags-help, get Usage plus
+ *                      Flags-help, without redundant list of Flag Names.
+ *                  Any help-request, exit Zero
+ *                  Error in Option switches, or missing input-file name,
+ *                      get error-description plus Usage
+ *                  Error in Flag Names, get list of Flag Names.
+ *                  Any error, exit One.
+ *         Behavior:
+ *                Exit (non-failure) after printing "Help" message
+ *
+ *      Error Detection:     Exit with failure status on:
+ *          Unknown Option switches or Flag Names
+ *          Missing input file name
+ *
+ *      Process Explanation:
+ *           The following switches are recognized:
+ *               v
+ *               h
+ *               ?
+ *               i
+ *               I
+ *               l
+ *               P
+ *               o
+ *               d
+ *               f
+ *               T
+ *           The conditions they set remain in effect through
+ *               the entire program run.
+ *
+ *      Revision History:
+ *          Updated Fri, 15 Jul 2005 by David L. Paktor
+ *              Don't bail on first invalid option.
+ *              Flags to control special features
+ *              Usage messages for "special-feature" flags
+ *          Updated Mon, 18 Jul 2005 by David L. Paktor
+ *              Fine-tune Usage and Flags-list or Flags-help display.
+ *          Updated Sun, 27 Nov 2005 by David L. Paktor
+ *              Add FLoad-List flag
+ *          Updated Wed, 29 Nov 2005 by David L. Paktor
+ *              Make getopt() case-insensitive
+ *          Updated Fri, 17 Mar 2006 by David L. O'Paktor
+ *              Make getopt() case-sensitive again,
+ *                  add include-list support and dePendency-list switch
+ *
+ *      Extraneous Remarks:
+ *          We were originally thinking about defining various classes
+ *              of "Warning" Messages and (somehow) controlling their
+ *              display, but now that we have "special-feature" flags
+ *              that control the generation of specific messages, that
+ *              step has become unnecessary...
+ *
+ **************************************************************************** */
+
+static void get_args( int argc, char **argv )
 {
-	const char *optstring="vhio:?";
-	char *outputname = NULL;
+	const char *optstring="vhilPo:d:f:I:T:?";
    int c;
+	int argindx = 0;
+	bool inval_opt = FALSE;
+	bool help_mssg = FALSE;
+	bool cl_flag_error = FALSE;

while (1) {
 #ifdef __GLIBC__
    	int option_index = 0;
    	static struct option long_options[] = {
    		{ "verbose", 0, 0, 'v' },
+			{ "help", 0, 0, 'h' },
    		{ "ignore-errors", 0, 0, 'i' },
-			{ "help", 0, 0, 'h' },
+			{ "load-list",     0, 0, 'l' },
+			{ "dependencies",  0, 0, 'P' },
+			{ "output-name",   1, 0, 'o' },
+			{ "define",        1, 0, 'd' },
+			{ "flag",          1, 0, 'f' },
+			{ "Include",       1, 0, 'I' },
+			{ "Trace",         1, 0, 'T' },
    		{ 0, 0, 0, 0 }
    	};

@@ -86,63 +259,169 @@
    	if (c == -1)
    		break;

+		argindx++;
    	switch (c) {
    	case 'v':
-			verbose=1;
+			verbose=TRUE;
    		break;
    	case 'o':
    		outputname = optarg;
    		break;
    	case 'i':
-			noerrors=1;
+			noerrors = TRUE;
    		break;
+		case 'l':
+			fload_list = TRUE;
+			break;
+		case 'P':
+			dependency_list = TRUE;
+			break;
+		case 'd':
+			{
+			    char *user_symb = optarg;
+			    add_user_symbol(user_symb);
+			}
+			break;
+		case 'f':
+			cl_flag_error = set_cl_flag(optarg, FALSE) ;
+			break;
+		case 'I':
+			{
+			    char *incl_list_elem = optarg;
+			    add_to_include_list(incl_list_elem);
+			}
+			break;
+		case 'T':
+			add_to_trace_list(optarg);
+			break;
+		case '?':
+			/*  Distinguish between a '?' from the user
+			 *  and one  getopt()  returned
+			 */
+			if ( argv[argindx][1] != '?' )
+			{
+			    inval_opt = TRUE;
+			    break;
+			}
    	case 'h':
-		case '?':
-			print_copyright();
-			usage(argv[0]);
-			return 0;		
+		case 'H':
+			 help_mssg = TRUE;		
+			break;
    	default:
-			print_copyright();
+			/*  This is never executed
+			 *  because  getopt()  prints the
+			 *    "unknown option -- X"
+			 *  message and returns a '?'
+			 */
    		printf ("%s: unknown options.\n",argv[0]);
    		usage(argv[0]);
-			return 1;
+			exit( 1 );
    	}
    }

-	if (verbose)
-		print_copyright();
+	if ( help_mssg )
+	{
+	    usage(argv[0]);
+	    if ( ! clflag_help )
+	    {
+	        list_cl_flag_names();
+	    }
+	}
+	if ( clflag_help )  cl_flags_help();
+	if ( help_mssg || clflag_help )
+	{
+	    exit( 0 );
+	}

-	if (optind >= argc) {
-		print_copyright();
-		printf ("%s: filename missing.\n",argv[0]);
+	if ( inval_opt )      printf ("unknown options.\n");
+	if (optind >= argc)   printf ("Input file name missing.\n");
+	if ( inval_opt || (optind >= argc) )
+	{
    	usage(argv[0]);
-		return 1;
    }
+	if ( cl_flag_error )  list_cl_flag_names();

+	if ( inval_opt || (optind >= argc) || cl_flag_error )
+	{
+	    exit( 1);
+	}
+
+	if (verbose)
+	{
+	    list_user_symbols();
+	    list_cl_flag_settings();
+	    display_include_list();
+	}
+	save_cl_flags();
+}
+
+/* **************************************************************************
+ *
+ *      Main body of program.  Return 0 for success, 1 for failure.
+ *
+ *      Still to be done:
+ *          Devise a syntax to allow the command-line to specify multiple
+ *              input files together with an output file name for each.
+ *          Currently, the syntax allows only one output file name to be
+ *              specified; when multiple input file names are specified,
+ *              the specification of an output file name is disallowed,
+ *              and only the default output file names are permitted.
+ *              While this works around the immediate problem, a more
+ *              elegant solution could be devised...
+ *
+ **************************************************************************** */
+
+int main(int argc, char **argv)
+{
+	int retval = 0;
+
+	print_copyright();
+	get_args( argc, argv );
+
    init_stack();
    init_dictionary();
-	init_macros();
+
    init_scanner();
    
-	while (optind < argc) {
-		if (init_stream(argv[optind])) {
-			printf ("%s: warning: could not open file "%s"\n",
-					argv[0], argv[optind]);
-			optind++;
-			continue;
+	if ( outputname != NULL )
+	{
+	    if ( argc > optind + 1 )
+	    {
+	    /*  Multiple input file names w/ single output file name  */
+		/*  Work-around  */
+		printf( "Cannot specify single output file name "
+			"with multiple input file names.\n"
+			"Please either remove output-file-name specification,\n"
+			"or use multiple commands.\n");
+	        exit ( -2 );
+	    }
    	}
+
+	for ( ; optind < argc ; optind++ )
+	{
+	    bool stream_ok ;
+
+	    printf("\nTokenizing  %s   ", argv[optind]);
+	    init_error_handler();
+	    stream_ok = init_stream( argv[optind]);
+	    if ( stream_ok )
+	    {
    	init_output(argv[optind], outputname);

+		init_scan_state();
+
+		reset_vocabs();
+		reset_cl_flags();
+
    	tokenize();
    	finish_headers();
    	
-		close_output();
-		close_stream();
-		
-		optind++;
+		close_stream( NULL);
+		if ( close_output() )  retval = 1;
+	    }
    }
    
    exit_scanner();
-	return 0;
+	return retval;
 }

Modified: fcode-utils/toke/toke.h
===================================================================
--- fcode-utils/toke/toke.h	2006-08-18 09:45:11 UTC (rev 75)
+++ fcode-utils/toke/toke.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -1,3 +1,6 @@
+#ifndef _TOKE_TOKE_H
+#define _TOKE_TOKE_H
+
 /*
  *                     OpenBIOS - free your system! 
  *                         ( FCode tokenizer )
@@ -24,30 +27,25 @@
  *
  */

-#ifndef _H_TOKE
-#define _H_TOKE
+/* **************************************************************************
+ *         Modifications made in 2005 by IBM Corporation
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Modifications Author:  David L. Paktor    [e-mail]
+ **************************************************************************** */

-#define u8 unsigned char
-#define u16 unsigned short
-#define u32 unsigned int
-#define s16 short
-#define bool int
-#define TRUE (-1)
-#define FALSE (0)
-#define ANSI_ONLY

-extern void	init_scanner( void );
-extern void	exit_scanner( void );
+#include "types.h"

-extern void	init_dictionary( void );
-extern void	init_macros( void );
-extern void	tokenize( void );

-extern u16	lookup_token(char *name);
-extern u16	lookup_fword(char *name);
-extern char	*lookup_macro(char *name);
-extern int	add_token(u16 number, char *name);
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */

-extern int	add_macro(char *name, char *alias);
+extern bool verbose;
+extern bool noerrors;
+extern bool fload_list;
+extern bool dependency_list;

-#endif   /* _H_TOKE */
+#endif   /* _TOKE_TOKE_H */

Added: fcode-utils/toke/tokzesc.c
===================================================================
--- fcode-utils/toke/tokzesc.c	                        (rev 0)
+++ fcode-utils/toke/tokzesc.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,771 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Process activity inside "Tokenizer Escape" mode.
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          init_tokz_esc_vocab      Initialize the relevant vocabulary.
+ *          enter_tokz_esc           Enter "Tokenizer Escape" mode
+ *          handle_tokz_esc          Perform a function in the
+ *                                       "Tokenizer Escape" Vocabulary
+ *          exists_in_tokz_esc       Confirm whether a given name exists in 
+ *                                       the "Tokz_Esc_Vocabulary"
+ *          create_tokz_esc_alias    Add an alias to "Tokenizer Escape" space
+ *          reset_tokz_esc           Reset the "Tokenizer Escape" Vocabulary
+ *                                      to its "Built-In" position.
+ *
+ **************************************************************************** */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include "ticvocab.h"
+#include "tokzesc.h"
+#include "stack.h"
+#include "emit.h"
+#include "stream.h"
+#include "scanner.h"
+#include "errhandler.h"
+#include "strsubvocab.h"
+#include "nextfcode.h"
+
+#undef TOKZTEST     /*  Define for testing only; else undef   */
+#ifdef TOKZTEST         /*  For testing only   */
+   #include "vocabfuncts.h"
+   #include "date_stamp.h"
+#endif                  /*  For testing only   */
+
+/* **************************************************************************
+ *
+ *          Global Variables Imported
+ *              in_tokz_esc    State of the tokenization operation.
+ *                                Value is FALSE if it's operating normally
+ *                                or TRUE if it's in "Tokenizer Escape" mode.
+ *              nextfcode  
+ *              statbuf    
+ *              dstack         Pointer to current item on top of Data Stack.   
+ *              base           Current tokenizer numeric conversion radix
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      We are going to implement a mini-forth using a strategy based
+ *          on the concept of Threaded Interpretive Code  (well, okay,
+ *          it won't really be interpretive ... )
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *              Internal Static Variables -- (well, almost)
+ *      tokz_esc_vocab        Pointer to "tail" of "Tokenizer Escape" Vocab
+ *
+ *      While we would prefer to keep  tokz_esc_vocab  private to this file,
+ *          we find we need it in one other place, namely, macros.c
+ *      We will declare it "extern" within that file, rather than
+ *          exporting it widely by including it in a  .h  file
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      We will define and initialize a structure consisting of all the
+ *          functions we initially support for "Tokenizer Escape" mode,
+ *          called  tokz_esc_vocab_tbl  for "Tokenizer Escape Vocab Table".
+ *      I expect we can initialize its body, but I don't think there's a
+ *          convenient way in "C" to initialize its links; we'll have to
+ *          do that dynamically, at run-time.  Oh, well...
+ *
+ *      We'll call the pointer to it the "Tokenizer Escape" Vocabulary.
+ *          We have to declare it here, because we need to refer to it in
+ *          a function that will be entered into the Table.
+ *
+ *      We would like to pre-initialize it; to do so, we would have to
+ *          declare it extern here, then later create its instance and
+ *          assign its initial value.  That would be all well and good,
+ *          except that some linkers don't handle that very well, so, to
+ *          accommodate those, we have to include its initialization in
+ *          the same routine where we initialize the Table's links.
+ *
+ *      Revision History:
+ *          Updated Wed, 04 Jan 2006 by David L. Paktor
+ *          Initialization of tokz_esc_vocab is now included with
+ *              the call to  init_tic_vocab()
+ *
+ **************************************************************************** */
+
+tic_hdr_t *tokz_esc_vocab = NULL ;
+
+/* **************************************************************************
+ *
+ *      We'll give short prologs to the simpler functions that will be
+ *          used in the "Tokenizer Escape" Vocabulary.
+ *
+ *      All these take, as an argument, the "parameter field" pointer.
+ *          To satisfy C's strong-typing, it will always be declared
+ *          of a consistent type.  The routine itself can internally
+ *          recast -- or ignore -- it, as needed.
+ *      Many of these will refer to the Global Variable  statbuf .  It
+ *          all such cases, it is used for Error reporting.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Function name:  enter_tokz_esc
+ *      Synopsis:       When you start "Tokenizer Escape" mode.
+ *
+ *      Associated Tokenizer directive:        TOKENIZER[
+ *
+ *      Process Explanation:
+ *          Enter "Tokenizer Escape" mode ...  Save the current tokenizer
+ *              numeric conversion radix, and set the radix to sixteen
+ *              (hexadecimal)
+ *
+ **************************************************************************** */
+
+static int saved_base ;    /*  Place to save the numeric conversion radix  */
+
+void enter_tokz_esc( void )
+{
+    saved_base = base ;
+    base = 16;
+    in_tokz_esc = TRUE;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  end_tokz_esc
+ *      Synopsis:       When you've reached the end of "Tokenizer Escape" mode.
+ *
+ *      Associated Tokenizer directive:        ]TOKENIZER
+ *
+ *      Process Explanation:
+ *          Exit "Tokenizer Escape" mode, resume "Normal"-mode behavior.
+ *              Restore the tokenizer numeric conversion radix to the value
+ *              saved by tokenizer[ and exit "Tokenizer Escape" mode . . .
+ *
+ **************************************************************************** */
+
+static void end_tokz_esc( tic_param_t pfield )
+{
+    in_tokz_esc = FALSE;
+    base = saved_base ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  tokz_esc_emit_byte
+ *      Synopsis:       Emit the byte found on the data-stack
+ *                      
+ *      Associated Tokenizer directive:        EMIT-BYTE
+ *                      
+ *      Error Detection:
+ *          If number on stack is larger than a byte:  Truncate and Warn
+ *
+ **************************************************************************** */
+
+static void tokz_esc_emit_byte ( tic_param_t pfield )
+{
+    long num_on_stk = dpop();
+    u8 byt_to_emit = (u8)num_on_stk;
+    if ( byt_to_emit != num_on_stk )
+    {
+        tokenization_error( WARNING,
+	    "Value on stack for %s command is 0x%0x.  "
+		"Truncating to 0x%02x.\n",
+		     strupr(statbuf), num_on_stk, byt_to_emit);
+    }
+    user_emit_byte(byt_to_emit);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  get_fcode_from_stack
+ *      Synopsis:       Retrieve an FCode value from the data-stack.
+ *                          Indicate if erroneous
+ *
+ *      Inputs:
+ *         Parameters:
+ *             the_num           Pointer to where to put the number
+ *             the_action        Phrase to be used in ERROR message
+ *         Data-Stack Items:
+ *             Top:              Value to be retrieved
+ *
+ *      Outputs:
+ *         Returned Value:       TRUE if number is within valid FCode range
+ *         Supplied Pointers:
+ *             *the_num          Value retrieved from the data-stack
+ *
+ *      Error Detection:
+ *          If the number on the stack is larger than 16 bits, truncate
+ *              it, with a WARNING message.
+ *          If the (possibly truncated) number from the stack is larger
+ *              than the legal maximum for an FCode (0x0fff), or
+ *              less than the legal minimum (0x0800), issue an ERROR,
+ *              leave the_num unchanged and return FALSE.
+ *
+ **************************************************************************** */
+
+static bool get_fcode_from_stack( u16 *the_num, char *the_action)
+{
+    bool retval = FALSE;
+    long num_on_stk = dpop();
+    u16 test_fcode = (u16)num_on_stk;
+    if ( test_fcode != num_on_stk )
+    {
+        tokenization_error( WARNING,
+	    "Value on stack for %s command is 0x%0x.  "
+		"Truncating to 0x%03x.\n",
+		     strupr(statbuf), num_on_stk, test_fcode);
+    }
+    if ( ( test_fcode >= 0x800 ) && ( test_fcode <= 0xfff ) )
+    {
+	retval = TRUE;
+	*the_num = test_fcode;
+    }else{	tokenization_error( TKERROR, "Attempt to %s "
+	    "0x%x, which violates limit specified by IEEE-1275.  "
+		"Disallowing.\n", the_action, test_fcode );
+    }
+    return( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  tokz_esc_next_fcode
+ *      Synopsis:       Set the next-fcode to the value on the data-stack
+ *                      
+ *      Associated Tokenizer Directive:          next-fcode
+ *                      
+ *      Error Detection:
+ *          If the number on the stack is not legal for an FCode, as
+ *              detected by  get_fcode_from_stack(), issue an ERROR
+ *              message and don't change nextfcode.
+ *
+ *         Printout:
+ *             Advisory showing change in FCode token Assignment Counter
+ *
+ **************************************************************************** */
+
+static void tokz_esc_next_fcode( tic_param_t pfield )
+{
+    u16 test_fcode;
+
+    if ( get_fcode_from_stack( &test_fcode, "set next fcode to") )
+    {
+	if ( test_fcode == nextfcode )
+	{
+	    tokenization_error( INFO, "FCode-token Assignment Counter "
+		"is unchanged from 0x%x.\n",
+		    nextfcode);
+	}else{
+	    tokenization_error( INFO, "FCode-token Assignment Counter "
+		"was 0x%x; has been %s to 0x%x.\n",
+		    nextfcode,
+			test_fcode > nextfcode ? "advanced" : "reset",
+			    test_fcode );
+	    set_next_fcode( test_fcode);
+	}
+    }
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  tokz_emit_fcode
+ *      Synopsis:       Emit the value on the data-stack as an FCode token
+ *                      
+ *      Associated Tokenizer Directive:          emit-fcode
+ *                      
+ *      Error Detection:
+ *          If the number on the stack is not legal for an FCode, as
+ *              detected by  get_fcode_from_stack(), issue an ERROR
+ *              message and don't emit anything.
+ *
+ *         Printout:
+ *             Advisory showing FCode being emitted.
+ *
+ **************************************************************************** */
+
+static void tokz_emit_fcode( tic_param_t pfield )
+{
+    u16 test_fcode;
+
+    if ( get_fcode_from_stack( &test_fcode, "Emit FCode value of") )
+    {
+	tokenization_error( INFO,
+	    "Emitting FCode value of 0x%x\n", test_fcode);
+	emit_fcode( test_fcode);
+    }
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  zero_equals
+ *      Synopsis:       Boolean-inversion of item on top of stack.
+ *                      If zero, make it minus-one; if non-zero, make it zero. 
+ *
+ *      Associated FORTH word-name:          0=
+ *
+ **************************************************************************** */
+
+static void  zero_equals ( tic_param_t pfield )
+{
+    *dstack = (*dstack == 0) ? -1 : 0 ;
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  tokz_esc_swap
+ *      Synopsis:       "Tokenizer Escape" mode-time data-stack SWAP operation
+ *                      
+ *      Associated FORTH word-name:          swap
+ *
+ **************************************************************************** */
+
+static void  tokz_esc_swap ( tic_param_t pfield )
+{
+    swap();
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  tokz_esc_two_swap
+ *      Synopsis:       "Tokenizer Escape" mode-time data-stack 2SWAP operation
+ *                      
+ *      Associated FORTH word-name:          2swap
+ *
+ **************************************************************************** */
+
+static void  tokz_esc_two_swap ( tic_param_t pfield )
+{
+    two_swap();
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  tokz_esc_noop
+ *      Synopsis:       "Tokenizer Escape" mode-time non-operation
+ *                      
+ *      Associated FORTH word-name:          noop
+ *
+ **************************************************************************** */
+
+static void  tokz_esc_noop ( tic_param_t pfield )
+{
+    return;
+}
+
+#ifdef TOKZTEST         /*  For testing only   */
+
+   static void  tokz_esc_emit_string( tic_param_t pfield )
+   {
+      int lenny ;
+      lenny = strlen ( pfield.chr_ptr );
+      emit_token("b(")");
+      emit_string(pfield.chr_ptr, lenny);
+   }
+
+#endif                  /*  For testing only   */
+
+/* **************************************************************************
+ *
+ *      Function name:  do_constant
+ *      Synopsis:       The function to perform when a named constant
+ *                          that was defined in "Tokenizer Escape" mode
+ *                          is invoked in "Tokenizer Escape" mode
+ *
+ *      Associated FORTH word:                 A user-defined constant
+ *      
+ *      Inputs:
+ *         Parameters:
+ *             The param-field of the table-entry contains
+ *                 the value of the constant
+ *
+ *      Outputs:
+ *         Returned Value:         NONE
+ *         Items Pushed onto Data-Stack:
+ *             Top:              The table-entry's param-field's value
+ *
+ *      Error Detection:
+ *          An attempt, while operating in normal tokenization mode, to invoke
+ *              a named constant that was defined in "Tokenizer Escape" mode
+ *              will be detected by the main scanning loop, and need not
+ *              concern us here.
+ *
+ **************************************************************************** */
+
+static void do_constant ( tic_param_t pfield )
+{
+    dpush( pfield.long_val );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  create_constant
+ *      Synopsis:       Create a user-defined constant that will be
+ *                          recognized in "Tokenizer Escape" mode
+ *
+ *      Associated FORTH word-name:             CONSTANT  (when issued
+ *                                                  in "Tokenizer Escape" mode)
+ *
+ *      Inputs:
+ *         Parameters:                NONE
+ *         Global Variables:
+ *             statbuf         The constant's name will be taken from the
+ *                                 next word in the input stream.
+ *             do_constant     The function assigned to the definition
+ *             tokz_esc_vocab  The "Tokenizer Escape" Vocabulary pointer
+ *         Data-Stack Items:
+ *             Top:            The constant's value is popped from the stack
+ *
+ *      Outputs:
+ *         Returned Value:            NONE
+ *         Global Variables:
+ *             statbuf         Advanced to the next word in the input stream.
+ *             tokz_esc_vocab      Updated to point to new vocab entry.
+ *         Memory Allocated:
+ *             for the name and for the new entry.
+ *         When Freed?
+ *             When RESET-SYMBOLS is issued in "Tokenizer Escape" mode,
+ *                or upon end of tokenization.
+ *         Data-Stack, # of Items Popped:             1
+ *
+ *      Error Detection:
+ *          Failure to allocate memory is a Fatal Error.
+ *          Warning on excessively long name
+ *          Warning on duplicate name
+ *          Name to be defined not in same file, ERROR
+ *
+ *      Process Explanation:
+ *          Get the next word, STRDUP it (which implicitly allocates memory). 
+ *              Get the number popped off the stack.
+ *              Pass the pointer and the value to the add_tic_entry() routine.
+ *
+ **************************************************************************** */
+
+static void create_constant( tic_param_t pfield )
+{
+    char *c_name_space ;          /*  Space for copy of the name    */
+    long valu ;                   /*  Value, popped off the stack   */
+    signed long wlen;
+
+    /*  Get the name that is to be defined  */
+    wlen = get_word();
+    if ( wlen <= 0 )
+    {
+	warn_unterm( TKERROR, "Constant definition", lineno);
+	return;
+    }
+
+    valu = dpop();
+
+    /*  If ever we implement more than just this one
+     *      defining-word in "Tokenizer Escape" mode,
+     *      the lines from here to the end of the
+     *      routine should be re-factored...
+     */
+    trace_creation( CONST, statbuf);
+
+    warn_if_duplicate( statbuf );
+    check_name_length( wlen );
+
+    c_name_space = strdup( statbuf );
+
+    add_tic_entry(
+	 c_name_space,
+	     do_constant,
+		  (TIC_P_DEFLT_TYPE)valu,
+		       CONST ,
+			   0 , NULL,
+		               &tokz_esc_vocab );
+
+}
+
+/* **************************************************************************
+ *
+ *      Let's create usable named constants for "Tokenizer Escape" mode.
+ *          It's useful, it's easy and ...  well, you get the idea.
+ *
+ **************************************************************************** */
+/*  I don't think we need to any more
+static const int zero = 0 ;
+static const int minus_one = -1 ;
+static const char double_quote = '"' ;
+static const char close_paren = ')' ;
+ *  Except for this next one, of course...   */
+#ifdef TOKZTEST        /*  For testing only   */
+   static const char date_me[] =  DATE_STAMP;
+#endif                 /*  For testing only   */
+
+/* **************************************************************************
+ *
+ *      Here, at long last, we define and initialize the structure containing
+ *          all the functions we support in "Tokenizer Escape" mode.
+ *      Let's call it the "Tokenizer Escape Vocabulary Table" and the
+ *          pointer to it, the "Tokenizer Escape" Vocabulary.
+ *      We can initialize the start of the "Tokenizer Escape" Vocabulary
+ *          easily, except for the link-pointers, as an array.
+ *
+ **************************************************************************** */
+
+#define TKZESC_CONST(nam, pval)   \
+                        VALPARAM_TIC(nam, do_constant, pval, CONST )
+#define TKZ_ESC_FUNC(nam, afunc, pval, ifunc)   \
+                        DUALFUNC_TIC(nam, afunc, pval, ifunc, UNSPECIFIED)
+
+static tic_hdr_t tokz_esc_vocab_tbl[] = {
+    NO_PARAM_IGN( "]tokenizer" , end_tokz_esc                           ) ,
+
+    /*  An IBM-ish synonym.  */
+    NO_PARAM_IGN( "f]"         , end_tokz_esc                           ) ,
+    /*  An alternate synonym.  */
+    NO_PARAM_IGN( "]f"         , end_tokz_esc                           ) ,
+
+    NO_PARAM_TIC( "emit-byte"  , tokz_esc_emit_byte                     ) ,
+    NO_PARAM_TIC( "next-fcode" , tokz_esc_next_fcode                    ) ,
+    NO_PARAM_TIC( "emit-fcode" , tokz_emit_fcode                        ) ,
+    NO_PARAM_TIC( "constant"   , create_constant                        ) ,
+    NO_PARAM_TIC( "0="         , zero_equals                            ) ,
+    NO_PARAM_TIC( "swap"       , tokz_esc_swap                          ) ,
+    NO_PARAM_TIC( "2swap"      , tokz_esc_two_swap                      ) ,
+    NO_PARAM_TIC( "noop"       , tokz_esc_noop                          ) ,
+    TKZESC_CONST( "false"      ,  0                                     ) ,
+    TKZESC_CONST( "true"       , -1                                     ) ,
+    TKZ_ESC_FUNC( ".("         , user_message, ')', skip_user_message   ) ,
+    TKZ_ESC_FUNC( ".""        , user_message, '"', skip_user_message   ) ,
+#ifdef TOKZTEST        /*  For testing only   */
+    /*  Data is a pointer to a constant string in the compiler;    */
+    /*      no need to copy, hence data_size can remain zero.      */
+    /*  We could almost use the Macro macro, except for the type.  */
+    TKZ_ESC_FUNC( "emit-date"  , tokz_esc_emit_string, date_me , NULL   ) ,
+#endif                 /*  For testing only   */
+};
+
+/* **************************************************************************
+ *
+ *      Also, keep a pointer to the "Built-In" position of
+ *          the "Tokenizer Escape" Vocabulary
+ *
+ **************************************************************************** */
+
+static const tic_hdr_t *built_in_tokz_esc =
+    &tokz_esc_vocab_tbl[(sizeof(tokz_esc_vocab_tbl)/sizeof(tic_hdr_t))-1];
+
+/* **************************************************************************
+ *
+ *      Function name:  init_tokz_esc_vocab
+ *      Synopsis:       Initialize the "Tokenizer Escape" Vocabulary
+ *                          link-pointers dynamically.
+ *
+ **************************************************************************** */
+
+void init_tokz_esc_vocab ( void )
+{
+    static const int tokz_esc_vocab_max_indx =
+	 sizeof(tokz_esc_vocab_tbl)/sizeof(tic_hdr_t) ;
+
+    tokz_esc_vocab = NULL ;   /*  Belt-and-suspenders...  */
+    init_tic_vocab(tokz_esc_vocab_tbl,
+                       tokz_esc_vocab_max_indx,
+		           &tokz_esc_vocab );
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  handle_tokz_esc
+ *      Synopsis:       Perform a function in the "Tokenizer Escape" Vocabulary
+ *                      Indicate whether the name is valid in this vocabulary.
+ *                      Handle "Tokenizer Escape" aliases implicitly.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             tname                The name to handle
+ *         Global Variables:  
+ *             tokz_esc_vocab       Pointer to "Tokenizer Escape" Vocabulary 
+ *
+ *      Outputs:
+ *         Returned Value:   TRUE if the given name is valid in tokz_esc
+ *
+ *      Process Explanation:
+ *          Find the name and execute its associated function.
+ *          If the name is not in the "Tokenizer Escape" Vocabulary,
+ *              let the calling routine determine whether to try it
+ *              out as a number or to print an error message.
+ *
+ **************************************************************************** */
+
+bool handle_tokz_esc( char *tname )
+{
+    bool retval = handle_tic_vocab( tname, tokz_esc_vocab );
+    return ( retval ) ;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_tokz_esc
+ *      Synopsis:       Return a pointer to the data-structure of the named
+ *                      word in the"Tokenizer Escape" Vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             name                 The given name for which to look
+ *         Local Static Variables:
+ *             tokz_esc_vocab       Pointer to "Tokenizer Escape" Vocabulary  
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE if name is found,
+ *
+ **************************************************************************** */
+
+tic_hdr_t *lookup_tokz_esc(char *name)
+{
+    tic_hdr_t *retval = lookup_tic_entry( name, tokz_esc_vocab );
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_in_tokz_esc
+ *      Synopsis:       Confirm whether a given name exists in the
+ *                          "Tokenizer Escape" Vocabulary
+ *
+ *      Inputs:
+ *         Parameters:
+ *             name                 The given name to confirm
+ *         Global Variables:  
+ *             tokz_esc_vocab       Pointer to "Tokenizer Escape" Vocabulary  
+ *
+ *      Outputs:
+ *         Returned Value:     TRUE if name is found,
+ *
+ **************************************************************************** */
+
+bool exists_in_tokz_esc(char *name)
+{
+    bool retval = exists_in_tic_vocab( name, tokz_esc_vocab );
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  create_tokz_esc_alias
+ *      Synopsis:       Create an alias in the "Tokenizer Escape" Vocabulary
+ *
+ *      Associated FORTH word:              ALIAS (in "Tokenizer Escape" mode)
+ *
+ *      Inputs:
+ *         Parameters:
+ *             old_name             Name of existing entry
+ *             new_name             New name for which to create an entry
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if  old_name  found in "Tok Esc" vocab
+ *         Global Variables:    
+ *             tokz_esc_vocab       Will point to the new entry
+ *         Memory Allocated:
+ *             Memory for the new entry will be allocated
+ *                 by the support routine
+ *         When Freed?
+ *             When RESET-SYMBOLS is issued in "Tokenizer Escape" mode,
+ *                or upon end of tokenization.
+ *
+ **************************************************************************** */
+
+bool create_tokz_esc_alias(char *new_name, char *old_name)
+{
+    bool retval = create_tic_alias( new_name, old_name, &tokz_esc_vocab );
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  reset_tokz_esc
+ *      Synopsis:       Reset the "Tokenizer Escape" Vocabulary to
+ *                          its "Built-In" position.
+ *
+ *      Associated Tokenizer directive:       RESET-SYMBOLS  (when issued
+ *                                                in "Tokenizer Escape" mode)
+ *
+ *      Inputs:
+ *         Parameters:                 NONE
+ *         Global Variables:
+ *             tokz_esc_vocab      Pointer to "Tokenizer Escape" Vocabulary
+ *             built_in_tokz_esc   Pointer to "Built-In" position
+ *
+ *      Outputs:
+ *         Returned Value:             NONE
+ *         Global Variables:
+ *             tokz_esc_vocab      Reset to "Built-In" position
+ *         Memory Freed
+ *             Memory will be freed by the support routine
+ *
+ **************************************************************************** */
+
+void reset_tokz_esc( void )
+{
+    reset_tic_vocab( &tokz_esc_vocab, (tic_hdr_t *)built_in_tokz_esc);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  pop_next_fcode
+ *      Synopsis:       Vector off to the  tokz_esc_next_fcode  function,
+ *                      but without the pseudo-param.  A retro-fit...
+ *
+ *      Associated Tokenizer directive:   FCODE-POP  (issued in either mode)
+ *
+ *      Inputs:
+ *         Parameters:                    NONE
+ *         Data-Stack Items:
+ *             Top:                       Next FCode value, presumably saved
+ *                                            by an  FCODE-PUSH  issued earlier.
+ *
+ *      Outputs:
+ *         Returned Value: 
+ *         Global Variables:
+ *             nextfcode                  FCode token Assignment Counter
+ *
+ **************************************************************************** */
+
+void pop_next_fcode( void)
+{
+   tic_param_t dummy_param;
+   tokz_esc_next_fcode( dummy_param);
+}

Added: fcode-utils/toke/tokzesc.h
===================================================================
--- fcode-utils/toke/tokzesc.h	                        (rev 0)
+++ fcode-utils/toke/tokzesc.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,50 @@
+#ifndef _TOKE_TOKZESC_H
+#define _TOKE_TOKZESC_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      External and Prototype definitions for functions that process
+ *          activity in "Tokenizer Escape" mode
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "types.h"
+#include "ticvocab.h"
+
+void init_tokz_esc_vocab( void );
+bool create_tokz_esc_alias(char *new_name, char *old_name);
+void enter_tokz_esc( void );
+bool handle_tokz_esc( char *tname );
+tic_hdr_t *lookup_tokz_esc(char *name);
+bool exists_in_tokz_esc(char *name);
+void reset_tokz_esc( void );
+void pop_next_fcode( void);
+
+#endif   /*  _TOKE_TOKZESC_H    */

Added: fcode-utils/toke/tracesyms.c
===================================================================
--- fcode-utils/toke/tracesyms.c	                        (rev 0)
+++ fcode-utils/toke/tracesyms.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,172 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Support routines for "Trace-Symbols" debugging feature
+ *
+ *      (C) Copyright 2006 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          add_to_trace_list         Add the given name to the Trace List
+ *          show_initial_traces       Show pre-defined names the user
+ *                                        asked to Trace (if any)
+ *          is_on_trace_list          Indicate whether the given name is
+ *                                        on the Trace List
+ *
+ **************************************************************************** */
+
+#include <string.h>
+
+#include "tracesyms.h"
+#include "errhandler.h"
+
+
+/* **************************************************************************
+ *
+ *          Internal Static Variables
+ *     trace_list     Pointer to last entry in the Trace List linked-list
+ *                        data structure
+ *              
+ *              
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Internal (Static) Structures:
+ *          trace_entry_t           Linked-list of entries in the Trace List
+ *
+ *   Fields:
+ *       tracee              Name of the symbol to be traced
+ *       prev                Pointer to previous entry in the linked-list
+ *
+ **************************************************************************** */
+
+typedef struct trace_entry {
+      char *tracee;
+      struct trace_entry *prev;
+} trace_entry_t;
+
+static trace_entry_t *trace_list = NULL;
+
+
+
+/* **************************************************************************
+ *
+ *      Function name:  add_to_trace_list
+ *      Synopsis:       Add the given name to the Trace List
+ *                      
+ *
+ *      Inputs:
+ *         Parameters:
+ *             trace_symb            Name of the symbol to be added
+ *         Local Static Variables:
+ *             trace_list           Pointer to the Trace List
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Local Static Variables:
+ *             trace_list           Points to new entry in Trace List
+ *         Memory Allocated
+ *             For Trace List entry
+ *             For copy of Symbol Name
+ *         When Freed?
+ *             Never.  Well, only on termination of the program.  Trace-list
+ *                 endures for the entire batch of tokenizations.
+ *
+ *      Error Detection:
+ *          Memory allocation failure is a FATAL error.
+ *
+ **************************************************************************** */
+
+void add_to_trace_list( char *trace_symb)
+{
+    trace_entry_t *new_t_l_entry = safe_malloc( sizeof( trace_entry_t),
+        "adding to trace-list");
+    new_t_l_entry->tracee = strdup( trace_symb);
+    new_t_l_entry->prev = trace_list;
+
+    trace_list = new_t_l_entry;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  is_on_trace_list
+ *      Synopsis:       Indicate whether the given name is on the Trace List
+ *
+ *      Inputs:
+ *         Parameters:
+ *             symb_name            Symbol-name to test
+ *         Local Static Variables:
+ *             trace_list           Pointer to the Trace List
+ *
+ *      Outputs:
+ *         Returned Value:          TRUE if Symbol-name is on the Trace List
+ *
+ **************************************************************************** */
+
+bool is_on_trace_list( char *symb_name)
+{
+    bool retval = FALSE;
+    trace_entry_t *test_entry = trace_list;
+    while ( test_entry != NULL )
+    {
+        if ( strcasecmp( symb_name, test_entry->tracee) == 0 )
+	{
+	    retval = TRUE;
+	    break;
+	}
+	test_entry = test_entry->prev;
+    }
+    return ( retval );
+}
+
+
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Implement a function -- name it  show_initial_traces  --
+ *              that will show any pre-defined names the user asked
+ *              to Trace.  That is, if any of the names the user asked
+ *              to Trace belongs to a pre-defined function, macro or
+ *              directive, then, at the beginning of the output, issue 
+ *              Advisory Messages identifying the scope of those names.
+ * 
+ *          E.g, if the user had  -T 3DUP  -T SWAP   the function would
+ *              issue Messages like:
+ *          3DUP is pre-defined as a Macro with Global scope
+ *          SWAP is pre-defined with Global scope
+ *          SWAP is pre-defined in Tokenizer-Escape mode
+ * 
+ *          The names would, of course, remain on the Trace List and
+ *              any re-definitions of them would be reported.
+ *
+ **************************************************************************** */

Added: fcode-utils/toke/tracesyms.h
===================================================================
--- fcode-utils/toke/tracesyms.h	                        (rev 0)
+++ fcode-utils/toke/tracesyms.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,48 @@
+#ifndef _TOKE_TRACESYMS_H
+#define _TOKE_TRACESYMS_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      External/Prototype/Structure definitions for "Trace-Symbols" feature
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+#include "types.h"
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+void add_to_trace_list( char *trace_symb);
+bool is_on_trace_list( char *symb_name);
+
+#endif   /*  _TOKE_TRACESYMS_H    */

Added: fcode-utils/toke/usersymbols.c
===================================================================
--- fcode-utils/toke/usersymbols.c	                        (rev 0)
+++ fcode-utils/toke/usersymbols.c	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,363 @@
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      General-purpose support functions for
+ *          User-defined command-line compilation-control symbols
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      The syntax for user-defined command-line compilation-control symbols
+ *          is <NAME>[=<VALUE>]
+ *
+ *      The name is always required; the equal-sign and value is optional.
+ *          If you wish the "value" to contain spaces or quotes, you can
+ *          accomplish that using the shell escape conventions.
+ *
+ *      The operations that can be performed upon these symbols will be
+ *          described by the operators that use them as operands, but,
+ *          broadly speaking, the tests will either be to simply verify
+ *          the existence of a symbol, or to evaluate the defined value.
+ *
+ *      Once a symbol is defined on the command-line, it stays in effect
+ *          for the duration of the entire batch of tokenizations (i.e.,
+ *          if there are multiple input files named on the command line).
+ *          Also, there are no symbols defined at the outset.  Therefore,
+ *          there is no need for either an "init" or a "reset" routine.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      User-defined command-line compilation-control symbols are
+ *          implemented as a String-Substitution-type vocabulary.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Functions Exported:
+ *          add_user_symbol            Add a user-defined symbol to the list
+ *          lookup_user_symbol         Look for a user-defined symbol, return
+ *                                         the assigned value.
+ *          exists_as_user_symbol      Confirm whether a given name exists
+ *                                         as a user-defined symbol.
+ *          eval_user_symbol           Tokenize the value assigned to a user
+ *                                         symbol.
+ *          list_user_symbols          Print the list of user-defined symbols
+ *                                         for the Logfile.
+ *
+ **************************************************************************** */
+
+/* **************************************************************************
+ *
+ *      Still to be done:
+ *          Convert the handling of user-defined symbols to the T.I.C.
+ *              data-structure and its support routines.  This should
+ *              eliminate any further need of String-Substitution-type
+ *              vocabularies.  User-defined symbols will, however, still
+ *              need to be a separate vocabulary from the Global, because
+ *              they are required to stay in effect for the duration of
+ *              the entire batch of tokenizations...
+ *          (Afterthought:  This is only true for user-defined symbols that
+ *              were created on the command-line; if we ever allow symbols
+ *              to be defined in the Source file, they should be as volatile
+ *              as anything else that comes from a source file...
+ *           Putting source-file-derived user-defined symbols into the Global
+ *              Vocabulary could be a quasi-simple way to accomplish this.)
+ *
+ *          Enable the definition of user-symbols from the Source file, using
+ *              a syntax like:  [define] symbol   or  [define] symbol=<value>
+ *              (How to allow spaces into the <value>?  Maybe make the syntax
+ *              [define] symbol = <value components to end of line> 
+ *              delimited in a manner similar to Macro definitions.
+ *          There might be a need to be able to  [undefine]  a user-symbol
+ *              that would entail defining an  unlink_tic_entry  function.
+ *              Not difficult; just keeping this around as a reminder...
+ *
+ **************************************************************************** */
+
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#if defined(__linux__) && ! defined(__USE_BSD)
+#define __USE_BSD
+#endif
+#include <string.h>
+
+#include "errhandler.h"
+#include "strsubvocab.h"
+#include "usersymbols.h"
+#include "scanner.h"
+
+
+/* **************************************************************************
+ *
+ *              Internal Static Variables
+ *      user_symbol_list          Pointer to the "tail" of the list of
+ *                                    user-defined symbols.
+ *      user_symbol_count         Count of how many are defined
+ *
+ **************************************************************************** */
+
+static str_sub_vocab_t *user_symbol_list = NULL;
+static int user_symbol_count = 0;
+
+/* **************************************************************************
+ *
+ *      Function name:  add_user_symbol
+ *      Synopsis:       Add a user-defined symbol to the list
+ *
+ *      Inputs:
+ *         Parameters:
+ *             raw_symb             The string as supplied on the command-line.
+ *         Local Static Variables:
+ *             user_symbol_list     Pointer to the list of user-defined symbols.
+ *
+ *      Outputs:
+ *         Returned Value:                NONE
+ *         Local Static Variables:
+ *             user_symbol_list     Will be updated.
+ *             user_symbol_count    Will be incremented
+ *         Memory Allocated:
+ *             for the string(s) and the new entry
+ *         When Freed?
+ *             Never.  Well, only on termination of the program.  User-defined
+ *                 symbols endure for the entire batch of tokenizations.
+ *
+ *      Process Explanation:
+ *          The string in  raw_symb  may or may not include the optional
+ *              equal-sign and value pair.  If the equal-sign is present,
+ *              the remainder of the string will become the "value" that
+ *              will be returned by the "lookup" routine.
+ *          Memory for the name string and for the value, if there is one,
+ *              will be allocated here, in one step.  Memory for the data
+ *              structure itself will be allocated by the support routine.
+ *
+ **************************************************************************** */
+
+void add_user_symbol(char *raw_symb)
+{
+    char *symb_nam;
+    char *symb_valu;
+
+    symb_nam = strdup(raw_symb);
+    symb_valu = strchr(symb_nam,'=');
+    if ( symb_valu != NULL )
+    {
+	*symb_valu = 0;
+	symb_valu++;
+    }
+    add_str_sub_entry(symb_nam, symb_valu, &user_symbol_list );
+    user_symbol_count++;
+}
+
+
+/* **************************************************************************
+ *
+ *      Function name:  lookup_user_symbol
+ *      Synopsis:       Look for the given name as user-defined symbol, return
+ *                          the assigned value.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             symb_nam             The name for which to look.
+ *         Local Static Variables:
+ *             user_symbol_list     Pointer to the list of user-defined symbols.
+ *
+ *      Outputs:
+ *         Returned Value:          Pointer to the "value" string, or NULL
+ *                                      pointer if the name was not found.
+ *                                  May also be NULL if "value" is NULL.
+ *
+ **************************************************************************** */
+ 
+char *lookup_user_symbol(char *symb_nam)
+{
+    char *symb_valu;
+
+    symb_valu = lookup_str_sub(symb_nam, user_symbol_list );
+    return (symb_valu);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  exists_as_user_symbol
+ *      Synopsis:       Confirm whether a given name exists
+ *                      as a user-defined symbol.
+ *
+ *      Inputs:
+ *         Parameters:
+ *             symb_nam             The name for which to look.
+ *         Local Static Variables:
+ *             user_symbol_list     Pointer to the list of user-defined symbols.
+ *
+ *      Outputs:
+ *         Returned Value:      TRUE if the name is found
+ *
+ **************************************************************************** */
+
+bool exists_as_user_symbol(char *symb_nam)
+{
+    bool retval;
+
+    retval = exists_in_str_sub(symb_nam, user_symbol_list );
+    return (retval);
+}
+
+/* **************************************************************************
+ *
+ *      Function name:  eval_user_symbol
+ *      Synopsis:       Tokenize the value assigned to a user-symbol.
+ *
+ *      Associated Tokenizer directive (synonyms):      [DEFINED]
+ *                                                      #DEFINED
+ *                                                      [#DEFINED]
+ *
+ *      Syntax Notes:
+ *          (1)  The User-Defined-Symbol must appear
+ *                   on the same line as the directive.
+ *          (2)  This is not (yet) implemented in contexts that
+ *                   directly read input from the stream, e.g.,
+ *                   after  [']  or after  H#  etc.
+ *
+ *      Inputs:
+ *         Parameters:       
+ *             symbol               The User-Defined-Symbol to evaluate
+ *         Local Static Variables:
+ *             user_symbol_list     Pointer to the list of user-defined symbols.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         The assigned value will be tokenized.
+ *
+ *      Error Detection:
+ *          Calling routine is responsible for verifying that the user-symbol
+ *              is on the same line as the directive.
+ *          WARNING if the symbol is not found or has no assigned value. 
+ *
+ *      Process Explanation:
+ *          Look up the parameter in the User Symbol List, and retrieve
+ *              its associated value.
+ *          If it is not found, or if it has no associated value, issue
+ *              a WARNING and do nothing further.  Otherwise...
+ *          Interpret the associated value as though it were source.
+ *
+ *      Still to be done:
+ *          Hook-in this routine to the processing of:  [']  F[']  H#  FLOAD
+ *              etc., and wherever else it might be needed or useful.
+ *
+ **************************************************************************** */
+
+void eval_user_symbol(char *symbol )
+{
+    char *symb_valu;
+    symb_valu = lookup_user_symbol(symbol );
+    if ( symb_valu == NULL )
+    {
+        tokenization_error ( WARNING,
+	    "No value assigned to command-line symbol %s\n", symbol );
+    }else{
+	eval_string( symb_valu );
+    }
+
+}
+/* **************************************************************************
+ *
+ *      Function name:  list_user_symbols
+ *      Synopsis:       Print the list of user symbols for the Logfile.
+ *      
+ *      Inputs:
+ *         Parameters:              NONE
+ *         Local Static Variables:
+ *             user_symbol_list     Pointer to the list of user-defined symbols.
+ *             user_symbol_count    Count of user-defined symbols.
+ *
+ *      Outputs:
+ *         Returned Value:          NONE
+ *         Printout:                List of user symbols and their definitions;
+ *                                      nothing if user_symbol_list is NULL.
+ *
+ *      Process Explanation:
+ *          We want to display the symbols in the same order they were created.
+ *          We will:
+ *              Allocate a temporary array of pointers.
+ *              Step backwards through the linked-list of symbols, and
+ *                  enter their pointers into the array.
+ *              Collect the maximum length of the symbol names.
+ *              Step through the array in the reverse order, printing
+ *                  as we go.
+ *                  Use the max name length to space the equal-signs evenly 
+ *              Free the temporary array.
+ *
+ **************************************************************************** */
+
+void list_user_symbols(void )
+{
+    str_sub_vocab_t *curr;
+
+    if ( user_symbol_list != NULL )
+    {
+	/*  Collect the pointers and max length  */
+	str_sub_vocab_t **symb_ptr;
+	int indx = 0;
+	int maxlen = 0;
+	
+	symb_ptr = (str_sub_vocab_t **)safe_malloc(
+	   (sizeof(str_sub_vocab_t *) * user_symbol_count),
+	       "collecting user-symbol pointers" );
+	
+	for (curr = user_symbol_list ; curr != NULL ; curr=curr->next)
+	{
+            symb_ptr[indx] = curr;
+	    indx++;
+	    if ( strlen(curr->name) > maxlen ) maxlen = strlen(curr->name);
+	}
+	
+	/*  Now print 'em out  */
+	printf("\nUser-Defined Symbols:\n");
+	while ( indx > 0 )
+	{
+	    indx--;
+	    curr = symb_ptr[indx];
+	    printf("\t%s",curr->name);
+	    if ( curr->alias != NULL )
+	    {
+	        int strindx;
+		for ( strindx = strlen(curr->name) ;
+		      strindx < maxlen ; strindx++ ) printf(" ");
+		printf(" = %s",curr->alias);
+	    }
+	    printf("\n");
+	}
+	free(symb_ptr);
+    }
+}

Added: fcode-utils/toke/usersymbols.h
===================================================================
--- fcode-utils/toke/usersymbols.h	                        (rev 0)
+++ fcode-utils/toke/usersymbols.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,46 @@
+#ifndef _TOKE_USERSYMBOLS_H
+#define _TOKE_USERSYMBOLS_H
+
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      Prototype/External Declarations for general-purpose functions
+ *          for support of User-defined command-line compilation-control
+ *          symbols, as defined in  usersymbols.c
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+void add_user_symbol(char *raw_symb);
+char *lookup_user_symbol(char *symb_nam);
+bool exists_as_user_symbol(char *symb_nam);
+void eval_user_symbol(char *symbol );
+void list_user_symbols(void );
+
+#endif   /* _TOKE_USERSYMBOLS_H    */

Added: fcode-utils/toke/vocabfuncts.h
===================================================================
--- fcode-utils/toke/vocabfuncts.h	                        (rev 0)
+++ fcode-utils/toke/vocabfuncts.h	2006-08-18 12:47:12 UTC (rev 76)
@@ -0,0 +1,92 @@
+#ifndef _TOKE_VOCABFUNCTS_H
+#define _TOKE_VOCABFUNCTS_H
+
+/*
+ *                     OpenBIOS - free your system!
+ *                         ( FCode tokenizer )
+ *
+ *  This program is part of a free implementation of the IEEE 1275-1994
+ *  Standard for Boot (Initialization Configuration) Firmware.
+ *
+ *  Copyright (C) 2001-2005 Stefan Reinauer, 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA, 02110-1301 USA
+ *
+ */
+
+/* **************************************************************************
+ *
+ *      External/Prototype definitions for Vocabulary functions
+ *           in dictionary.c for Tokenizer
+ *
+ *      (C) Copyright 2005 IBM Corporation.  All Rights Reserved.
+ *      Module Author:  David L. Paktor    [e-mail]
+ *
+ **************************************************************************** */
+
+
+#include "types.h"
+#include "ticvocab.h"
+
+
+/* ************************************************************************** *
+ *
+ *      Global Variables Exported
+ *
+ **************************************************************************** */
+
+extern bool scope_is_global;
+
+/* ************************************************************************** *
+ *
+ *      Function Prototypes / Functions Exported:
+ *
+ **************************************************************************** */
+
+
+tic_hdr_t *lookup_core_word( char *tname);
+bool exists_in_core( char *name);
+bool handle_core_word( char *tname );
+bool create_core_alias( char *new_name, char *old_name);
+
+void enter_global_scope( void );
+void resume_device_scope( void );
+
+tic_hdr_t *lookup_current( char *name);
+bool exists_in_current( char *tname);
+bool handle_current( char *tname );
+tic_hdr_t *lookup_in_dev_node( char *tname);
+void add_to_current( char *name,
+                           TIC_P_DEFLT_TYPE fc_token,
+			       fwtoken definer,
+			           bool define_token);
+void hide_last_colon ( void );
+void reveal_last_colon ( void );
+bool create_current_alias( char *new_name, char *old_name );
+
+void emit_token( const char *fc_name);
+tic_hdr_t *lookup_token( char *tname);
+bool entry_is_token( tic_hdr_t *test_entry );
+void token_entry_warning( tic_hdr_t *t_entry);
+
+tic_hdr_t *lookup_shared_word( char *tname);
+bool handle_shared_word( char *tname );
+tic_hdr_t *lookup_shared_f_exec_word( char *tname);
+
+void init_dictionary( void );
+void reset_normal_vocabs( void );
+void reset_vocabs( void );
+
+
+#endif   /*  _TOKE_VOCABFUNCTS_H    */