:::::::::::::::::::::::::: token.h ::::::::::::::::::::::::::::::
/* tokens are:     *  ++  +  <word>
    \ escapes (removes meaning of) special characters * +
    whitespace is not part of any token and forces end of token
    maximal munch
*/  
#define T_PRODUCT    1	
#define T_PLUSPLUS   2
#define T_PLUS	     3
#define T_WORD       4
#define T_EOF	     5
extern int gettoken(char *word);  /* return string <word> in location "word" */
                                  /* calling function must allocate space    */
:::::::::::::::::::::::::: token.c ::::::::::::::::::::::::::::::
/*          start
              v               +
    ----> S_NEUTRAL  -------------------------------->  S_PLUS
    |     |   |      \                                       |
    -------   |       \       *  EOF                         |
  whitespace  |others  ------------------------------        |
              |                                      \       |
              V               *  +  whitespace EOF    \      V
           S_WORD ---------------------------------->  return with token
           ^  |
           |  |others
           ----
*/
#include <stdio.h>
#include "token.h"

/* states */
#define S_NEUTRAL 1
#define S_WORD    2
#define S_PLUS    3

int gettoken(char *word)
{
    int c;
    char *partword;
    int state = S_NEUTRAL;
    
    partword = word;
    for(;;) /* infinite loop */
    {	c = getchar();
	switch (state)
	{ case S_NEUTRAL:
	    switch (c)
	    { case '+':
		state = S_PLUS;
		break;		/* 'continue' could be used instead */
	      case '*':
		return(T_PRODUCT);
              case ' ':
	      case '\n':
              case '\t':
		break;
	      case EOF:
		return(T_EOF);
	      default:
		state = S_WORD;
		ungetc(c,stdin); 
                /* process all characters including initial one
                   in S_WORD so that initial \ handled right 
                */
		break;		/* not really necessary */
	    }
	    break;		/* essential! */
	  case S_PLUS:
	    switch (c)
	    { case '+':
		return(T_PLUSPLUS);
	      case EOF:
		return(T_PLUS);
	      default:
		ungetc(c,stdin);
		return(T_PLUS);
	    }
	    break;
	  case S_WORD:
	    switch (c)
	    { case '\\':  /*  handles escape character \  */
		{   int ctemp;
		    if((ctemp = getchar()) != EOF)
			*partword++ = ctemp;  /* put next char in partword */
		    else  /* \EOF     Ignore the \ and return */
		    {     /* return EOF if have no word so far */  
			if (partword == word) return(T_EOF);
			*partword = '\0';
                        return(T_WORD);
		    }
		}
		break;
	      case '*':
	      case '+':
		ungetc(c,stdin);
	      case ' ':
	      case '\n':
              case '\t':
	      case EOF:
		*partword = '\0';
		return(T_WORD);
	      default:
		*partword++ = c;
		break;
	    }
	    break;
	}
    }
}
:::::::::::::::::::::::::: driver.c ::::::::::::::::::::::::::::::
/* driver for token */
#include <stdio.h>
#include "token.h"

int main()
   {
    char word[1024] = "";
    int tokenint;

    do {tokenint = gettoken(word);
        printf("Token %d  Word :%s:\n",tokenint,word);
        word[0] = '\0';
       } while (tokenint != T_EOF);
    return 0;
   }
:::::::::::::::::::::::::: Makefile ::::::::::::::::::::::::::::::
OBJS=driver.o token.o
driver: $(OBJS)                    # tab is at beginning of next line
	$(CC) $(CFLAGS) $(OBJS) -o driver
$(OBJS): token.h