com.sun.labs.minion.document.tokenizer
Class JCCTokenizerTokenManager

java.lang.Object
  extended by com.sun.labs.minion.document.tokenizer.JCCTokenizerTokenManager
All Implemented Interfaces:
JCCTokenizerConstants

public class JCCTokenizerTokenManager
extends java.lang.Object
implements JCCTokenizerConstants


Field Summary
protected  char curChar
           
 java.io.PrintStream debugStream
           
protected  SimpleCharStream input_stream
           
static java.lang.String[] jjstrLiteralImages
           
static java.lang.String[] lexStateNames
           
 
Fields inherited from interface com.sun.labs.minion.document.tokenizer.JCCTokenizerConstants
DEFAULT, EOF, NGRAMTOKEN, NONSPACESEPCHAR, PUNCTUATION, SPACESEPCHAR, SPACESEPCHAR1, SPACESEPCHAR2, SPACESEPCHAR3, SPACESEPCHAR4, SPACESEPCHAR5, SPACESEPCHAR6, SPACESEPCHAR7, SPACESEPCHAR8, SPACESEPCHAR9, SPACESEPTOKEN, tokenImage, WHITECHAR, WHITESPACE
 
Constructor Summary
JCCTokenizerTokenManager(SimpleCharStream stream)
           
JCCTokenizerTokenManager(SimpleCharStream stream, int lexState)
           
 
Method Summary
 Token getNextToken()
           
protected  Token jjFillToken()
           
 void ReInit(SimpleCharStream stream)
           
 void ReInit(SimpleCharStream stream, int lexState)
           
 void setDebugStream(java.io.PrintStream ds)
           
 void SwitchTo(int lexState)
           
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

debugStream

public java.io.PrintStream debugStream

jjstrLiteralImages

public static final java.lang.String[] jjstrLiteralImages

lexStateNames

public static final java.lang.String[] lexStateNames

input_stream

protected SimpleCharStream input_stream

curChar

protected char curChar
Constructor Detail

JCCTokenizerTokenManager

public JCCTokenizerTokenManager(SimpleCharStream stream)

JCCTokenizerTokenManager

public JCCTokenizerTokenManager(SimpleCharStream stream,
                                int lexState)
Method Detail

setDebugStream

public void setDebugStream(java.io.PrintStream ds)

ReInit

public void ReInit(SimpleCharStream stream)

ReInit

public void ReInit(SimpleCharStream stream,
                   int lexState)

SwitchTo

public void SwitchTo(int lexState)

jjFillToken

protected Token jjFillToken()

getNextToken

public Token getNextToken()