com.sun.labs.minion.document.tokenizer
Class JCCTokenizerTokenManager
java.lang.Object
com.sun.labs.minion.document.tokenizer.JCCTokenizerTokenManager
- All Implemented Interfaces:
- JCCTokenizerConstants
public class JCCTokenizerTokenManager
- extends java.lang.Object
- implements JCCTokenizerConstants
Fields inherited from interface com.sun.labs.minion.document.tokenizer.JCCTokenizerConstants |
DEFAULT, EOF, NGRAMTOKEN, NONSPACESEPCHAR, PUNCTUATION, SPACESEPCHAR, SPACESEPCHAR1, SPACESEPCHAR2, SPACESEPCHAR3, SPACESEPCHAR4, SPACESEPCHAR5, SPACESEPCHAR6, SPACESEPCHAR7, SPACESEPCHAR8, SPACESEPCHAR9, SPACESEPTOKEN, tokenImage, WHITECHAR, WHITESPACE |
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
debugStream
public java.io.PrintStream debugStream
jjstrLiteralImages
public static final java.lang.String[] jjstrLiteralImages
lexStateNames
public static final java.lang.String[] lexStateNames
input_stream
protected SimpleCharStream input_stream
curChar
protected char curChar
JCCTokenizerTokenManager
public JCCTokenizerTokenManager(SimpleCharStream stream)
JCCTokenizerTokenManager
public JCCTokenizerTokenManager(SimpleCharStream stream,
int lexState)
setDebugStream
public void setDebugStream(java.io.PrintStream ds)
ReInit
public void ReInit(SimpleCharStream stream)
ReInit
public void ReInit(SimpleCharStream stream,
int lexState)
SwitchTo
public void SwitchTo(int lexState)
jjFillToken
protected Token jjFillToken()
getNextToken
public Token getNextToken()