Class Tokenizer
java.lang.Object
org.sonatype.maven.polyglot.atom.parsing.Tokenizer
Taken from the Loop programming language compiler pipeline.
-
Field Summary
FieldsModifier and TypeFieldDescriptionprivate static final int[]
private final String
private static final int
private static final int
private static final int
private static final boolean[]
-
Constructor Summary
Constructors -
Method Summary
Modifier and TypeMethodDescriptionprivate static void
static String
detokenize
(List<Token> tokens) private static boolean
isDelimiter
(char c) (package private) static boolean
isSingleTokenDelimiter
(char c) private static boolean
isWhitespace
(char c) tokenize()
-
Field Details
-
input
-
NON
private static final int NON- See Also:
-
SINGLE_TOKEN
private static final int SINGLE_TOKEN- See Also:
-
SEQUENCE_TOKEN
private static final int SEQUENCE_TOKEN- See Also:
-
DELIMITERS
private static final int[] DELIMITERS -
STRING_TERMINATORS
private static final boolean[] STRING_TERMINATORS
-
-
Constructor Details
-
Tokenizer
-
-
Method Details
-
tokenize
-
isWhitespace
private static boolean isWhitespace(char c) -
isSingleTokenDelimiter
static boolean isSingleTokenDelimiter(char c) -
detokenize
-
isDelimiter
private static boolean isDelimiter(char c) -
bakeToken
-