public interface Tokenizer
Modifier and Type | Field and Description |
---|---|
static java.lang.String |
PRECEDED_ROOT_OPTION |
static java.lang.String |
ROOT_TOKEN |
Modifier and Type | Method and Description |
---|---|
TokenizationResult |
tokenize(java.lang.String input,
de.ims.icarus.util.Options options) |
static final java.lang.String PRECEDED_ROOT_OPTION
static final java.lang.String ROOT_TOKEN
TokenizationResult tokenize(java.lang.String input, de.ims.icarus.util.Options options)