 package scone.util.tokenstream;


/**
 * A Tokenizer tokenizes (parses) a Reader.
 * @author Volkert Buchmann, IBM Almaden Research Center, (c)
 *         Copyright IBM Corp 2000
 * @version 1.0
 */

public interface Tokenizer {

    /**
     * extracts the next Token from the Reader.
     * @return the next Token, or null if there are no more Tokens
     */
    public Token nextToken();

    /**
     * assigns a Reader to the Tokenizer. Only one Reader can be assigned to the Tokenizer.
     * @param reader the Reader
     */
    public void assign(java.io.Reader reader);
}
