Tokenizer rules

// Each TokenClass gets a function that takes the next character in the stream as a parameter,
// and returns true if that character is allowed to start a token of the given TokenClass.
this.startingRules = new Dictionary<TokenClass, Func<TokenClass, char, bool>>
{
    { TokenClass.Whitespace, (lastToken, c) => whitespaceCharacters.Contains(c) },
    { TokenClass.Identifier, (lastToken, c) => char.IsUpper(c) && (lastToken == TokenClass.Whitespace || lastToken == TokenClass.SpecialSymbol) },
    { TokenClass.ReservedWord, (lastToken, c) => char.IsLower(c) && reservedWordStartingCharacters.Contains(c) && (lastToken == TokenClass.Whitespace || lastToken == TokenClass.SpecialSymbol) },
    { TokenClass.SpecialSymbol, (lastToken, c) => specialSymbolStartingCharacters.Contains(c) },
    { TokenClass.Integer, (lastToken, c) => char.IsDigit(c) && (lastToken == TokenClass.Whitespace || lastToken == TokenClass.SpecialSymbol) }
};

// Each TokenClass gets a function that takes the current state of a token and the next
// character in the stream as parameters, and returns true if the next character is also
// part of the current token.
this.continuationRules = new Dictionary<TokenClass, Func<StringBuilder, char, bool>>
{
    { TokenClass.Whitespace, (builder, c) => whitespaceCharacters.Contains(c) },
    { TokenClass.Identifier, (builder, c) => char.IsUpper(c) || char.IsDigit(c) },
    { TokenClass.ReservedWord, (builder, c) => char.IsLower(c) && reservedWordContinuationCharacters.Contains(c) },
    { TokenClass.SpecialSymbol, (builder, c) => builder.Length <= 1 && specialSymbolContinuationCharacters.Contains(c) },
    { TokenClass.Integer, (builder, c) => char.IsDigit(c) }
};