declare
fun {Tokenize StringToTokenize CurrentState}
    case StringToTokenize
    of nil then nil
    [] H|T then
        case CurrentState
        of 'keyword' then
            // do keyword stuff
            {Tokenize T 'keyword'}
        [] 'operator' then
            // do operator stuff
            {Tokenize T 'operator'}
        [] 'atom' then
            // do atom stuff
            {Tokenize T 'atom'}
        [] 'int' then
            // do int stuff
            {Tokenize T 'int'}
        [] 'float' then
            // do float stuff
            {Tokenize T 'float'}
        [] 'separator' then
            // do separator stuff
            {Tokenize T 'separator'}
        end
    end
end