Tokenizers

var strDefs = """generic: /
block: [cs]
binop: lhs rhs
infix: lhs op rhs
"""

type
    TokenKind = enum
        tkIdent
        tkColon
        tkLSquare
        tkRSquare
        tkSlash

    Token = object
        case kind: TokenKind
        of tkIdent:
            ident: string
        of tkColon, tkLSquare, tkRSquare, tkSlash:
            discard

    TokenizerResult = tuple[succ: bool, index: int]

proc discardWhitespace(str: string, index: int) : TokenizerResult =
    var index = index
    while index < str.len():
        let c = str[index]
        if c notin {' ', '\t'}:
            return (true, index)
        index.inc(1)
    return (true, index+1)

proc readIdentToken(str: string, index: int, token: var Token) : TokenizerResult =
    var
        index = index
        ident = ""
    while index < str.len():
        let c = str[index]
        if c in {'a'..'z'}:
            ident.add(c)
        else:
            break
    if ident.len()==0:
        return (false, -1)
    token = Token(kind: tkIdent, ident: ident)
    return (true, index+1)

proc readSymbolToken(str: string, index: int, token: var Token) : TokenizerResult =
    let c = str[index]
    case c
    of '[':
        token = Token(kind: tkLSquare)
        return (true, index+1)
    of ']':
        token = Token(kind: tkRSquare)
        return (true, index+1)
    of ':':
        token = Token(kind: tkColon)
        return (true, index+1)
    of '/':
        token = Token(kind: tkSlash)
        return (true, index+1)
    else:
        return (false, -1)

proc tokenizeDefs(str: string) : seq[Token] =
    result = @[]
    var
        index = 0
        token : Token = Token(kind: tkSlash)
    while index < str.len():
        index = discardWhitespace(str, index).index
        var tokResult = readIdentToken(str, index, token)
        if tokResult.succ:
            index = tokResult.index
            result.add(token)
            echo "Read an ident"
            continue
        tokResult = readSymbolToken(str, index, token)
        if tokResult.succ:
            index = tokResult.index
            result.add(token)
            echo "Read a symbol"
            continue
        break
    return result


echo tokenizeDefs(strDefs)