Simple lexer in Python

from abc import ABC, abstractmethod
from enum import Enum
from types import NoneType
from typing import Optional
import string

with open("hidden/src.txt", "r") as f:
    src = f.read()

lexeme_list = []


class ReservedWordToken(Enum):
    TokenReturn = "return"
    TokenIf = "if"
    TokenElse = "else"
    TokenFor = "for"
    TokenWhile = "while"
    TokenInt = "int"
    TokenReal = "real"
    TokenString = "string"
    TokenUnion = "union"
    TokenDef = "def"
    TokenBool = "bool"
    TokenArray = "array"
    TokenEnum = "enum"
    TokenContinue = "continue"
    TokenBreak = "break"
    TokenMatch = "match"
    TokenDefault = "default"


class SingleCharToken(Enum):
    TokenAdd = "+"
    TokenSub = "-"
    TokenMul = "*"
    TokenDiv = "/"
    TokenMod = "%"
    TokenAssignment = "="
    TokenGE = ">"
    TokenLE = "<"
    TokenAnd = "&"
    TokenOr = "|"
    # not operators
    TokenLeftBracket = "("
    TokenRightBracket = ")"
    TokenComma = ","


class NewInstructionToken:
    NEW_INSTRUCTION_CHAR = ";"

    def __init__(self) -> None:
        self.name = type(self).__name__
        self.value = self.NEW_INSTRUCTION_CHAR
        pass


single_chr_toks = [x.value for x in list(SingleCharToken.__members__.values())]
keyword_toks = set(x.value for x in list(ReservedWordToken.__members__.values()))


class DataTypeToken(ABC):
    def __init__(self) -> None:
        pass

    @abstractmethod
    def check_termination(self, chr) -> bool: ...

    def __repr__(self) -> str:
        return type(self).__name__


class NumericToken(DataTypeToken):
    def check_termination(self, chr):
        # legal chars are numbers, dots, single ops and spaces
        # terminate if NOT (number or dot) and legal char
        # print("a" + chr + "b")
        numeric_condition = chr in (string.digits + ".")
        space = chr.isspace()
        single_op_chrs = chr in "".join(single_chr_toks)
        legal = numeric_condition or space or single_op_chrs
        if not legal:
            raise ValueError(f"Illegal character: {chr}")
        return not numeric_condition and ((single_op_chrs) or (chr.isspace()))


class TypeIntToken(NumericToken):
    pass


class TypeRealToken(NumericToken):
    pass


class TypeStringToken(DataTypeToken):
    def __init__(self, opener=None) -> None:
        self.opener = opener

    def check_termination(self, chr) -> bool:
        return chr == self.opener if hasattr(self, "opener") else False


class UserIdentifierToken(DataTypeToken):
    def check_termination(self, chr) -> bool:
        # TODO make things like hello(arg1, arg2) work, parse args too
        return chr not in (string.ascii_letters + "_" + string.digits)


class DataToken(Enum):
    TokenInt = TypeIntToken()
    TokenReal = TypeRealToken()
    TokenString = TypeStringToken()
    TokenUserDef = UserIdentifierToken()


class Lexeme:
    def __init__(
        self,
        token_type: SingleCharToken
        | DataToken
        | ReservedWordToken
        | NewInstructionToken,
        start: int,
        end: Optional[int] = None,
    ) -> None:
        self.token_type = token_type
        self.start = start
        self.end = end

    def __repr__(self) -> str:
        return f"Lexeme({src[self.start : self.end] if not (isinstance(self.start, NoneType) and isinstance(self.end, NoneType)) else ''}, {self.start=}, {self.end=}, self.token_type={self.token_type.name})"


def lex(src):
    current_lexeme: Optional[Lexeme] = None
    for i, s in enumerate(src):
        # check for newline
        if s == NewInstructionToken.NEW_INSTRUCTION_CHAR:
            # check if there is an existing lexeme
            if current_lexeme:
                current_lexeme.end = i
            lexeme_list.append(Lexeme(NewInstructionToken(), i, i + 1))
            current_lexeme = None
            continue

        # check for reserved keywords
        target_set = set()
        if current_lexeme and current_lexeme.token_type == DataToken.TokenUserDef:
            target_set.add(src[current_lexeme.start : i + 1])

        if target_set.intersection(keyword_toks):
            current_lexeme.token_type = ReservedWordToken(
                src[current_lexeme.start : i + 1]
            )
            current_lexeme.end = i + 1
            current_lexeme = None
            continue

        # check for termination
        if (
            current_lexeme
            and isinstance(current_lexeme.token_type, DataToken)
            and current_lexeme.token_type.value.check_termination(s)
        ):
            if current_lexeme.token_type == DataToken.TokenString:
                current_lexeme.end = i + 1
                current_lexeme = None
                continue
            else:
                current_lexeme.end = i
            current_lexeme = None

        # check for single chr lexemes
        if s in single_chr_toks and not current_lexeme:
            lexeme_list.append(Lexeme(SingleCharToken(s), i, i + 1))
            current_lexeme = None
            continue

        # discriminate against raw strings, first token is NOT a quotation mark
        if not current_lexeme and s in (string.ascii_letters + "_"):
            current_lexeme = Lexeme(DataToken.TokenUserDef, i)
            lexeme_list.append(current_lexeme)

        # check strings
        # NOTE: escape chars not supported
        if (s == '"' or s == "'") and (not current_lexeme):
            current_lexeme = Lexeme(DataToken.TokenString, i)
            current_lexeme.token_type.value.opener = s
            lexeme_list.append(current_lexeme)
            continue

        # check if digits are reals (prerequisite is to satisfy int + ".")
        if s == ".":
            if current_lexeme and current_lexeme.token_type == DataToken.TokenInt:
                current_lexeme.token_type = DataToken.TokenReal
                continue
            raise TypeError("Illegal character '.'")

        # check if chr can be an int
        if s.isdigit() and not current_lexeme:
            current_lexeme = Lexeme(DataToken.TokenInt, i)
            lexeme_list.append(current_lexeme)
            continue

    # check for unterminated strings

    if current_lexeme and current_lexeme.token_type == DataToken.TokenString:
        raise ValueError("Unterminated string")

    # terminate last lexeme if not ended alr

    elif current_lexeme:
        current_lexeme.end = len(src)
    return lexeme_list