const TOKEN_NUMBER = 0
const TOKEN_IDENTIFIER = 1

function main()
{
    // USAGE
    // USAGE
    // USAGE
    const input = "hello adrian 123";
    let tokenizer = {text: input, index: 0};

    const token = get_token(tokenizer); // returns type = identifier, text = "hello"
    token = get_token(tokenizer); // returns type = identifier, text = "adrian"
    token = get_token(tokenizer); // returns type = number, text = "123"

    // If you want to look at the next token but not advance the tokenizer:
    token = get_token({...tokenizer})

    return 0;
}

function get_token(tokenizer)
{
    let token = {type: null, text: null};
    eat_whitespace(tokenizer);
    if(is_number(tokenizer.text[tokenizer.index])) {
        token.type = TOKEN_NUMBER;
        let len = 0;
        while(is_number(tokenizer.text[tokenizer.index])) {
            len += 1;
        }
        token.text = tokenizer.text.substring(tokenizer.index, tokenizer.index + len);
        tokenizer.index += len;
    }
    else if(...) { // logic for an identifier

    }
    else if(...) { // logic for something else

    }
    // ...
    else {
        printf("We do not understand the input!\n");
    }
    return token;
}

function eat_whitespace(tokenizer)
{
    while(
        tokenizer.text[tokenizer.index] == ' ' || tokenizer.text[tokenizer.index] == '\n' || tokenizer.text[tokenizer.index] == '\t' ||
        tokenizer.text[tokenizer.index] == '\r'
    ) {
        tokenizer.index += 1;
    }
}

function is_number(char)
{
    return char >= '0' && char <= '9';
}