const TOKEN_NUMBER = 0
const TOKEN_IDENTIFIER = 1

function main()
{
	// USAGE
	// USAGE
	// USAGE
	const input = "hello adrian 123";
	let tokenizer = {text: input, index: 0};

	const token = get_token(tokenizer); // returns type = identifier, text = "hello"
	token = get_token(tokenizer); // returns type = identifier, text = "adrian"
	token = get_token(tokenizer); // returns type = number, text = "123"

	// If you want to look at the next token but not advance the tokenizer:
	token = get_token({...tokenizer})

	return 0;
}

function get_token(tokenizer)
{
	let token = {type: null, text: null};
	eat_whitespace(tokenizer);
	if(is_number(tokenizer.text[tokenizer.index])) {
		token.type = TOKEN_NUMBER;
		let len = 0;
		while(is_number(tokenizer.text[tokenizer.index])) {
			len += 1;
		}
		token.text = tokenizer.text.substring(tokenizer.index, tokenizer.index + len);
		tokenizer.index += len;
	}
	else if(...) { // logic for an identifier

	}
	else if(...) { // logic for something else

	}
	// ...
	else {
		printf("We do not understand the input!\n");
	}
	return token;
}

function eat_whitespace(tokenizer)
{
	while(
		tokenizer.text[tokenizer.index] == ' ' || tokenizer.text[tokenizer.index] == '\n' || tokenizer.text[tokenizer.index] == '\t' ||
		tokenizer.text[tokenizer.index] == '\r'
	) {
		tokenizer.index += 1;
	}
}

function is_number(char)
{
	return char >= '0' && char <= '9';
}