Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- read_token :: proc(using parser: ^Parser) -> (Token, bool) {
- for is_whitespace(current_rune) {
- switch current_rune {
- case '\n':
- current_line += 1;
- current_column = 1;
- }
- if _, ok := next_rune(parser); !ok do return {}, false;
- }
- loc := get_current_sourceloc(parser);
- loc.column -= 1;
- if offset > len(data) {
- return Token{TokenKind.Eof, loc, "end of file"}, true;
- }
- using TokenKind;
- switch current_rune {
- case 0:
- return Token{TokenKind.Eof, loc, "end of file"}, true;
- case '#':
- for current_rune != '\n' {
- if _, ok := next_rune(parser); !ok do return {}, false;
- loc = get_current_sourceloc(parser);
- loc.column -= 1;
- if offset == len(data) do return Token{TokenKind.Eof, loc, "end of file"}, true;
- }
- return read_token(parser);
- case '0'..'9':
- //...
- start := current_rune_offset;
- found_dot := false;
- for is_number(current_rune) || current_rune == '.' {
- if current_rune == '.' {
- if found_dot {
- break;
- } else {
- found_dot = true;
- }
- }
- if _, ok := next_rune(parser); !ok do return {}, false;
- }
- tok := Token{};
- tok.lexeme = string(data[start:current_rune_offset]);
- tok.kind = found_dot ? Number : Integer;
- tok.loc = loc;
- return tok, true;
- case '"':
- if _, ok := next_rune(parser); !ok do return {}, false;
- start := current_rune_offset;
- for current_rune != '"' {
- prev := current_rune;
- if _, ok := next_rune(parser); !ok do return {}, false;
- if prev == '\\' && current_rune == '"' {
- if _, ok := next_rune(parser); !ok do return {}, false;
- }
- }
- tok := Token{};
- tok.lexeme = string(data[start:current_rune_offset]);
- tok.kind = String;
- tok.loc = loc;
- if _, ok := next_rune(parser); !ok do return {}, false;
- return tok, true;
- case '{':
- if _, ok := next_rune(parser); !ok do return {}, false;
- return Token{LeftBrace, loc, "{"}, true;
- case '}':
- if _, ok := next_rune(parser); !ok do return {}, false;
- return Token{RightBrace, loc, "}"}, true;
- case ';':
- if _, ok := next_rune(parser); !ok do return {}, false;
- return Token{SemiColon, loc, ";"}, true;
- case:
- start := current_rune_offset;
- for is_valid_identifier(current_rune) {
- if _, ok := next_rune(parser); !ok do return {}, false;
- }
- tok := Token{};
- tok.lexeme = string(data[start:current_rune_offset]);
- tok.kind = Ident;
- tok.loc = loc;
- return tok, true;
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement