Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- std::unordered_map<std::string, tokentype> keywords = {
- { "Ohai", TokenType::Ohai }
- };
- // For tokens which are always exactly one character
- std::unordered_map<char, tokentype> punctuation = {
- { '}', TokenType::CloseCurlyBrace },
- { '{', TokenType::OpenCurlyBrace },
- };
- struct Token {
- TokenType t;
- Token(TokenType type) : t(type) {}
- };
- std::vector<Token> lex(std::string s) {
- std::vector<Token> out;
- for(auto it = s.begin(); it != s.end(); ++it) {
- auto next = [&] { return *(it + 1); };
- auto single_equals = [&](char c, TokenType single, TokenType equals) -> bool {
- if (*it == c) {
- if (next() == '=') {
- out.push_back(equals);
- ++it;
- return true;
- }
- out.push_back(single);
- return true;
- }
- return false;
- };
- auto double_equals = [&](char c, TokenType single, TokenType double, TokenType equals) -> bool {
- if (*it == c) {
- if (next() == '=') {
- out.push_back(equals);
- ++it;
- return true;
- }
- if (next() == c) {
- out.push_back(double);
- ++it;
- return true;
- }
- out.push_back(single);
- return true;
- }
- return false;
- };
- if (*it == '/') {
- if (next() == '/') {
- while(*it != '\n') {
- ++it;
- ++it;
- continue;
- }
- if (next() == '*') {
- while(*it != '*') {
- ++it;
- if (next() == '/')
- ++it;
- break;
- }
- }
- continue;
- }
- }
- if (punctuation.find(*it) != punctuation.end()) {
- out.push_back(punctuation[*it]);
- ++it;
- continue;
- }
- if (single_equals('/', TokenType::Div, TokenType::DivEquals)) continue;
- if (single_equals('!', TokenType::Not, TokenType::NotEquals)) continue;
- if (double_equals('+', TokenType::Plus, TokenType::Increment, TokenType::PlusEquals)) continue;
- if (double_equals('-', TokenType::Sub, TokenType::Decrement, TokenType::SubEquals)) continue;
- }
- return out;
- }
Advertisement
Add Comment
Please, Sign In to add comment