lx01

std::deque<std::string>* tokenize (char** in, int len) {
    std::deque<std::string>* queue = new std::deque<std::string>();
    bool comment = false;
    for (int i = 0; i < len; i++) {
        char* target = in[i];
        while (strlen(target) != 0) {
            if (comment) {
                if (strlen(target) >= 2 && !strncmp(target, "*/", 2)) {
                    target++;
                    comment = false;
                }
                target++;
                continue;
            }
            if (target[0] == ' ') {
                target++;
                continue;
            } else if (strlen(target) >= 2 && !strncmp(target, "//", 2)) {
                break;
            } else if (strlen(target) >= 2 && !strncmp(target, "/*", 2)) {
                comment = true;
            }
            char* token = tokenize(target);
            if (!token) throw 0;
            queue.push_back(token);
            target += strlen(token);
        }
    }
}

char* tokenize (char* in) {
    char* match = nullptr;
    if (tokenizeKeyword(in, &match) || tokenizeLiteral(in, &match) || tokenizeSeparator(in, &match) || tokenizeGeneric(in, &match))
        return match;
    return nullptr;
}

bool tokenizeKeyword (char* in, char** match) {
    for (int i = 0; i < KEYWORDS_LENGTH; i++) {
        char* keyword = KEYWORDS[i];
        if (strlen(in) < strlen(keyword)) continue;
        if (strncmp(in, keyword, strlen(keyword))) {
            *match = keyword;
            return true;
        }
    }
    return false;
}

bool tokenizeSeparator (char* in, char** match) {
    for (int i = 0; i < SEPARATORS_LENGTH; i++) {
        char* keyword = SEPARATORS[i];
        if (strlen(in) < strlen(keyword)) continue;
        if (strncmp(in, keyword, strlen(keyword))) {
            *match = keyword;
            return true;
        }
    }
    return false;
}

bool tokenizeGeneric (char* in, char** match) {
    int i = 0;
    for (; i < strlen(in); i++) {
        if (!genericChar(in))
            break;
    }
    *match = new char[i];
    strncpy(in, *match, i);
    return true;
}

bool genericChar (char c) {
    if (c >= 'a' && c <= 'z') return true;
    if (c >= 'A' && c <= 'Z') return true;
    if (c == '_') return true;
}