parser.cpp

#if 1
struct ParserWorkerThreadData {
    Parser *parser;

    gbSemaphore error_handled; //NOTE(thebirk): Use to signal that the worker thead has a new work to do
    ParseFileError err;

    gbMutex lock; //NOTE(thebirk): All variables below are locked by this mutex
    bool error_available;
    bool should_exit;
};


GB_THREAD_PROC(parse_worker_file_proc) {
    GB_ASSERT(thread != nullptr);

    ParserWorkerThreadData* data = cast(ParserWorkerThreadData*) thread->user_data;

    for(;;) {
        gb_semaphore_wait(&data->error_handled);
        //printf("I am a worker and got signalled to start working!\n");

        gb_mutex_lock(&data->lock);
        //printf("I am a worker and got a lock!\n");
        if (data->should_exit) {
            gb_mutex_unlock(&data->lock);
            //printf("I am being told to quit!\n");
            return isize(0);
        }

        Parser *p = data->parser;
        isize index = thread->user_index;
        gb_mutex_lock(&p->file_add_mutex);
        auto file_to_process = p->files_to_process[index];
        gb_mutex_unlock(&p->file_add_mutex);
        data->err = process_imported_file(p, file_to_process);

        data->error_available = true;
        gb_mutex_unlock(&data->lock);
        //printf("I am a worker and did unlock!\n");
    }

    //GB_PANIC("A worker thread should not be able to reach the end!!!");
    //return isize(0);
}
#else
GB_THREAD_PROC(parse_worker_file_proc) {
    if (thread == nullptr) return 0;
    auto *p = cast(Parser *)thread->user_data;
    isize index = thread->user_index;
    gb_mutex_lock(&p->file_add_mutex);
    auto file_to_process = p->files_to_process[index];
    gb_mutex_unlock(&p->file_add_mutex);
    ParseFileError err = process_imported_file(p, file_to_process);
    return cast(isize)err;
}
#endif

ParseFileError parse_packages(Parser *p, String init_filename) {
    GB_ASSERT(init_filename.text[init_filename.len] == 0);

    // char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char const *)&init_filename[0]);
    // String init_fullpath = string_trim_whitespace(make_string_c(fullpath_str));
    String init_fullpath = path_to_full_path(heap_allocator(), init_filename);
    if (!path_is_directory(init_fullpath)) {
        String const ext = str_lit(".odin");
        if (!string_ends_with(init_fullpath, ext)) {
            gb_printf_err("Expected either a directory or a .odin file, got '%.*s'\n", LIT(init_filename));
            return ParseFile_WrongExtension;
        }
    }


    TokenPos init_pos = {};
    if (!build_context.generate_docs) {
        String s = get_fullpath_core(heap_allocator(), str_lit("runtime"));
        try_add_import_path(p, s, s, init_pos, Package_Runtime);
    }

    try_add_import_path(p, init_fullpath, init_fullpath, init_pos, Package_Init);
    p->init_fullpath = init_fullpath;

#if 1
    isize thread_count = gb_max(build_context.thread_count, 1);
    if (thread_count > 1) {
        isize volatile curr_import_index = 0;
        isize initial_file_count = p->files_to_process.count;
        // NOTE(bill): Make sure that these are in parsed in this order
        for (isize i = 0; i < initial_file_count; i++) {
            ParseFileError err = process_imported_file(p, p->files_to_process[i]);
            if (err != ParseFile_None) {
                return err;
            }
            curr_import_index++;
        }

        auto worker_threads_data = array_make<ParserWorkerThreadData>(heap_allocator(), thread_count);
        defer (array_free(&worker_threads_data));

        for_array(i, worker_threads_data) {
            ParserWorkerThreadData* data = &worker_threads_data[i];
            gb_mutex_init(&data->lock);
            gb_semaphore_init(&data->error_handled);
            data->parser = p;
            data->err = ParseFile_None;
            data->should_exit = false;
            data->error_available = false;
        }

        defer(for_array(i, worker_threads_data) {
            ParserWorkerThreadData* data = &worker_threads_data[i];
            gb_mutex_destroy(&data->lock);
            gb_semaphore_destroy(&data->error_handled);
        });

        auto worker_threads = array_make<gbThread>(heap_allocator(), thread_count);
        defer (array_free(&worker_threads));

        for_array(i, worker_threads) {
            gbThread* t = &worker_threads[i];
            gb_thread_init(t);
            char buffer[64];
            gb_snprintf(buffer, 64, "Parser Worker #%lld", i);
            gb_thread_set_name(t, buffer);
            gb_thread_start(t, parse_worker_file_proc, &worker_threads_data[i]);
        }
        defer(for_array(i, worker_threads) {
            gb_thread_destroy(&worker_threads[i]);
        });

        auto errors = array_make<ParseFileError>(heap_allocator(), 0, 16);

        for (;;) {
            bool are_any_alive = false;

            for_array(i, worker_threads) {
                gbThread *t = &worker_threads[i];
                ParserWorkerThreadData* data = &worker_threads_data[i];

                if (gb_mutex_try_lock(&data->lock)) {
                    if (data->error_available) {
                        auto curr_err = data->err;
                        if (curr_err != ParseFile_None) {
                            array_add(&errors, curr_err);
                        }

                        data->error_available = false;
                    }

                    if (curr_import_index < p->files_to_process.count) {
                        t->user_index = curr_import_index;
                        curr_import_index++;
                        are_any_alive = true;

                        gb_semaphore_release(&data->error_handled);
                    }

                    gb_mutex_unlock(&data->lock);
                } else {
                    //NOTE(thebirk): If we cant lock a thread it must be working
                    are_any_alive = true;
                }
#if 0
                if (gb_thread_is_running(t)) {
                    are_any_alive = true;
                } else if (curr_import_index < p->files_to_process.count) {
                    auto curr_err = cast(ParseFileError)t->return_value;
                    if (curr_err != ParseFile_None) {
                        array_add(&errors, curr_err);
                    } else {
                        t->user_index = curr_import_index;
                        curr_import_index++;
                        gb_thread_start(t, parse_worker_file_proc, p);
                        are_any_alive = true;
                    }
                }
#endif
            }

            //NOTE(thebirk): Everything collapses without this, but it shouldnt really
            gb_yield();

            if (!are_any_alive && (curr_import_index >= p->files_to_process.count)) {
                break;
            }
        }

        for_array(i, worker_threads_data) {
            ParserWorkerThreadData* data = &worker_threads_data[i];
            data->should_exit = true;
            gb_semaphore_release(&data->error_handled);
        }

        if (errors.count > 0) {
            return errors[errors.count-1];
        }
    } else {
        for_array(i, p->files_to_process) {
            ParseFileError err = process_imported_file(p, p->files_to_process[i]);
            if (err != ParseFile_None) {
                return err;
            }
        }
    }
#else
    for_array(i, p->files_to_process) {
        ImportedFile f = p->files_to_process[i];
        ParseFileError err = process_imported_file(p, f);
        if (err != ParseFile_None) {
            return err;
        }
    }
#endif

    return ParseFile_None;
}