Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package multi_score
- import "core:c"
- import "core:c/libc"
- import "core:flags"
- import "core:fmt"
- import "core:mem"
- import "core:os"
- import "core:os/os2"
- import "core:slice"
- import "core:strings"
- import "core:thread"
- //foreign import cg3 "system:libcg3.so"
- foreign import cg3 "../../apertium/cg3/src/libcg3.so"
- @(default_calling_convention="c")
- foreign cg3 {
- cg3_init :: proc(stdin: ^c.FILE, stdout: ^c.FILE, stderr: ^c.FILE) -> c.int ---
- cg3_cleanup :: proc() -> c.int ---
- cg3_grammar_load_buffer :: proc(buffer: ^c.char, length: c.size_t) -> rawptr ---
- cg3_grammar_free :: proc(grammar: rawptr) ---
- cg3_applicator_create_binary :: proc(grammmar: rawptr) -> rawptr ---
- cg3_applicator_free_binary :: proc(applicator: rawptr) ---
- cg3_run_grammar_on_buffer_binary :: proc(applicator: rawptr, input: ^c.char, in_length: c.size_t, output: ^c.char, out_length: c.size_t) -> c.size_t ---
- }
- MODE :: enum {
- SELECT,
- REPLACE,
- ADD,
- FEATURE,
- DELETE,
- }
- CURRENT_MODE := MODE.SELECT
- LANG :: enum {
- BLX,
- ENG,
- GRC,
- }
- CURRENT_LANG := LANG.ENG
- SOURCE: []u8
- TARGET: []u8
- TARGET_OFFSETS: [][]u8
- ARENA_MEM: [][]u8
- ScoreRuleTask :: struct {
- rule: string,
- grammar: rawptr,
- applicator: rawptr,
- score: uint,
- buffer: []u8,
- }
- score_rule_handler :: proc(task: thread.Task) {
- data := cast(^ScoreRuleTask)task.data
- fmt.println("starting", data.rule)
- //grammar := cg3_grammar_load_buffer(raw_data(grammar_str), len(grammar_str))
- //defer cg3_grammar_free(grammar)
- //applicator := cg3_applicator_create_binary(grammar)
- //defer cg3_applicator_free(applicator)
- output := make([]u8, len(SOURCE) << 1)
- defer delete(output)
- ln := cg3_run_grammar_on_buffer_binary(
- data.applicator, raw_data(SOURCE), len(SOURCE),
- raw_data(output), len(output))
- // TODO: iter windows
- // TODO: score windows
- data.score = ln
- }
- main :: proc() {
- Options :: struct {
- src: string `args:"pos=0,required"`,
- tgt: string `args:"pos=1,required"`,
- target_lang: string `args:"pos=2,required"`,
- mode: string `args:"pos=3,required"`,
- threads: int,
- }
- opt: Options
- style : flags.Parsing_Style = .Unix
- flags.parse_or_exit(&opt, os.args, style)
- switch opt.target_lang {
- case "blx":
- CURRENT_LANG = .BLX
- case "eng":
- CURRENT_LANG = .ENG
- case "grc":
- CURRENT_LANG = .GRC
- }
- if opt.threads == 0 {
- //opt.threads = os.get_processor_core_count()
- opt.threads = 10
- }
- if cg3_init(libc.stdin, libc.stdout, libc.stderr) == 0 {
- panic("cg3 init failed")
- }
- defer cg3_cleanup()
- serr: os2.Error
- SOURCE, serr = os2.read_entire_file_from_path(opt.src, context.allocator)
- defer delete(SOURCE)
- if serr != nil {
- panic("failed to read source")
- }
- TARGET, terr := os2.read_entire_file_from_path(opt.tgt, context.allocator)
- defer delete(TARGET)
- //rule_blob, r_ok := os.read_entire_file(os.stdin)
- //rule_str := transmute(string)rule_blob
- rule_str := "REMOVE (NOUN);\nREMOVE (VERB) ;\n"
- rules := strings.split(strings.trim(rule_str, " \n"), "\n")
- chunk_size := len(SOURCE) << 2
- arena_data := make([]u8, chunk_size * len(rules))
- pool: thread.Pool
- pool_allocator: mem.Allocator
- pool_allocator = context.allocator
- thread.pool_init(&pool, pool_allocator, opt.threads)
- thread.pool_start(&pool)
- defer thread.pool_destroy(&pool)
- tasks := make([]ScoreRuleTask, len(rules))
- arenas := make([]mem.Arena, len(rules))
- for r, i in rules {
- grammar_str: string
- switch CURRENT_MODE {
- case .SELECT:
- grammar_str = fmt.aprintf(
- "DELIMITERS = \"<$$$>\" ;\nPROTECT (SOURCE) ;\n%s\n",
- r)
- case .REPLACE:
- grammar_str = "hi"
- case .ADD:
- grammar_str = "hi"
- case .FEATURE:
- grammar_str = "hi"
- case .DELETE:
- grammar_str = "hi"
- }
- tasks[i].rule = r
- tasks[i].grammar = cg3_grammar_load_buffer(raw_data(grammar_str), len(grammar_str))
- tasks[i].applicator = cg3_applicator_create_binary(tasks[i].grammar)
- tasks[i].buffer = arena_data[chunk_size*i:chunk_size*(i+1)]
- mem.arena_init(&arenas[i], tasks[i].buffer)
- alloc := mem.arena_allocator(&arenas[i])
- thread.pool_add_task(&pool, alloc,
- score_rule_handler, &tasks[i], i)
- }
- thread.pool_finish(&pool)
- for task in tasks {
- fmt.println(task.score)
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment