Guest User

Untitled

a guest
Apr 9th, 2026
40
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.67 KB | None | 0 0
  1. package multi_score
  2.  
  3. import "core:c"
  4. import "core:c/libc"
  5. import "core:flags"
  6. import "core:fmt"
  7. import "core:mem"
  8. import "core:os"
  9. import "core:os/os2"
  10. import "core:slice"
  11. import "core:strings"
  12. import "core:thread"
  13.  
  14. //foreign import cg3 "system:libcg3.so"
  15. foreign import cg3 "../../apertium/cg3/src/libcg3.so"
  16.  
  17. @(default_calling_convention="c")
  18. foreign cg3 {
  19. cg3_init :: proc(stdin: ^c.FILE, stdout: ^c.FILE, stderr: ^c.FILE) -> c.int ---
  20. cg3_cleanup :: proc() -> c.int ---
  21. cg3_grammar_load_buffer :: proc(buffer: ^c.char, length: c.size_t) -> rawptr ---
  22. cg3_grammar_free :: proc(grammar: rawptr) ---
  23. cg3_applicator_create_binary :: proc(grammmar: rawptr) -> rawptr ---
  24. cg3_applicator_free_binary :: proc(applicator: rawptr) ---
  25. cg3_run_grammar_on_buffer_binary :: proc(applicator: rawptr, input: ^c.char, in_length: c.size_t, output: ^c.char, out_length: c.size_t) -> c.size_t ---
  26. }
  27.  
  28. MODE :: enum {
  29. SELECT,
  30. REPLACE,
  31. ADD,
  32. FEATURE,
  33. DELETE,
  34. }
  35.  
  36. CURRENT_MODE := MODE.SELECT
  37.  
  38. LANG :: enum {
  39. BLX,
  40. ENG,
  41. GRC,
  42. }
  43.  
  44. CURRENT_LANG := LANG.ENG
  45.  
  46. SOURCE: []u8
  47. TARGET: []u8
  48. TARGET_OFFSETS: [][]u8
  49. ARENA_MEM: [][]u8
  50.  
  51. ScoreRuleTask :: struct {
  52. rule: string,
  53. grammar: rawptr,
  54. applicator: rawptr,
  55. score: uint,
  56. buffer: []u8,
  57. }
  58.  
  59. score_rule_handler :: proc(task: thread.Task) {
  60. data := cast(^ScoreRuleTask)task.data
  61. fmt.println("starting", data.rule)
  62.  
  63. //grammar := cg3_grammar_load_buffer(raw_data(grammar_str), len(grammar_str))
  64. //defer cg3_grammar_free(grammar)
  65. //applicator := cg3_applicator_create_binary(grammar)
  66. //defer cg3_applicator_free(applicator)
  67. output := make([]u8, len(SOURCE) << 1)
  68. defer delete(output)
  69. ln := cg3_run_grammar_on_buffer_binary(
  70. data.applicator, raw_data(SOURCE), len(SOURCE),
  71. raw_data(output), len(output))
  72. // TODO: iter windows
  73. // TODO: score windows
  74. data.score = ln
  75. }
  76.  
  77. main :: proc() {
  78. Options :: struct {
  79. src: string `args:"pos=0,required"`,
  80. tgt: string `args:"pos=1,required"`,
  81. target_lang: string `args:"pos=2,required"`,
  82. mode: string `args:"pos=3,required"`,
  83. threads: int,
  84. }
  85. opt: Options
  86. style : flags.Parsing_Style = .Unix
  87. flags.parse_or_exit(&opt, os.args, style)
  88.  
  89. switch opt.target_lang {
  90. case "blx":
  91. CURRENT_LANG = .BLX
  92. case "eng":
  93. CURRENT_LANG = .ENG
  94. case "grc":
  95. CURRENT_LANG = .GRC
  96. }
  97.  
  98. if opt.threads == 0 {
  99. //opt.threads = os.get_processor_core_count()
  100. opt.threads = 10
  101. }
  102.  
  103. if cg3_init(libc.stdin, libc.stdout, libc.stderr) == 0 {
  104. panic("cg3 init failed")
  105. }
  106. defer cg3_cleanup()
  107.  
  108. serr: os2.Error
  109. SOURCE, serr = os2.read_entire_file_from_path(opt.src, context.allocator)
  110. defer delete(SOURCE)
  111. if serr != nil {
  112. panic("failed to read source")
  113. }
  114.  
  115. TARGET, terr := os2.read_entire_file_from_path(opt.tgt, context.allocator)
  116. defer delete(TARGET)
  117.  
  118. //rule_blob, r_ok := os.read_entire_file(os.stdin)
  119. //rule_str := transmute(string)rule_blob
  120. rule_str := "REMOVE (NOUN);\nREMOVE (VERB) ;\n"
  121. rules := strings.split(strings.trim(rule_str, " \n"), "\n")
  122.  
  123. chunk_size := len(SOURCE) << 2
  124. arena_data := make([]u8, chunk_size * len(rules))
  125.  
  126. pool: thread.Pool
  127. pool_allocator: mem.Allocator
  128. pool_allocator = context.allocator
  129. thread.pool_init(&pool, pool_allocator, opt.threads)
  130. thread.pool_start(&pool)
  131. defer thread.pool_destroy(&pool)
  132. tasks := make([]ScoreRuleTask, len(rules))
  133. arenas := make([]mem.Arena, len(rules))
  134. for r, i in rules {
  135. grammar_str: string
  136. switch CURRENT_MODE {
  137. case .SELECT:
  138. grammar_str = fmt.aprintf(
  139. "DELIMITERS = \"<$$$>\" ;\nPROTECT (SOURCE) ;\n%s\n",
  140. r)
  141. case .REPLACE:
  142. grammar_str = "hi"
  143. case .ADD:
  144. grammar_str = "hi"
  145. case .FEATURE:
  146. grammar_str = "hi"
  147. case .DELETE:
  148. grammar_str = "hi"
  149. }
  150. tasks[i].rule = r
  151. tasks[i].grammar = cg3_grammar_load_buffer(raw_data(grammar_str), len(grammar_str))
  152. tasks[i].applicator = cg3_applicator_create_binary(tasks[i].grammar)
  153. tasks[i].buffer = arena_data[chunk_size*i:chunk_size*(i+1)]
  154. mem.arena_init(&arenas[i], tasks[i].buffer)
  155. alloc := mem.arena_allocator(&arenas[i])
  156. thread.pool_add_task(&pool, alloc,
  157. score_rule_handler, &tasks[i], i)
  158. }
  159. thread.pool_finish(&pool)
  160. for task in tasks {
  161. fmt.println(task.score)
  162. }
  163. }
  164.  
Advertisement
Add Comment
Please, Sign In to add comment