batch 32

main: decoded 160 tokens in 583.57 s, speed: 0.27 t/s

llama_print_timings:        load time =  285509.13 ms
llama_print_timings:      sample time =       6.67 ms /   192 runs   (    0.03 ms per token, 28776.98 tokens per second)
llama_print_timings: prompt eval time =  684425.85 ms /   163 tokens ( 4198.93 ms per token,     0.24 tokens per second)
llama_print_timings:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
llama_print_timings:       total time =  869074.32 ms










batch 64

main: decoded 320 tokens in 692.25 s, speed: 0.46 t/s

llama_print_timings:        load time =  288526.37 ms
llama_print_timings:      sample time =      14.86 ms /   384 runs   (    0.04 ms per token, 25844.66 tokens per second)
llama_print_timings: prompt eval time =  791794.03 ms /   323 tokens ( 2451.37 ms per token,     0.41 tokens per second)
llama_print_timings:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
llama_print_timings:       total time =  980780.97 ms



batch 256

main: decoded 1276 tokens in 1357.56 s, speed: 0.94 t/s

llama_print_timings:        load time =  287171.26 ms
llama_print_timings:      sample time =      54.07 ms /  1532 runs   (    0.04 ms per token, 28332.59 tokens per second)
llama_print_timings: prompt eval time = 1457523.04 ms /  1279 tokens ( 1139.58 ms per token,     0.88 tokens per second)
llama_print_timings:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
llama_print_timings:       total time = 1644731.26 ms