LLAMA_BIN=./llama.cpp/main
MODEL_DIR=./llama.cpp/models
PROMPT_LLAMA=$(cat <<EOF
<s>[INST] Repeat this text: "The different accidents of life are not so changeable as the feelings of human nature. I had worked hard for nearly two years, for the sole purpose of infusing life into an inanimate body. For this I had deprived myself of rest and health. I had desired it with an ardour that far exceeded moderation; but now that I had finished, the beauty of the dream vanished, and breathless horror and disgust filled my heart." [/INST]
EOF
)

for MODEL in llama-2-7b-chat.Q4_K_M.gguf llama-2-7b-chat.Q8_0.gguf llama-2-13b-chat.Q4_K_M.gguf llama-2-13b-chat.Q8_0.gguf llama-2-70b-chat.Q4_K_M.gguf llama-2-70b-chat.Q8_0.gguf
do
    echo 3 > /proc/sys/vm/drop_caches
    for NTHREADS in 8 12 16 24 32 48 64
    do
        echo "Testing model $MODEL with $NTHREADS threads"
        $LLAMA_BIN --numa distribute -s 42 -t $NTHREADS -m $MODEL_DIR/$MODEL -b 1024 -c 1024 --temp 0.01 -p "$PROMPT_LLAMA"
    done
done