API tools faq
paste
Tom_Neverwinter

LLAMA BENCHMARK [PY] 1/04/2026

Tom_Neverwinter
Dec 22nd, 2025 (edited)
3,373
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 17.87 KB | None | 0 0
raw download clone embed print report
  1. import streamlit as st
  2. import pandas as pd
  3. import subprocess
  4. import os
  5. import re
  6. import datetime
  7. import time
  8.  
  9. # ==========================================
  10. # CONFIGURATION
  11. # ==========================================
  12. BASE_DIR = os.getcwd()
  13. MODELS_DIR = os.path.join(BASE_DIR, "models")
  14. LOGS_DIR = os.path.join(BASE_DIR, "bench_logs")
  15. BENCH_EXE = os.path.join(BASE_DIR, "llama-bench.exe")
  16. CLI_EXE = os.path.join(BASE_DIR, "llama-cli.exe")
  17. DB_FILE = os.path.join(BASE_DIR, "model_library_benchmark.csv")
  18. HISTORY_FILE = os.path.join(BASE_DIR, "benchmark_history.csv")
  19. KEYWORDS = ["NSFW", "Uncensored", "Amoral", "Dark", "Fallen", "Rivermind", "Snowpiercer", "Abyss", "Erebus", "Lewd"]
  20.  
  21. if not os.path.exists(LOGS_DIR):
  22. os.makedirs(LOGS_DIR)
  23.  
  24. # --- SCHEMA ---
  25. COLS = [
  26. # Identity
  27. "Model_Name", "Size_GB", "Quant", "Role", "B_Tag", "Params_Actual", "Backend", "Llama_Version",
  28. # Metrics
  29. "Gen_TG128", "Gen_TG128_StDev", "Ingest_PP512", "Rating_Speed",
  30. # Human Eval
  31. "HE_Overall", "HE_Math", "HE_Science", "HE_History", "HE_Music",
  32. "HE_English", "HE_Coding", "HE_Logic", "HE_Creative", "HE_RP_Quality",
  33. "HE_Personality", "HE_Emotes", "HE_Formatting", "HE_Instructions",
  34. "HE_Censorship", "HE_Memory", "HE_Summarize", "HE_Translation",
  35. "HE_JSON", "HE_RAG", "HE_Vision", "HE_Fun_Factor"
  36. ]
  37.  
  38. st.set_page_config(page_title="Llama.cpp Librarian", layout="wide", page_icon="🦙")
  39.  
  40. # ==========================================
  41. # FUNCTIONS
  42. # ==========================================
  43. def get_hardware_info():
  44. try:
  45. gpu_info = subprocess.check_output("nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv,noheader,nounits", shell=True).decode().strip()
  46. gpu_name, total_vram, free_vram = gpu_info.split(',')
  47. return f"{gpu_name} | VRAM: {free_vram.strip()}MB Free / {total_vram.strip()}MB Total"
  48. except:
  49. return "GPU Detection Failed"
  50.  
  51. def get_llama_version():
  52. try:
  53. # Try to get version from llama-cli
  54. if os.path.exists(CLI_EXE):
  55. cmd = [CLI_EXE, "--version"]
  56. result = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8')
  57. output = result.stdout + result.stderr
  58. # Look for "version: 7549 (c9ced4910)"
  59. match = re.search(r'version:\s*(\d+)\s*\(([^)]+)\)', output)
  60. if match:
  61. return f"b{match.group(1)}-{match.group(2)[:7]}"
  62. return "Unknown Version"
  63. return "CLI Not Found"
  64. except:
  65. return "Version Check Failed"
  66.  
  67. def parse_model_properties(filename):
  68. name = filename
  69. try:
  70. size_gb = round(os.path.getsize(os.path.join(MODELS_DIR, filename)) / (1024**3))
  71. except:
  72. size_gb = 0
  73.  
  74. role = "Specialized/NSFW" if any(k.lower() in name.lower() for k in KEYWORDS) else "General"
  75.  
  76. quant = "Unknown"
  77. quants = ["Q8_0", "Q6_K", "Q5_K_M", "Q5_K_S", "Q5_0", "Q4_K_M", "Q4_K_S", "Q4_0", "IQ3", "Q2_K", "F16", "F32"]
  78. for q in quants:
  79. if q.lower() in name.lower():
  80. quant = q
  81. break
  82.  
  83. b_tag = "Other"
  84. b_match = re.search(r'(\d+(?:\.\d+)?)B', name, re.IGNORECASE)
  85. if b_match:
  86. b_tag = b_match.group(0).upper()
  87.  
  88. return size_gb, role, quant, b_tag
  89.  
  90. def run_benchmark(model_name):
  91. model_path = os.path.join(MODELS_DIR, model_name)
  92. cmd = [BENCH_EXE, "-m", model_path, "-t", "8", "-ngl", "999"]
  93.  
  94. try:
  95. result = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8')
  96. output = result.stdout
  97.  
  98. # Log
  99. timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
  100. log_file = os.path.join(LOGS_DIR, f"{model_name}_{timestamp}.log")
  101. with open(log_file, "w", encoding="utf-8") as f:
  102. f.write(f"COMMAND: {' '.join(cmd)}\n")
  103. f.write("="*40 + "\n")
  104. f.write(output)
  105. f.write("\n" + "="*40 + "\n")
  106. f.write(result.stderr)
  107.  
  108. # Parse
  109. pp_match = re.search(r'\|\s*pp512\s*\|\s*([0-9\.]+)\s*\+/-\s*([0-9\.]+)', output)
  110. tg_match = re.search(r'\|\s*tg128\s*\|\s*([0-9\.]+)\s*\+/-\s*([0-9\.]+)', output)
  111.  
  112. if not pp_match: pp_match = re.search(r'\|\s*pp512\s*\|\s*([0-9\.]+)', output)
  113. if not tg_match: tg_match = re.search(r'\|\s*tg128\s*\|\s*([0-9\.]+)', output)
  114.  
  115. pp_val = float(pp_match.group(1)) if pp_match else 0.0
  116. tg_val = float(tg_match.group(1)) if tg_match else 0.0
  117. tg_std = float(tg_match.group(2)) if tg_match and len(tg_match.groups()) > 1 else 0.0
  118.  
  119. backend = "Unknown"
  120. if "CUDA" in output: backend = "CUDA"
  121. elif "ROCm" in output: backend = "ROCm"
  122. elif "Metal" in output: backend = "Metal"
  123. elif "CPU" in output and "CUDA" not in output: backend = "CPU"
  124.  
  125. params = "Unknown"
  126. param_match = re.search(r'\|\s*([0-9\.]+)\s*B\s*\|', output)
  127. if param_match:
  128. params = f"{param_match.group(1)}B"
  129.  
  130. return {
  131. "pp": pp_val, "tg": tg_val, "tg_std": tg_std,
  132. "backend": backend, "params": params, "raw": output,
  133. "version": get_llama_version()
  134. }
  135. except Exception as e:
  136. return {"error": str(e)}
  137.  
  138. def load_db():
  139. if not os.path.exists(DB_FILE):
  140. return pd.DataFrame(columns=COLS)
  141. try:
  142. df = pd.read_csv(DB_FILE)
  143. for col in COLS:
  144. if col not in df.columns:
  145. if col.startswith("HE_"): df[col] = 0
  146. elif col == "Rating_Speed": df[col] = "Untested"
  147. elif col == "Backend": df[col] = "-"
  148. elif col == "Params_Actual": df[col] = "-"
  149. elif col == "Llama_Version": df[col] = "-"
  150. else: df[col] = 0.0
  151. return df[COLS]
  152. except:
  153. return pd.DataFrame(columns=COLS)
  154.  
  155. def save_db(df):
  156. df.to_csv(DB_FILE, index=False)
  157.  
  158. def load_history():
  159. if not os.path.exists(HISTORY_FILE):
  160. return pd.DataFrame(columns=["Timestamp", "Model_Name", "Llama_Version", "Speed", "Speed_StDev", "Backend"])
  161. return pd.read_csv(HISTORY_FILE)
  162.  
  163. def append_history(model_name, version, speed, stdev, backend):
  164. df_hist = load_history()
  165. new_row = {
  166. "Timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
  167. "Model_Name": model_name,
  168. "Llama_Version": version,
  169. "Speed": speed,
  170. "Speed_StDev": stdev,
  171. "Backend": backend
  172. }
  173. if df_hist.empty:
  174. df_hist = pd.DataFrame([new_row])
  175. else:
  176. df_hist = pd.concat([df_hist, pd.DataFrame([new_row])], ignore_index=True)
  177. df_hist.to_csv(HISTORY_FILE, index=False)
  178.  
  179. # ==========================================
  180. # MAIN UI
  181. # ==========================================
  182. st.title("🦙 Llama.cpp Command Center")
  183. st.caption(get_hardware_info())
  184.  
  185. if "last_bench_time" not in st.session_state:
  186. st.session_state.last_bench_time = 0
  187.  
  188. df = load_db()
  189.  
  190. # Auto-Discovery of new models
  191. if os.path.exists(MODELS_DIR):
  192. disk_models = [f for f in os.listdir(MODELS_DIR) if f.endswith(".gguf") and "mmproj" not in f]
  193. db_models = df['Model_Name'].tolist()
  194. new_found = [m for m in disk_models if m not in db_models]
  195.  
  196. if new_found:
  197. new_rows = []
  198. for f in new_found:
  199. s, r, q, b = parse_model_properties(f)
  200. row = {c: 0 for c in COLS}
  201. row.update({
  202. "Model_Name": f, "Size_GB": s, "Quant": q, "Role": r, "B_Tag": b,
  203. "Gen_TG128": 0.0, "Ingest_PP512": 0.0, "Rating_Speed": "Untested",
  204. "Backend": "-", "Params_Actual": "-", "Llama_Version": "-"
  205. })
  206. new_rows.append(row)
  207.  
  208. if new_rows:
  209. df_new = pd.DataFrame(new_rows)[COLS]
  210. df = pd.concat([df, df_new], ignore_index=True)
  211. save_db(df)
  212. st.toast(f"Auto-indexed {len(new_rows)} new models!", icon="🆕")
  213.  
  214. # Sidebar Actions
  215. st.sidebar.header("Actions")
  216. if st.sidebar.button("🔄 Scan & Index Library"):
  217. if not os.path.exists(MODELS_DIR):
  218. st.error(f"Models folder not found at: {MODELS_DIR}")
  219. else:
  220. files = [f for f in os.listdir(MODELS_DIR) if f.endswith(".gguf") and "mmproj" not in f]
  221. new_rows = []
  222. for f in files:
  223. existing = df[df['Model_Name'] == f]
  224. if not existing.empty:
  225. row = existing.iloc[0].to_dict()
  226. for c in COLS:
  227. if c not in row: row[c] = 0
  228. else:
  229. s, r, q, b = parse_model_properties(f)
  230. row = {c: 0 for c in COLS}
  231. row.update({
  232. "Model_Name": f, "Size_GB": s, "Quant": q, "Role": r, "B_Tag": b,
  233. "Gen_TG128": 0.0, "Ingest_PP512": 0.0, "Rating_Speed": "Untested",
  234. "Backend": "-", "Params_Actual": "-", "Llama_Version": "-"
  235. })
  236. new_rows.append(row)
  237. df = pd.DataFrame(new_rows)[COLS]
  238. save_db(df)
  239. st.sidebar.success(f"Indexed {len(new_rows)} models!")
  240. st.rerun()
  241.  
  242. # --- TABS ---
  243. tab1, tab2, tab3, tab4, tab5 = st.tabs(["📊 Speed", "📝 Human Ratings", "⚖️ Analysis & Winners", "🆚 Versions", "🚀 Benchmarking"])
  244.  
  245. with tab1:
  246. st.subheader("Speed Leaderboard")
  247. if not df.empty and df['Gen_TG128'].max() > 0:
  248. clean = df[df['Gen_TG128'] > 0].sort_values(by="Gen_TG128", ascending=False).head(15)
  249. st.bar_chart(clean, x="Model_Name", y="Gen_TG128", color="Quant")
  250.  
  251.  
  252.  
  253. display_cols = ["Model_Name", "Params_Actual", "Backend", "Llama_Version", "Quant", "Gen_TG128", "Gen_TG128_StDev", "Rating_Speed"]
  254. st.dataframe(clean[display_cols], hide_index=True, width="stretch")
  255. else:
  256. st.info("No benchmarks run yet.")
  257.  
  258. with tab2:
  259. st.subheader("Human Ratings Matrix")
  260. st.info("Edit your scores (0-10) below. 'Overall' is auto-calculated when you Save.")
  261.  
  262. if not df.empty:
  263. # Determine columns to edit (everything starting with HE_)
  264. he_cols = [c for c in COLS if c.startswith("HE_")]
  265.  
  266. # Prepare Dataframe for Editor (Index = Model_Name to lock it)
  267. df_for_editing = df[["Model_Name"] + he_cols].set_index("Model_Name")
  268.  
  269. edited_df = st.data_editor(
  270. df_for_editing,
  271. key="human_eval_editor",
  272. width="stretch",
  273. column_config={
  274. # Lock the Overall column so it's read-only
  275. "HE_Overall": st.column_config.NumberColumn("⭐ Overall (Auto)", format="%.1f", disabled=True),
  276. }
  277. )
  278.  
  279. if st.button("💾 Save & Calculate Averages"):
  280. # Update master DF from the edited version
  281. for model_name, row in edited_df.iterrows():
  282. master_idx = df.index[df['Model_Name'] == model_name]
  283.  
  284. if not master_idx.empty:
  285. # 1. Update the specific category columns
  286. # We skip HE_Overall here because we calculate it below
  287. for col in he_cols:
  288. if col != "HE_Overall":
  289. df.loc[master_idx, col] = row[col]
  290.  
  291. # 2. AUTO-TABULATE LOGIC
  292. # Extract the scores for this model
  293. # We only want he_cols excluding overall
  294. cat_scores = [row[c] for c in he_cols if c != "HE_Overall"]
  295.  
  296. # Filter: Only count scores > 0 (Ignore "Unrated")
  297. valid_scores = [s for s in cat_scores if s > 0]
  298.  
  299. if valid_scores:
  300. new_avg = sum(valid_scores) / len(valid_scores)
  301. else:
  302. new_avg = 0
  303.  
  304. # 3. Write the calculated average
  305. df.loc[master_idx, "HE_Overall"] = round(new_avg, 1)
  306.  
  307. save_db(df)
  308. st.success("Saved! Overall scores have been updated.")
  309. time.sleep(1) # Brief pause for visual feedback
  310. st.rerun() # Refresh page to show new numbers
  311.  
  312. with tab3:
  313. st.subheader("⚖️ The Value Matrix (Speed vs. Smarts)")
  314. if df.empty or df['Gen_TG128'].max() == 0:
  315. st.warning("Please run benchmarks and add some human ratings to unlock this tab.")
  316. else:
  317. col_ctrl1, col_ctrl2 = st.columns([1, 2])
  318. with col_ctrl1:
  319. st.markdown("#### 🎚️ Your Priorities")
  320. weight = st.slider("Preference Balance:", 0, 100, 50, format="%d%% Quality")
  321. w_qual = weight / 100.0
  322. w_speed = 1.0 - w_qual
  323. st.caption(f"Weights: Speed {int(w_speed*100)}% | Quality {int(w_qual*100)}%")
  324.  
  325. adf = df.copy()
  326. max_speed = adf['Gen_TG128'].max()
  327. if max_speed == 0: max_speed = 1
  328.  
  329. adf['Norm_Speed'] = adf['Gen_TG128'] / max_speed
  330. adf['Norm_Qual'] = adf['HE_Overall'] / 10.0
  331. adf['Win_Score'] = (adf['Norm_Speed'] * w_speed * 100) + (adf['Norm_Qual'] * w_qual * 100)
  332. winners = adf.sort_values(by="Win_Score", ascending=False)
  333.  
  334. st.markdown(f"### 🏆 Top Models based on your {weight}% Quality Preference")
  335. winners['Win_Score'] = winners['Win_Score'].round(1)
  336. disp_cols = ["Model_Name", "Win_Score", "Gen_TG128", "HE_Overall", "Quant", "Size_GB"]
  337.  
  338. st.dataframe(
  339. winners[disp_cols].head(15),
  340. hide_index=True,
  341. width="stretch",
  342. column_config={
  343. "Win_Score": st.column_config.ProgressColumn("Composite Score", format="%.1f", min_value=0, max_value=100),
  344. "Gen_TG128": st.column_config.NumberColumn("Speed (t/s)", format="%.1f"),
  345. "HE_Overall": st.column_config.NumberColumn("Quality", format="%.1f"),
  346. }
  347. )
  348.  
  349.  
  350.  
  351. with tab4:
  352. st.subheader("🆚 Version Comparison")
  353. st.info("Compare how different versions of Llama.cpp perform on the same model.")
  354.  
  355. df_hist = load_history()
  356. if df_hist.empty:
  357. st.warning("No history found. Run some benchmarks first!")
  358. else:
  359. # Filter logic
  360. models_in_hist = df_hist['Model_Name'].unique()
  361. selected_model = st.selectbox("Select Model to Compare", models_in_hist)
  362.  
  363. if selected_model:
  364. model_data = df_hist[df_hist['Model_Name'] == selected_model].sort_values(by="Timestamp")
  365.  
  366. if len(model_data) < 2:
  367. st.warning("Need at least 2 benchmark runs for this model to compare.")
  368. st.dataframe(model_data)
  369. else:
  370. # Chart
  371. st.line_chart(model_data, x="Llama_Version", y="Speed")
  372.  
  373. # Pivot for easier reading
  374. st.markdown("### 📋 History Log")
  375. st.dataframe(
  376. model_data[["Llama_Version", "Speed", "Speed_StDev", "Timestamp", "Backend"]].style.highlight_max(axis=0, subset=["Speed"]),
  377. use_container_width=True,
  378. hide_index=True
  379. )
  380.  
  381. # Calcs
  382. best = model_data.loc[model_data['Speed'].idxmax()]
  383. worst = model_data.loc[model_data['Speed'].idxmin()]
  384. diff = best['Speed'] - worst['Speed']
  385. pct_diff = (diff / worst['Speed']) * 100 if worst['Speed'] > 0 else 0
  386.  
  387. st.metric(
  388. label=f"Best Version ({best['Llama_Version']})",
  389. value=f"{best['Speed']} t/s",
  390. delta=f"{pct_diff:.1f}% vs {worst['Llama_Version']}"
  391. )
  392.  
  393. with tab5:
  394. st.subheader("Benchmarking Station")
  395. if not df.empty:
  396. model = st.selectbox("Select Model", df['Model_Name'].unique())
  397.  
  398. if st.button(f"🔥 Run Benchmark: {model}"):
  399. now = time.time()
  400. if now - st.session_state.last_bench_time < 2.0:
  401. st.warning("⏳ Too fast!")
  402. else:
  403. st.session_state.last_bench_time = now
  404. with st.spinner("Running llama-bench..."):
  405. data = run_benchmark(model)
  406.  
  407. if "error" in data:
  408. st.error(f"Failed: {data['error']}")
  409. else:
  410. tg = data['tg']
  411. rating = "SLOW 🔴"
  412. if tg > 10: rating = "USABLE 🟡"
  413. if tg > 25: rating = "SMOOTH 🟢"
  414. if tg > 50: rating = "GODLIKE 🚀"
  415.  
  416. ver_str = data['version']
  417. st.success(f"Speed: {tg} t/s (±{data['tg_std']}) | Ver: {ver_str}")
  418.  
  419. idx = df.index[df['Model_Name'] == model].tolist()[0]
  420. df.at[idx, 'Gen_TG128'] = tg
  421. df.at[idx, 'Gen_TG128_StDev'] = data['tg_std']
  422. df.at[idx, 'Ingest_PP512'] = data['pp']
  423. df.at[idx, 'Rating_Speed'] = rating
  424. df.at[idx, 'Backend'] = data['backend']
  425. df.at[idx, 'Params_Actual'] = data['params']
  426. df.at[idx, 'Params_Actual'] = data['params']
  427. df.at[idx, 'Llama_Version'] = ver_str
  428. save_db(df)
  429.  
  430. # Append to History
  431. append_history(model, ver_str, tg, data['tg_std'], data['backend'])
  432.  
  433. with st.expander("📄 View Raw Benchmark Output", expanded=True):
  434. st.code(data['raw'], language="text")
  435. st.caption(f"Log saved to: {LOGS_DIR}")
  436. else:
  437. st.warning("Library is empty.")
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!