Advertisement
Guest User

Untitled

a guest
Jun 3rd, 2025
14
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.84 KB | None | 0 0
  1. import time
  2. import os
  3. import pickle
  4. from collections import defaultdict
  5.  
  6.  
  7. # os.makedirs(".wandb_cache", exist_ok=True)
  8.  
  9.  
  10. class WandbCache:
  11. def __init__(self, cache_dir=".wandb_cache"):
  12. self.cache_dir = cache_dir
  13. os.makedirs(cache_dir, exist_ok=True)
  14.  
  15. def get_cache_path(self, run_id, keys):
  16. # Sort keys to ensure consistent filenames
  17. key_str = "_".join(sorted(keys))
  18. filename = f"{run_id}_{key_str}.pkl"
  19. filename = filename.replace("/", "__")
  20. return os.path.join(self.cache_dir, filename)
  21.  
  22. def get_history(self, run, keys):
  23. cache_path = self.get_cache_path(run.id, keys)
  24.  
  25. # Try to load from cache first
  26. if os.path.exists(cache_path):
  27. try:
  28. with open(cache_path, 'rb') as f:
  29. cached_data = pickle.load(f)
  30. # print(f"Loading history from cache for run {run.id}")
  31. return cached_data
  32. except:
  33. # print(f"Failed to load cache for run {run.id}, fetching from wandb")
  34. pass
  35.  
  36. # If not in cache or cache invalid, fetch from wandb
  37. print(f"Fetching history from wandb for run {run.id}")
  38. history = run.history(keys=keys)
  39.  
  40. # Only save to cache if run is finished
  41. if run.state == "finished":
  42. try:
  43. with open(cache_path, 'wb') as f:
  44. pickle.dump(history, f)
  45. print(f"Saved history to cache for run {run.id}")
  46. except Exception as e:
  47. print(f"Failed to save cache for run {run.id}: {e}")
  48.  
  49. return history
  50.  
  51. # Initialize cache
  52. # wandb_cache = WandbCache()
  53.  
  54. # Use cache to get history
  55. # df = wandb_cache.get_history(run, [x_metric] + y_metrics)
  56.  
  57.  
  58.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement