Guest User

Untitled

a guest
Nov 9th, 2024
37
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.38 KB | None | 0 0
  1. #!/usr/bin/env python3
  2. import os, io, zipfile, json
  3. import numpy as np
  4. from types import SimpleNamespace
  5.  
  6. import os
  7. import logging; logger = logging.getLogger()
  8.  
  9. ################################################################################
  10. def to_namspace(obj):
  11.     if   isinstance(obj, dict):
  12.         return SimpleNamespace(**{k: to_namspace(obj[k]) for k in obj})
  13.     elif isinstance(obj, list):
  14.         return [to_namspace(item) for item in obj]
  15.     elif isinstance(obj, tuple):
  16.         return tuple(to_namspace(item) for item in obj)
  17.     elif isinstance(obj, set):
  18.         return {to_namspace(item) for item in obj}
  19.     else:
  20.         return obj
  21.  
  22. ################################################################################
  23. class npEncoder(json.JSONEncoder):
  24.     def default(self, obj):
  25.         if isinstance(obj,   np.integer):
  26.             return int(obj)
  27.         elif isinstance(obj, np.floating):
  28.             return float(obj)
  29.         elif isinstance(obj, np.complex):
  30.             return complex(obj)
  31.         elif isinstance(obj, np.ndarray):
  32.             return list(obj)
  33.         elif isinstance(obj, AttributeDict):
  34.             return dict(obj)
  35.         elif isinstance(obj, types.SimpleNamespace):
  36.             return {k:self.default(v) for k,v in vars(obj).items()}
  37.         else:
  38.             return super(npEncoder, self).default(obj)
  39.  
  40. ################################################################################
  41.  
  42. class load_frame:
  43.  
  44.     def __init__(self, fname):
  45.         self._zipf = zipfile.ZipFile(fname, "r")
  46.         self.files = self._zipf.namelist()
  47.         logger.info(f"open {fname}")
  48.  
  49.     def __getitem__(self, key):
  50.         import io, json
  51.         ###############################
  52.         if f"{key}.json" in self.files:
  53.             return to_namspace(json.loads\
  54.             (io.BytesIO(self._zipf.read(f"{key}.json")).read().decode("utf-8")))
  55.         ###############################
  56.         if f"{key}.npy"  in self.files:
  57.             return np.load\
  58.             (io.BytesIO(self._zipf.read(f"{key}.npy")))
  59.         ###############################
  60.         if f"{key}.txt"  in self.files:
  61.             return self._zipf.read(f"{key}.txt").decode("utf-8")
  62.         ###############################
  63.         return None
  64.         #raise ValueError(f"invalid key: \"{key}\"")
  65.  
  66.     def __getattr__(self, key):
  67.         return self[key]
  68.  
  69.     def __contains__(self, key):
  70.         if f"{key}.json" in self.files:
  71.             return 3
  72.         if f"{key}.npy"  in self.files:
  73.             return 2
  74.         if f"{key}.txt"  in self.files:
  75.             return 1
  76.         return 0   
  77.  
  78. ################################################################################
  79.  
  80. def save_frame(fname, mode="w", **kwargs):
  81.     msg   = f"saving \"{fname}\".."
  82.     try:
  83.        
  84.         os.makedirs(os.path.dirname(fname), exist_ok=True)
  85.         #
  86.         with zipfile.ZipFile(fname, mode, compression=zipfile.ZIP_STORED) as zipf:
  87.             #
  88.             def save_entry(k, arg):
  89.                 if   isinstance(arg, np.ndarray):
  90.                     dump = io.BytesIO(); np.save(dump, arg)
  91.                     zipf.writestr(f"{k}.npy",
  92.                         data=dump.getbuffer().tobytes())
  93.                 elif isinstance(arg, str):
  94.                     zipf.writestr(f"{k}.txt",
  95.                         data=arg.encode("utf-8"))
  96.                 else:
  97.                     zipf.writestr(f"{k}.json",
  98.                         data=json.dumps(arg, cls=npEncoder, indent="\t").encode("utf-8"))
  99.             #
  100.             for k,arg in kwargs.items():
  101.                 save_entry(k, arg)
  102.             nsize = sum([zinfo.file_size for zinfo in zipf.filelist])
  103.            
  104.             szstr = " KMGT"
  105.             while nsize>1024:
  106.                 nsize /= 1024; szstr = szstr[1:]
  107.                
  108.             logger.info(f"{msg} ok ({nsize:.2f}{szstr[0]}B)")
  109.     except Exception as e:
  110.         logger.error(f"{msg} fail: \"{e}\"")
  111.         raise e
  112.  
  113. # Specify the entities to export
  114. __all__ = ["load_frame", "save_frame"]
  115.  
Advertisement
Add Comment
Please, Sign In to add comment