Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import argparse
- import numpy as np
- import os
- import shutil
- import torch
- import torch.nn.functional as F
- from safetensors.torch import safe_open, save_file
- class BinDataHandler():
- def __init__(self, data):
- self.data = data
- def get_tensor(self, key):
- return self.data[key]
- def read_tensors(file_path):
- if isinstance(file_path, dict):
- return BinDataHandler(file_path), set(file_path.keys())
- if file_path.endswith(".safetensors"):
- f = safe_open(file_path, framework="pt", device="cpu")
- return f, set(f.keys())
- if file_path.endswith(".bin"):
- data = torch.load(file_path, map_location=torch.device('cpu'))
- f = BinDataHandler(data)
- return f, set(data.keys())
- return None, None
- #TODO values?
- def combine_loras(tensor_map1, tensor_map2, tensor_map3, unet_strength=2.69, enc_strength=1, enc2_strength=1, peft_type="lora"):
- tensor_map = {}
- if peft_type == 'dora':
- for k in tensor_map1[1]:
- unet_strength=8
- enc_strength=1
- enc2_strength=1
- k2 = k
- if "lora_unet_up_blocks_0_upsamplers_0_conv" in k:
- k2 = k2.replace("lora_unet_up_blocks_0_upsamplers_0_conv", "diffusion_model_output_blocks_2_2_conv")
- if "lora_unet_up_blocks_1_upsamplers_0_conv" in k:
- k2 = k2.replace("lora_unet_up_blocks_1_upsamplers_0_conv", "diffusion_model_output_blocks_5_2_conv")
- if 'alpha' in k:
- tensor_map[k]=unet_strength*torch.ones([1])
- tensor_map[k2]=unet_strength*torch.ones([1])
- else:
- tensor_map[k]=tensor_map1[0].get_tensor(k)
- tensor_map[k2]=tensor_map1[0].get_tensor(k).clone()
- for k in tensor_map2[1]:
- if 'alpha' in k:
- tensor_map[k]=enc_strength*torch.ones([1])
- else:
- tensor_map[k]=tensor_map2[0].get_tensor(k)
- for k in tensor_map3[1]:
- if 'alpha' in k:
- tensor_map[k]=enc2_strength*torch.ones([1])
- else:
- tensor_map[k]=tensor_map3[0].get_tensor(k)
- else:
- for k in tensor_map1[1]:
- k2 = k
- if "lora_unet_up_blocks_0_upsamplers_0_conv" in k:
- k2 = k2.replace("lora_unet_up_blocks_0_upsamplers_0_conv", "diffusion_model_output_blocks_2_2_conv")
- if "lora_unet_up_blocks_1_upsamplers_0_conv" in k:
- k2 = k2.replace("lora_unet_up_blocks_1_upsamplers_0_conv", "diffusion_model_output_blocks_5_2_conv")
- if 'alpha' in k:
- tensor_map[k2]=torch.ones([1])
- tensor_map[k]=torch.ones([1])
- else:
- tensor_map[k2]=unet_strength*tensor_map1[0].get_tensor(k)
- tensor_map[k]=unet_strength*tensor_map1[0].get_tensor(k)
- for k in tensor_map2[1]:
- if 'alpha' in k:
- tensor_map[k]=torch.ones([1])
- else:
- tensor_map[k]=enc_strength* tensor_map2[0].get_tensor(k)
- for k in tensor_map3[1]:
- if 'alpha' in k:
- tensor_map[k]=torch.ones([1])
- else:
- tensor_map[k]=enc2_strength*tensor_map3[0].get_tensor(k)
- return tensor_map
- def main():
- # Parse command-line arguments
- parser = argparse.ArgumentParser(description='Merge two safetensor model files.')
- parser.add_argument('unet_model', type=str, help='The unet model safetensor file')
- parser.add_argument('encoder1_model', type=str, help='The encoder1 model safetensor file')
- parser.add_argument('encoder2_model', type=str, help='The encoder2 model safetensor file')
- parser.add_argument('output_model', type=str, help='The output merged model safetensor file')
- parser.add_argument('unet_strength', type=float, default=0.8, help='Strength of the unet')
- parser.add_argument('enc_strength', type=float, default=1, help='Strength of the unet')
- parser.add_argument('enc2_strength', type=float, default=1, help='Strength of the unet')
- args = parser.parse_args()
- tensor_map1 = read_tensors(args.unet_model)
- tensor_map2 = read_tensors(args.encoder1_model)
- tensor_map3 = read_tensors(args.encoder2_model)
- tensor_map = combine_loras(tensor_map1, tensor_map2, tensor_map3, args.unet_strength, args.enc_strength, args.enc_strength2)
- save_file(tensor_map, args.output_model)
- if __name__ == '__main__':
- main()
Advertisement
Add Comment
Please, Sign In to add comment