Guest User

Untitled

a guest
Aug 29th, 2026
115
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 10.07 KB | None | 0 0
  1. # nodes.py
  2.  
  3. import logging
  4. import os
  5.  
  6. import comfy.lora
  7. import comfy.model_base
  8. import comfy.utils
  9. import folder_paths
  10.  
  11. from .mapping import map_fastvideo_module, split_payload_key
  12.  
  13.  
  14. _LORA_A = ".lora_A.weight"
  15. _LORA_B = ".lora_B.weight"
  16. _DIFF = ".diff"
  17. _DIFF_B = ".diff_b"
  18. _SET_WEIGHT = ".set_weight"
  19. _ALPHA = ".alpha"
  20.  
  21.  
  22. def _analyze_payload(keys):
  23. payloads = {}
  24. for key in keys:
  25. module, suffix = split_payload_key(key)
  26. payloads.setdefault(module, set()).add(suffix)
  27.  
  28. expected_patches = 0
  29. for module, suffixes in payloads.items():
  30. has_a = _LORA_A in suffixes
  31. has_b = _LORA_B in suffixes
  32. if has_a != has_b:
  33. raise ValueError(f"FastH3 adapter has an incomplete LoRA pair for {module}")
  34. if _ALPHA in suffixes and not has_a:
  35. raise ValueError(f"FastH3 adapter has alpha without a LoRA pair for {module}")
  36.  
  37. weight_payloads = int(has_a) + int(_DIFF in suffixes) + int(_SET_WEIGHT in suffixes)
  38. if weight_payloads > 1:
  39. raise ValueError(f"FastH3 adapter has conflicting weight payloads for {module}")
  40. if weight_payloads == 0 and _DIFF_B not in suffixes:
  41. raise ValueError(f"FastH3 adapter has no applicable payload for {module}")
  42. expected_patches += weight_payloads + int(_DIFF_B in suffixes)
  43. return payloads, expected_patches
  44.  
  45.  
  46. def _validate_adapter(metadata, payloads):
  47. adapter_format = (metadata or {}).get("format")
  48. if adapter_format is not None and adapter_format != "fastvideo-lora-v2":
  49. raise ValueError(
  50. f"Expected a FastVideo fastvideo-lora-v2 adapter, found {adapter_format!r}"
  51. )
  52.  
  53. base_model = (metadata or {}).get("base_model")
  54. if base_model is not None and base_model != "MiniMaxAI/MiniMax-H3":
  55. raise ValueError(f"FastH3 adapter targets {base_model!r}, not MiniMaxAI/MiniMax-H3")
  56.  
  57. replacement_count = int((metadata or {}).get("set_weight_tensors", "0"))
  58. vsa_modules = [module for module, suffixes in payloads.items() if _SET_WEIGHT in suffixes]
  59. if replacement_count or vsa_modules:
  60. raise ValueError(
  61. "This is a VSA FastH3 adapter. Native ComfyUI has no VSA-H3 attention backend or "
  62. "to_gate_compress parameters. Use the dense-datafree adapter with this node, or run "
  63. "the VSA adapter through FastVideo."
  64. )
  65.  
  66.  
  67. def _build_key_map(lora, payloads, model_state):
  68. key_map = {}
  69. for source_module, suffixes in payloads.items():
  70. target = map_fastvideo_module(source_module)
  71. target_weight = f"diffusion_model.{target.module}.weight"
  72. if target_weight not in model_state:
  73. raise ValueError(
  74. f"FastH3 target {target_weight} is absent from the loaded model. "
  75. "Use a non-pruned MiniMax-H3 FL2VA base checkpoint."
  76. )
  77.  
  78. if _DIFF_B in suffixes:
  79. if target.qkv_slot is not None:
  80. raise ValueError(f"Fused QKV bias deltas are unsupported for {source_module}")
  81. target_bias = f"diffusion_model.{target.module}.bias"
  82. if target_bias not in model_state:
  83. raise ValueError(f"FastH3 target {target_bias} is absent from the loaded model")
  84.  
  85. if target.qkv_slot is None:
  86. key_map[source_module] = target_weight
  87. continue
  88.  
  89. if _DIFF_B in suffixes or _SET_WEIGHT in suffixes:
  90. raise ValueError(f"Unsupported fused QKV payload for {source_module}")
  91. qkv_rows = int(model_state[target_weight].shape[0])
  92. if qkv_rows % 3:
  93. raise ValueError(f"ComfyUI QKV target has an invalid first dimension: {target_weight}")
  94. width = qkv_rows // 3
  95. if _LORA_B in suffixes and int(lora[source_module + _LORA_B].shape[0]) != width:
  96. raise ValueError(f"FastH3 LoRA output width does not match {target_weight}")
  97. key_map[source_module] = (target_weight, (0, target.qkv_slot * width, width))
  98. return key_map
  99.  
  100.  
  101. class FastH3LoRALoader:
  102. """Load the dense FastVideo FastH3 Preview v1 adapter into native MiniMax-H3."""
  103.  
  104. def __init__(self):
  105. self.loaded_lora = None
  106.  
  107. @classmethod
  108. def INPUT_TYPES(cls):
  109. return {
  110. "required": {
  111. "model": ("MODEL",),
  112. "lora_name": (folder_paths.get_filename_list("loras"),),
  113. "strength_model": (
  114. "FLOAT",
  115. {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01},
  116. ),
  117. }
  118. }
  119.  
  120. RETURN_TYPES = ("MODEL",)
  121. FUNCTION = "load_lora_model_only"
  122. CATEGORY = "model/loaders"
  123. DESCRIPTION = (
  124. "Loads FastVideo FastH3 Preview v1 dense adapters into native ComfyUI MiniMax-H3. "
  125. "VSA adapters and pruned AdaLN base checkpoints are intentionally rejected."
  126. )
  127.  
  128. def _load_file(self, lora_path):
  129. if self.loaded_lora is not None and self.loaded_lora[0] == lora_path:
  130. return self.loaded_lora[1], self.loaded_lora[2]
  131.  
  132. self.loaded_lora = None
  133. lora, metadata = comfy.utils.load_torch_file(
  134. lora_path, safe_load=True, return_metadata=True
  135. )
  136. self.loaded_lora = (lora_path, lora, metadata)
  137. return lora, metadata
  138.  
  139. def load_lora_model_only(self, model, lora_name, strength_model):
  140. if strength_model == 0:
  141. return (model,)
  142. if not isinstance(model.model, comfy.model_base.MiniMaxH3):
  143. raise ValueError("FastH3 LoRA Loader requires a native ComfyUI MiniMax-H3 model")
  144.  
  145. lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
  146. lora, metadata = self._load_file(lora_path)
  147. payloads, expected_patches = _analyze_payload(lora.keys())
  148. _validate_adapter(metadata, payloads)
  149.  
  150. model_state = model.model.state_dict()
  151. if "diffusion_model.adaln_t_table" in model_state:
  152. raise ValueError(
  153. "FastH3 Preview v1 targets full-width AdaLN weights. The loaded MiniMax-H3 base "
  154. "is a pruned/curve-basis checkpoint; load a non-pruned FL2VA checkpoint instead."
  155. )
  156.  
  157. key_map = _build_key_map(lora, payloads, model_state)
  158. patches = comfy.lora.load_lora(lora, key_map, log_missing=False)
  159. if len(patches) != expected_patches:
  160. raise RuntimeError(
  161. f"FastH3 adapter conversion produced {len(patches)} patches; "
  162. f"{expected_patches} were expected"
  163. )
  164.  
  165. patched_model = model.clone()
  166. applied = patched_model.add_patches(patches, strength_model)
  167. if len(applied) != len(patches):
  168. raise RuntimeError(
  169. f"ComfyUI accepted {len(applied)} of {len(patches)} FastH3 patches"
  170. )
  171. if metadata:
  172. patched_model.set_attachments("lora_metadata", metadata)
  173.  
  174. logging.info(
  175. "Loaded FastH3 dense adapter %s at strength %g (%d patches)",
  176. os.path.basename(lora_path),
  177. strength_model,
  178. len(patches),
  179. )
  180. return (patched_model,)
  181.  
  182.  
  183. NODE_CLASS_MAPPINGS = {"FastH3LoRALoader": FastH3LoRALoader}
  184. NODE_DISPLAY_NAME_MAPPINGS = {
  185. "FastH3LoRALoader": "FastH3 LoRA Loader (Dense)",
  186. }
  187.  
  188.  
  189. # mapping.py
  190.  
  191. """Name translation for FastVideo MiniMax-H3 adapters.
  192.  
  193. This module deliberately has no ComfyUI or torch imports so its mapping rules can
  194. be tested without loading either runtime.
  195. """
  196.  
  197. from dataclasses import dataclass
  198. import re
  199.  
  200.  
  201. PAYLOAD_SUFFIXES = (
  202. ".lora_A.weight",
  203. ".lora_B.weight",
  204. ".set_weight",
  205. ".diff_b",
  206. ".diff",
  207. ".alpha",
  208. )
  209.  
  210.  
  211. @dataclass(frozen=True)
  212. class ModuleTarget:
  213. module: str
  214. qkv_slot: int | None = None
  215.  
  216.  
  217. def split_payload_key(key: str) -> tuple[str, str]:
  218. for suffix in PAYLOAD_SUFFIXES:
  219. if key.endswith(suffix):
  220. return key[: -len(suffix)], suffix
  221. raise ValueError(f"unsupported FastH3 adapter tensor: {key}")
  222.  
  223.  
  224. def _map_block_rest(rest: str) -> ModuleTarget:
  225. qkv = re.fullmatch(r"attn\.to_([qkv])", rest)
  226. if qkv:
  227. return ModuleTarget("attn.qkv_proj", {"q": 0, "k": 1, "v": 2}[qkv.group(1)])
  228.  
  229. direct = {
  230. "attn.to_out.0": "attn.out_proj",
  231. "ff.net.0.proj": "mlp.fc1",
  232. "ff.net.2": "mlp.fc2",
  233. "adaln_proj.linear": "adaln_proj.linear",
  234. "norm1": "norm1",
  235. "norm2": "norm2",
  236. }
  237. target = direct.get(rest)
  238. if target is None:
  239. raise ValueError(f"unsupported FastVideo MiniMax-H3 block module: {rest}")
  240. return ModuleTarget(target)
  241.  
  242.  
  243. def map_fastvideo_module(module: str) -> ModuleTarget:
  244. boundary = {
  245. "proj_in": "video_patch_proj",
  246. "audio_proj_in": "audio_patch_proj",
  247. "context_embedder": "condition_proj",
  248. "time_embedder.linear_1": "time_embedder.proj_in",
  249. "time_embedder.linear_2": "time_embedder.proj_out",
  250. "norm_out.norm": "final_layer.norm",
  251. "norm_out.linear": "final_layer.adaln_proj.linear",
  252. "proj_out": "final_layer.video_out",
  253. "audio_proj_out": "final_layer.audio_out",
  254. }
  255. target = boundary.get(module)
  256. if target is not None:
  257. return ModuleTarget(target)
  258.  
  259. block = re.fullmatch(r"transformer_blocks\.(\d+)\.(.+)", module)
  260. if block:
  261. mapped = _map_block_rest(block.group(2))
  262. return ModuleTarget(f"blocks.{block.group(1)}.{mapped.module}", mapped.qkv_slot)
  263.  
  264. refiner = re.fullmatch(r"token_refiner\.refiner_blocks\.(\d+)\.(.+)", module)
  265. if refiner:
  266. mapped = _map_block_rest(refiner.group(2))
  267. if mapped.module.startswith("adaln_proj."):
  268. raise ValueError("MiniMax-H3 token-refiner blocks do not have AdaLN projections")
  269. return ModuleTarget(f"token_refiner.blocks.{refiner.group(1)}.{mapped.module}", mapped.qkv_slot)
  270.  
  271. raise ValueError(f"unsupported FastVideo MiniMax-H3 module: {module}")
  272.  
  273.  
  274. # __init__.py
  275.  
  276. from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
  277.  
  278. __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
  279.  
Advertisement
Add Comment
Please, Sign In to add comment