Guest User

Untitled

a guest
Aug 11th, 2026
88
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 38.62 KB | None | 0 0
  1. # ПЕРВЫЙ ФАЙЛ
  2.  
  3. # SPDX-License-Identifier: GPL-3.0-or-later
  4. """Runtime patches for MiniMax H3 continuation conditioning.
  5.  
  6. Inspired by NikoDemon80/ComfyUI-H3-Motion-Context.
  7.  
  8. The patch is deliberately narrow:
  9. * allow motion-context keyframes at arbitrary pixel-frame positions by
  10. rewriting only PackedLayout.position_ids after stock construction;
  11. * keep all keyframe + reference visual latents in payload order;
  12. * optionally move one marked audio-reference block onto the target timeline.
  13.  
  14. No ComfyUI files are modified on disk.
  15. """
  16. from __future__ import annotations
  17.  
  18. import logging
  19. from typing import Any, Iterable
  20.  
  21. import torch
  22.  
  23. import comfy.ldm.minimax.model as mm
  24. import comfy.model_base as model_base
  25.  
  26. LOG = logging.getLogger("h3_continuation_context")
  27.  
  28. MC_INDEX = "h3_continuation_pixel_index"
  29. MC_AUDIO_END = "h3_continuation_audio_end_frame"
  30.  
  31. _layout_original = None
  32. _payload_original = None
  33. _layout_applied = False
  34. _payload_applied = False
  35.  
  36.  
  37. def _ref_cursor_advance(refs: Iterable[dict] | None) -> float:
  38. """Reproduce PackedLayout's reference cursor advance exactly."""
  39. if not refs:
  40. return 0.0
  41. cursor = 0.0
  42. for blk in refs:
  43. kind = blk.get("kind")
  44. if kind == "image":
  45. cursor += 1.0
  46. elif kind == "audio":
  47. cursor += float(blk.get("ref_audio_t", 0))
  48. elif kind in ("video", "video_audio"):
  49. rt = float(blk.get("ref_audio_t", 0))
  50. vt = int(blk.get("latent_t", 0))
  51. cursor += max(rt, sum(mm._video_t_spans(vt)))
  52. return cursor
  53.  
  54.  
  55. def _cond_time(text_len: int, latent_t: int, frame_count: int | None, pixel_index: float) -> float:
  56. """Time coordinate for a visual condition at a pixel-frame index."""
  57. p = float(pixel_index)
  58. if p == 0.0:
  59. return float(text_len)
  60. if frame_count is not None and p == float(frame_count - 1):
  61. return float(text_len) + sum(mm._video_t_spans(latent_t)) - mm.FRAME_RESCALE
  62. return float(text_len) + mm.FRAME_RESCALE * p
  63.  
  64.  
  65. def _fix_visual_positions(layout, text_len, latent_t, frame_count, keyframes, refs=None):
  66. """Align all keyframe condition rows to the target timeline.
  67.  
  68. Motion-context keyframes carry MC_INDEX. Stock first/last keyframes are
  69. left at their stock coordinate when there are no refs; when refs coexist,
  70. they receive the same reference-cursor offset as the target so they remain
  71. aligned instead of sliding backward relative to target frame 0.
  72. """
  73. cond_spans = [(a, b) for a, b, kind in layout.segments if kind == "cond"]
  74. if len(cond_spans) != len(keyframes):
  75. raise RuntimeError(
  76. f"h3_continuation_context: expected {len(keyframes)} cond segments, "
  77. f"layout has {len(cond_spans)}"
  78. )
  79.  
  80. offset = _ref_cursor_advance(refs)
  81. for (a, b), kf in zip(cond_spans, keyframes):
  82. custom = kf.get(MC_INDEX)
  83. if custom is not None:
  84. t = _cond_time(text_len, latent_t, frame_count, custom) + offset
  85. layout.position_ids[a:b, 0] = t
  86. elif offset:
  87. # Stock first/last keyframe + refs: preserve its stock position
  88. # relative to the target by shifting it by the ref cursor advance.
  89. layout.position_ids[a:b, 0] += offset
  90.  
  91.  
  92. def _ref_segment_spans(layout, refs):
  93. """Map each ref object to the PackedLayout segment(s) it created.
  94.  
  95. Returns list[(ref, [(a,b,kind), ...])]. This avoids selecting audio rows by
  96. a global coordinate range and therefore works when normal REF2VA image/video
  97. refs coexist with the continuation audio ref.
  98. """
  99. segs = [(a, b, kind) for a, b, kind in layout.segments if kind in ("ref_img", "ref_audio")]
  100. idx = 0
  101. mapped = []
  102. for ref in refs or []:
  103. kind = ref.get("kind")
  104. expect: list[str] = []
  105. if kind == "image":
  106. expect = ["ref_img"]
  107. elif kind == "audio":
  108. if int(ref.get("ref_audio_t", 0)) > 0:
  109. expect = ["ref_audio"]
  110. elif kind in ("video", "video_audio"):
  111. if int(ref.get("ref_audio_t", 0)) > 0:
  112. expect.append("ref_audio")
  113. expect.append("ref_img")
  114. owned = []
  115. for wanted in expect:
  116. if idx >= len(segs):
  117. raise RuntimeError("h3_continuation_context: ref segment table ended early")
  118. seg = segs[idx]
  119. idx += 1
  120. if seg[2] != wanted:
  121. raise RuntimeError(
  122. f"h3_continuation_context: expected ref segment {wanted}, got {seg[2]}"
  123. )
  124. owned.append(seg)
  125. mapped.append((ref, owned))
  126. if idx != len(segs):
  127. raise RuntimeError(
  128. f"h3_continuation_context: layout has {len(segs) - idx} unmapped ref segment(s)"
  129. )
  130. return mapped
  131.  
  132.  
  133. def _fix_audio_positions(layout, text_len, refs):
  134. """Move marked continuation audio from reference time onto target time."""
  135. if not refs:
  136. return
  137. marked = [r for r in refs if r.get(MC_AUDIO_END) is not None]
  138. if not marked:
  139. return
  140. if len(marked) != 1:
  141. raise RuntimeError(
  142. f"h3_continuation_context: expected one marked continuation audio ref, got {len(marked)}"
  143. )
  144. mark = marked[0]
  145. if mark.get("kind") != "audio":
  146. raise RuntimeError("h3_continuation_context: timeline marker may only be used on an audio ref")
  147. rt = int(mark.get("ref_audio_t", 0))
  148. if rt <= 0:
  149. return
  150.  
  151. owned = None
  152. for ref, spans in _ref_segment_spans(layout, refs):
  153. if ref is mark:
  154. owned = spans
  155. break
  156. if not owned or len(owned) != 1 or owned[0][2] != "ref_audio":
  157. raise RuntimeError("h3_continuation_context: could not locate marked audio ref segment")
  158.  
  159. a, b, _ = owned[0]
  160. t = layout.position_ids[a:b, 0]
  161. old_start = float(t.min())
  162. target_origin = float(text_len) + _ref_cursor_advance(refs)
  163. end_frame = float(mark[MC_AUDIO_END])
  164. new_start = target_origin + mm.FRAME_RESCALE * end_frame - float(rt)
  165. layout.position_ids[a:b, 0] = t + (new_start - old_start)
  166.  
  167.  
  168. def _patched_layout_init(self, text_len, latent_t, latent_h, latent_w, audio_t,
  169. keyframes=None, refs=None, frame_count=None):
  170. _layout_original(
  171. self, text_len, latent_t, latent_h, latent_w, audio_t,
  172. keyframes=keyframes, refs=refs, frame_count=frame_count,
  173. )
  174. if keyframes:
  175. has_custom = any(kf.get(MC_INDEX) is not None for kf in keyframes)
  176. # Ref coexistence also needs stock first/last anchors shifted with target.
  177. if has_custom or refs:
  178. _fix_visual_positions(self, text_len, latent_t, frame_count, keyframes, refs)
  179. if refs and any(r.get(MC_AUDIO_END) is not None for r in refs):
  180. _fix_audio_positions(self, text_len, refs)
  181.  
  182.  
  183. _patched_layout_init._h3_continuation_context_patch = True
  184.  
  185.  
  186. def _layout_self_test():
  187. text_len, latent_t, lh, lw, audio_t = 7, 7, 22, 38, 37
  188. frame_count = sum(mm.FRAME_PER_TOKEN[k % 5] for k in range(latent_t))
  189.  
  190. # Endpoint equivalence: our general mechanism must reproduce stock exactly.
  191. stock = [
  192. {"resolved_frame_index": 0},
  193. {"resolved_frame_index": frame_count - 1},
  194. ]
  195. ours = [
  196. {"resolved_frame_index": 0, MC_INDEX: 0},
  197. {"resolved_frame_index": 0, MC_INDEX: frame_count - 1},
  198. ]
  199. a = mm.PackedLayout.__new__(mm.PackedLayout)
  200. _layout_original(a, text_len, latent_t, lh, lw, audio_t,
  201. keyframes=stock, frame_count=frame_count)
  202. b = mm.PackedLayout.__new__(mm.PackedLayout)
  203. _layout_original(b, text_len, latent_t, lh, lw, audio_t,
  204. keyframes=ours, frame_count=frame_count)
  205. _fix_visual_positions(b, text_len, latent_t, frame_count, ours)
  206. if not torch.equal(a.position_ids, b.position_ids):
  207. raise RuntimeError("endpoint position self-test failed")
  208.  
  209. # Interior anchors must be strictly ordered and follow FRAME_RESCALE * frame.
  210. run_indices = [0, 1, 5, 9]
  211. run = [{"resolved_frame_index": 0, MC_INDEX: p} for p in run_indices]
  212. c = mm.PackedLayout.__new__(mm.PackedLayout)
  213. _layout_original(c, text_len, latent_t, lh, lw, audio_t,
  214. keyframes=run, frame_count=frame_count)
  215. _fix_visual_positions(c, text_len, latent_t, frame_count, run)
  216. spans = [(x, y) for x, y, k in c.segments if k == "cond"]
  217. got = [float(c.position_ids[x, 0]) for x, _ in spans]
  218. want = [_cond_time(text_len, latent_t, frame_count, p) for p in run_indices]
  219. if any(abs(g - w) > 1e-9 for g, w in zip(got, want)):
  220. raise RuntimeError(f"interior position self-test failed: {got} vs {want}")
  221.  
  222. # Refs shift target origin. Both custom and stock anchors must shift equally.
  223. refs = [{"kind": "image", "latent_h": 8, "latent_w": 8}]
  224. d = mm.PackedLayout.__new__(mm.PackedLayout)
  225. _layout_original(d, text_len, latent_t, lh, lw, audio_t,
  226. keyframes=run, refs=refs, frame_count=frame_count)
  227. _fix_visual_positions(d, text_len, latent_t, frame_count, run, refs)
  228. shifted = [float(d.position_ids[x, 0]) for x, _ in [(x,y) for x,y,k in d.segments if k=="cond"]]
  229. off = _ref_cursor_advance(refs)
  230. if any(abs((s - g) - off) > 1e-9 for s, g in zip(shifted, got)):
  231. raise RuntimeError("reference cursor compensation self-test failed")
  232.  
  233. # Audio marker may coexist with ordinary image refs.
  234. refs2 = [
  235. {"kind": "image", "latent_h": 8, "latent_w": 8},
  236. {"kind": "audio", "ref_audio_t": 8, MC_AUDIO_END: 22.0},
  237. {"kind": "image", "latent_h": 8, "latent_w": 8},
  238. ]
  239. e = mm.PackedLayout.__new__(mm.PackedLayout)
  240. _layout_original(e, text_len, latent_t, lh, lw, audio_t,
  241. keyframes=run, refs=refs2, frame_count=frame_count)
  242. before = e.position_ids.clone()
  243. _fix_visual_positions(e, text_len, latent_t, frame_count, run, refs2)
  244. after_visual = e.position_ids.clone()
  245. _fix_audio_positions(e, text_len, refs2)
  246. mapped = _ref_segment_spans(e, refs2)
  247. a0, b0, _ = mapped[1][1][0]
  248. changed = (after_visual[:, 0] != e.position_ids[:, 0]).nonzero().flatten().tolist()
  249. if changed != list(range(a0, b0)):
  250. raise RuntimeError("audio timeline self-test moved unexpected rows")
  251.  
  252.  
  253. def apply_layout_patch() -> bool:
  254. global _layout_original, _layout_applied
  255. if _layout_applied:
  256. return True
  257. current = getattr(mm.PackedLayout, "__init__", None)
  258. if current is None:
  259. LOG.warning("PackedLayout.__init__ not found; continuation patch disabled")
  260. return False
  261. if getattr(current, "_h3_continuation_context_patch", False):
  262. _layout_applied = True
  263. return True
  264.  
  265. # Avoid silently stacking on a foreign layout patch. This is especially
  266. # important if the original H3-Motion-Context package is installed too.
  267. if getattr(current, "__module__", "") != mm.__name__:
  268. LOG.warning(
  269. "PackedLayout.__init__ is already patched by %s; refusing to stack "
  270. "H3 continuation layout patches. Disable the other H3 layout patch.",
  271. getattr(current, "__module__", "unknown"),
  272. )
  273. return False
  274.  
  275. _layout_original = current
  276. try:
  277. _layout_self_test()
  278. except Exception as exc:
  279. _layout_original = None
  280. LOG.exception("H3 continuation layout self-test failed: %s", exc)
  281. return False
  282. mm.PackedLayout.__init__ = _patched_layout_init
  283. _layout_applied = True
  284. LOG.info("h3_continuation_context: interior visual anchors enabled")
  285. return True
  286.  
  287.  
  288. def _patched_extra_conds(self, **kwargs: Any):
  289. out = _payload_original(self, **kwargs)
  290. keyframes = kwargs.get("minimax_keyframes")
  291. refs = kwargs.get("minimax_refs")
  292. if not keyframes or not refs:
  293. return out
  294. holder = out.get("minimax_payload")
  295. payload = getattr(holder, "cond", None) if holder is not None else None
  296. if not isinstance(payload, dict):
  297. LOG.warning("could not reach MiniMax H3 payload; keyframe/ref merge skipped")
  298. return out
  299.  
  300. payload["cond_video_latents"] = (
  301. [kf["latent"] for kf in keyframes if kf.get("latent") is not None]
  302. + [r["latent"] for r in refs if r.get("latent") is not None]
  303. )
  304. payload["cond_audio_latents"] = [
  305. r["audio_latent"] for r in refs if r.get("audio_latent") is not None
  306. ]
  307. fc = kwargs.get("minimax_frame_count")
  308. if fc is not None:
  309. payload["frame_count"] = fc
  310. return out
  311.  
  312.  
  313. _patched_extra_conds._h3_continuation_payload_patch = True
  314.  
  315.  
  316. def apply_payload_patch() -> bool:
  317. global _payload_original, _payload_applied
  318. if _payload_applied:
  319. return True
  320. cls = getattr(model_base, "MiniMaxH3", None)
  321. if cls is None or not hasattr(cls, "extra_conds"):
  322. LOG.warning("MiniMaxH3.extra_conds not found; keyframe/ref merge disabled")
  323. return False
  324. current = cls.extra_conds
  325. if getattr(current, "_h3_continuation_payload_patch", False):
  326. _payload_applied = True
  327. return True
  328. _payload_original = current
  329. cls.extra_conds = _patched_extra_conds
  330. _payload_applied = True
  331. LOG.info("h3_continuation_context: keyframe/reference coexistence enabled")
  332. return True
  333.  
  334.  
  335. def layout_patch_applied() -> bool:
  336. return _layout_applied
  337.  
  338.  
  339. def payload_patch_applied() -> bool:
  340. return _payload_applied
  341.  
  342.  
  343. # ВТОРОЙ ФАЙЛ
  344.  
  345. # SPDX-License-Identifier: GPL-3.0-or-later
  346. """MiniMax H3 continuation-context nodes.
  347.  
  348. The main node turns the tail of a previous clip into H3's native never-denoised
  349. visual conditioning rows at consecutive positions on the *new* clip timeline.
  350. This is fundamentally different from writing values into the target latent.
  351. """
  352. from __future__ import annotations
  353.  
  354. import logging
  355. from typing import Any
  356.  
  357. import torch
  358.  
  359. import comfy.nested_tensor
  360. import comfy.utils
  361.  
  362. from .h3_patches import (
  363. MC_AUDIO_END,
  364. MC_INDEX,
  365. layout_patch_applied,
  366. payload_patch_applied,
  367. )
  368.  
  369. LOG = logging.getLogger("h3_continuation_context")
  370.  
  371. FPS = 24
  372. AUDIO_HZ = 40.0
  373. FRAME_RESCALE = 5.0 / 3.0
  374. FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
  375. VIDEO_RUN_GRID = (39, 22, 5, 1)
  376.  
  377.  
  378. def _pixel_frames(latent_t: int) -> int:
  379. return sum(FRAME_PER_TOKEN[k % 5] for k in range(int(latent_t)))
  380.  
  381.  
  382. def _step_offsets(latent_t: int) -> list[int]:
  383. out, acc = [], 0
  384. for k in range(int(latent_t)):
  385. out.append(acc)
  386. acc += FRAME_PER_TOKEN[k % 5]
  387. return out
  388.  
  389.  
  390. def _active_run(requested: int, available: int, allow_one: bool = True) -> int:
  391. n = min(int(requested), int(available))
  392. grid = VIDEO_RUN_GRID if allow_one else VIDEO_RUN_GRID[:-1]
  393. for g in grid:
  394. if g <= n:
  395. return g
  396. raise ValueError(
  397. "h3_continuation_context: not enough context frames for a valid run; "
  398. f"need at least {grid[-1]}, have {available}"
  399. )
  400.  
  401.  
  402. def _streams(latent: dict) -> list[torch.Tensor]:
  403. if not isinstance(latent, dict) or "samples" not in latent:
  404. raise TypeError("h3_continuation_context: expected a ComfyUI LATENT dict")
  405. samples = latent["samples"]
  406. if hasattr(samples, "unbind"):
  407. parts = list(samples.unbind())
  408. elif isinstance(samples, (tuple, list)):
  409. parts = list(samples)
  410. else:
  411. raise TypeError(
  412. "h3_continuation_context: expected H3 NestedTensor(video,audio), got "
  413. + type(samples).__name__
  414. )
  415. if len(parts) < 2:
  416. raise ValueError("h3_continuation_context: H3 latent has fewer than 2 streams")
  417. return parts
  418.  
  419.  
  420. def _video(latent: dict) -> torch.Tensor:
  421. z = _streams(latent)[0]
  422. if z.ndim == 4:
  423. z = z.unsqueeze(0)
  424. if z.ndim != 5 or z.shape[1] != 24:
  425. raise ValueError(
  426. f"h3_continuation_context: expected video [B,24,T,H,W], got {tuple(z.shape)}"
  427. )
  428. return z
  429.  
  430.  
  431. def _audio(latent: dict) -> torch.Tensor:
  432. z = _streams(latent)[1]
  433. if z.ndim == 3:
  434. z = z.unsqueeze(0)
  435. if z.ndim != 4 or z.shape[1] != 32 or z.shape[2] != 2:
  436. raise ValueError(
  437. f"h3_continuation_context: expected audio [B,32,2,T], got {tuple(z.shape)}"
  438. )
  439. return z
  440.  
  441.  
  442. def _resize_frames(images: torch.Tensor, width: int, height: int, crop: str) -> torch.Tensor:
  443. if images.ndim != 4:
  444. raise ValueError(f"h3_continuation_context: IMAGE must be [N,H,W,C], got {tuple(images.shape)}")
  445. x = images[..., :3].movedim(-1, 1)
  446. x = comfy.utils.common_upscale(x, int(width), int(height), "lanczos", crop)
  447. return x.movedim(1, -1)
  448.  
  449.  
  450. def _encode_video_context(vae, frames: torch.Tensor, target_video: torch.Tensor, requested: int, crop: str):
  451. if vae is None:
  452. raise ValueError("h3_continuation_context: video_source=frames requires vae")
  453. available = int(frames.shape[0])
  454. run = _active_run(requested, available, allow_one=True)
  455. if run != min(int(requested), available):
  456. LOG.warning(
  457. "h3_continuation_context: context length snapped down to %d frames "
  458. "(usable video-VAE runs: 1, 5, 22, 39)", run,
  459. )
  460. width = int(target_video.shape[-1]) * 16
  461. height = int(target_video.shape[-2]) * 16
  462. tail = _resize_frames(frames[available - run:], width, height, crop)
  463. enc = vae.encode(tail)
  464. if getattr(enc, "ndim", 0) != 5 or int(enc.shape[1]) != 24:
  465. raise ValueError(
  466. "h3_continuation_context: VAE video encode returned "
  467. f"{tuple(getattr(enc, 'shape', ()))}, expected [B,24,T,H,W]"
  468. )
  469. if tuple(enc.shape[-2:]) != tuple(target_video.shape[-2:]):
  470. raise ValueError(
  471. "h3_continuation_context: encoded context spatial grid does not match target: "
  472. f"context={tuple(enc.shape[-2:])}, target={tuple(target_video.shape[-2:])}"
  473. )
  474. steps = int(enc.shape[2])
  475. covered = _pixel_frames(steps)
  476. if covered != run:
  477. raise RuntimeError(
  478. "h3_continuation_context: H3 VAE temporal grid changed: "
  479. f"{run} RGB frames encoded to {steps} latent steps covering {covered} frames"
  480. )
  481. blocks = [enc[:, :, k:k + 1] for k in range(steps)]
  482. return blocks, _step_offsets(steps), run, f"frames/VAE {run}f->{steps}t"
  483.  
  484.  
  485. def _latent_steps_for_run(run: int) -> int:
  486. # Valid continuation runs end on the same 5-token temporal phase as a full H3 clip.
  487. mapping = {5: 2, 22: 7, 39: 12}
  488. if run not in mapping:
  489. raise ValueError(
  490. "h3_continuation_context: direct latent video context supports 5, 22 or 39 frames"
  491. )
  492. return mapping[run]
  493.  
  494.  
  495. def _encode_latent_context(context_latent: dict, target_video: torch.Tensor, requested: int):
  496. prev = _video(context_latent)
  497. if tuple(prev.shape[-2:]) != tuple(target_video.shape[-2:]):
  498. raise ValueError(
  499. "h3_continuation_context: previous/target video latent spatial mismatch: "
  500. f"previous={tuple(prev.shape[-2:])}, target={tuple(target_video.shape[-2:])}"
  501. )
  502. available_frames = _pixel_frames(int(prev.shape[2]))
  503. run = _active_run(requested, available_frames, allow_one=False)
  504. steps = _latent_steps_for_run(run)
  505. if int(prev.shape[2]) < steps:
  506. raise ValueError(
  507. f"h3_continuation_context: previous latent has only {prev.shape[2]} temporal steps, need {steps}"
  508. )
  509. # H3 target latents have T = 5k+2. Taking 2/7/12 tail steps therefore starts
  510. # at phase 0, matching a fresh 5/22/39-frame VAE run.
  511. phase = (int(prev.shape[2]) - steps) % 5
  512. if phase != 0:
  513. raise ValueError(
  514. "h3_continuation_context: previous latent temporal phase is unexpected; "
  515. f"tail would start at phase {phase}, refusing direct latent context"
  516. )
  517. tail = prev[:1, :, -steps:].clone()
  518. blocks = [tail[:, :, k:k + 1] for k in range(steps)]
  519. return blocks, _step_offsets(steps), run, f"latent-direct {run}f/{steps}t"
  520.  
  521.  
  522. def _audio_tail_from_latent(latent: dict, a_frames: int):
  523. video = _video(latent)
  524. audio = _audio(latent)
  525. total_t = int(audio.shape[-1])
  526. frames = _pixel_frames(int(video.shape[2]))
  527. overhang = total_t - FRAME_RESCALE * frames
  528. if not (0.0 <= overhang < 1.0):
  529. LOG.warning(
  530. "h3_continuation_context: unexpected audio grid (%d steps for %d frames); "
  531. "assuming no overhang", total_t, frames,
  532. )
  533. overhang = 0.0
  534. rt = int(round(float(a_frames) / FPS * AUDIO_HZ))
  535. rt = min(rt, total_t)
  536. if rt < 1:
  537. raise ValueError("h3_continuation_context: audio context window is empty")
  538. return audio[:1, ..., total_t - rt:].clone(), rt, float(overhang)
  539.  
  540.  
  541. def _encode_audio_tail(audio_vae, audio: dict, a_frames: int):
  542. if audio_vae is None:
  543. raise ValueError("h3_continuation_context: context_audio requires audio_vae")
  544. waveform = audio["waveform"]
  545. sr = int(audio["sample_rate"])
  546. vae_sr = int(getattr(audio_vae, "audio_sample_rate", 32000))
  547. if sr != vae_sr:
  548. try:
  549. import torchaudio
  550. except ImportError as exc:
  551. raise RuntimeError(
  552. f"h3_continuation_context: audio resample {sr}->{vae_sr} needs torchaudio"
  553. ) from exc
  554. waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
  555. want = int(round(a_frames / FPS * vae_sr))
  556. if waveform.shape[-1] > want:
  557. waveform = waveform[..., -want:]
  558. z = audio_vae.encode(waveform[:1].movedim(1, -1))
  559. return z, int(z.shape[-1]), 0.0
  560.  
  561.  
  562. def _merge_conditioning(conditioning, keyframes, frame_count, audio_ref=None,
  563. preserve_existing=True):
  564. """Add H3 keyframes without silently deleting upstream FL2VA anchors.
  565.  
  566. The first version of this fork replaced ``minimax_keyframes`` outright.
  567. That meant putting this node after MiniMaxH3ImageToVideo discarded the
  568. stock first-frame keyframe before adding ours. For first-frame work that
  569. made the new path effectively no stronger than native FL2VA.
  570.  
  571. Existing keyframes are retained by default. The layout patch already
  572. compensates both stock and custom keyframes when references coexist, so
  573. mixed endpoint/interior anchors remain aligned to the target timeline.
  574. """
  575. out = []
  576. for entry in conditioning:
  577. if not isinstance(entry, (list, tuple)) or len(entry) != 2:
  578. raise TypeError("h3_continuation_context: malformed CONDITIONING entry")
  579. tensor, meta = entry
  580. meta = dict(meta)
  581.  
  582. existing = list(meta.get("minimax_keyframes") or []) if preserve_existing else []
  583. old_fc = meta.get("minimax_frame_count")
  584. if old_fc is not None and int(old_fc) != int(frame_count):
  585. raise ValueError(
  586. "h3_continuation_context: upstream minimax_frame_count does not match "
  587. f"target latent: conditioning={old_fc}, target={frame_count}"
  588. )
  589.  
  590. meta["minimax_keyframes"] = existing + list(keyframes)
  591. meta["minimax_frame_count"] = int(frame_count)
  592. if audio_ref is not None:
  593. refs = list(meta.get("minimax_refs") or [])
  594. refs.append(audio_ref)
  595. meta["minimax_refs"] = refs
  596. out.append([tensor, meta])
  597. return out
  598.  
  599.  
  600. def _conditioning_keyframe_count(conditioning) -> int:
  601. if not conditioning:
  602. return 0
  603. entry = conditioning[0]
  604. if not isinstance(entry, (list, tuple)) or len(entry) != 2:
  605. return 0
  606. meta = entry[1] if isinstance(entry[1], dict) else {}
  607. return len(meta.get("minimax_keyframes") or [])
  608.  
  609.  
  610. def _first_frame_blocks(vae, image: torch.Tensor, target_video: torch.Tensor,
  611. mode: str, repeats: int, crop: str):
  612. """Build one-image H3 conditioning blocks for a strong frame-0 anchor.
  613.  
  614. Modes:
  615. single one ordinary H3 first-frame block (native-equivalent)
  616. reinforce duplicate that same never-denoised block at time 0
  617. static_5 encode five copies as one H3 video run -> 2 blocks at 0,1
  618. static_22 encode 22 copies as one H3 video run -> 7 blocks
  619.  
  620. ``reinforce`` does not consume output time: all copies have the exact same
  621. frame-0 position. ``static_*`` deliberately pins a short static prefix and
  622. therefore is much stronger but can create an initial pause.
  623. """
  624. if vae is None:
  625. raise ValueError("h3_continuation_context: first-frame context requires vae")
  626. if image is None or getattr(image, "ndim", 0) != 4 or int(image.shape[0]) < 1:
  627. raise ValueError("h3_continuation_context: first_frame must contain at least one IMAGE")
  628.  
  629. image = image[:1]
  630. if mode in ("single", "reinforce"):
  631. blocks, _, _, _ = _encode_video_context(vae, image, target_video, 1, crop)
  632. block = blocks[0]
  633. count = 1 if mode == "single" else max(1, int(repeats))
  634. # Same clean condition repeated at the same temporal coordinate. This
  635. # increases its representation in attention without freezing later
  636. # target frames or writing into the denoised target latent.
  637. return [block for _ in range(count)], [0 for _ in range(count)], 1
  638.  
  639. if mode == "static_5":
  640. run = 5
  641. elif mode == "static_22":
  642. run = 22
  643. else:
  644. raise ValueError(f"h3_continuation_context: unknown first_frame_mode {mode!r}")
  645.  
  646. repeated = image.repeat(run, 1, 1, 1)
  647. blocks, offsets, span, _ = _encode_video_context(
  648. vae, repeated, target_video, run, crop
  649. )
  650. return blocks, offsets, span
  651.  
  652.  
  653. class MiniMaxH3ContinuationContext:
  654. @classmethod
  655. def INPUT_TYPES(cls):
  656. return {
  657. "required": {
  658. "conditioning": ("CONDITIONING",),
  659. "latent": ("LATENT",),
  660. "video_source": (["frames", "latent", "first_frame"], {"default": "frames"}),
  661. "context_length": ("INT", {"default": 22, "min": 1, "max": 39, "step": 1}),
  662. "anchor_mode": (["head", "before"], {"default": "head"}),
  663. "crop": (["center", "disabled"], {"default": "center"}),
  664. "audio_mode": (["timeline", "ref", "off"], {"default": "timeline"}),
  665. "audio_context_length": ("INT", {"default": 22, "min": 0, "max": 240, "step": 1}),
  666. },
  667. "optional": {
  668. "vae": ("VAE",),
  669. "context_frames": ("IMAGE",),
  670. "first_frame": ("IMAGE",),
  671. "context_latent": ("LATENT",),
  672. "context_audio": ("AUDIO",),
  673. "audio_vae": ("VAE",),
  674. },
  675. }
  676.  
  677. RETURN_TYPES = ("CONDITIONING", "INT", "STRING")
  678. RETURN_NAMES = ("conditioning", "trim_frames", "info")
  679. FUNCTION = "apply"
  680. CATEGORY = "MiniMax H3/Continuation"
  681. DESCRIPTION = (
  682. "Pins a first frame or a consecutive tail from the previous H3 clip as native "
  683. "never-denoised visual conditioning rows. Unlike a latent noise mask, this uses "
  684. "H3's own keyframe timeline conditioning."
  685. )
  686.  
  687. def apply(self, conditioning, latent, video_source, context_length, anchor_mode,
  688. crop, audio_mode, audio_context_length,
  689. vae=None, context_frames=None, first_frame=None, context_latent=None,
  690. context_audio=None, audio_vae=None):
  691. if not layout_patch_applied():
  692. raise RuntimeError(
  693. "h3_continuation_context: layout patch is not active. Check startup logs; "
  694. "another H3 layout patch may already be installed."
  695. )
  696.  
  697. target_video = _video(latent)
  698. latent_t = int(target_video.shape[2])
  699. frame_count = _pixel_frames(latent_t)
  700. width = int(target_video.shape[-1]) * 16
  701. height = int(target_video.shape[-2]) * 16
  702.  
  703. first_frame_source = video_source == "first_frame"
  704. effective_anchor_mode = anchor_mode
  705.  
  706. if video_source == "latent":
  707. if int(context_length) < 5:
  708. raise ValueError(
  709. "h3_continuation_context: direct latent context cannot represent a clean "
  710. "single RGB frame from the tail of a normal H3 clip. Use video_source=frames "
  711. "with context_length=1, or video_source=first_frame with an IMAGE + VAE."
  712. )
  713. if context_latent is None:
  714. raise ValueError("h3_continuation_context: video_source=latent requires context_latent")
  715. blocks, offsets, span, source_info = _encode_latent_context(
  716. context_latent, target_video, int(context_length)
  717. )
  718. elif first_frame_source:
  719. if first_frame is None:
  720. raise ValueError("h3_continuation_context: video_source=first_frame requires first_frame")
  721. if int(first_frame.shape[0]) > 1:
  722. LOG.warning(
  723. "h3_continuation_context: first_frame contains %d images; using only the first",
  724. int(first_frame.shape[0]),
  725. )
  726. blocks, offsets, span, source_info = _encode_video_context(
  727. vae, first_frame[:1], target_video, 1, crop
  728. )
  729. # A first-frame anchor always belongs at target frame 0. Unlike a
  730. # continuation head, it is part of the requested output and must not
  731. # be advertised for trimming.
  732. effective_anchor_mode = "head"
  733. source_info = "first-frame/VAE 1f->1t"
  734. else:
  735. if context_frames is None:
  736. raise ValueError("h3_continuation_context: video_source=frames requires context_frames")
  737. blocks, offsets, span, source_info = _encode_video_context(
  738. vae, context_frames, target_video, int(context_length), crop
  739. )
  740.  
  741. if span >= frame_count:
  742. raise ValueError(
  743. f"h3_continuation_context: context span {span} must be smaller than target clip {frame_count}"
  744. )
  745.  
  746. indices = [o - span for o in offsets] if effective_anchor_mode == "before" else list(offsets)
  747. keyframes = [
  748. {
  749. "resolved_frame_index": 0, # always legal to stock constructor
  750. MC_INDEX: int(pixel_index),
  751. "latent": block,
  752. }
  753. for pixel_index, block in zip(indices, blocks)
  754. ]
  755.  
  756. audio_ref = None
  757. audio_info = "off"
  758. if first_frame_source and audio_mode != "off":
  759. audio_info = "off (first_frame source)"
  760. elif audio_mode != "off" and (context_latent is not None or context_audio is not None):
  761. if not payload_patch_applied():
  762. raise RuntimeError(
  763. "h3_continuation_context: payload patch is not active; audio refs would "
  764. "overwrite visual keyframe latents"
  765. )
  766. a_frames = int(audio_context_length) or span
  767. if context_latent is not None:
  768. audio_latent, rt, overhang = _audio_tail_from_latent(context_latent, a_frames)
  769. audio_info = f"latent {a_frames}f->{rt}t"
  770. else:
  771. audio_latent, rt, overhang = _encode_audio_tail(audio_vae, context_audio, a_frames)
  772. audio_info = f"decoded/VAE {a_frames}f->{rt}t"
  773. audio_ref = {
  774. "kind": "audio",
  775. "ref_audio_t": int(rt),
  776. "audio_latent": audio_latent,
  777. }
  778. if audio_mode == "timeline":
  779. end_frame = float(span if effective_anchor_mode == "head" else 0)
  780. end_frame += float(overhang) / FRAME_RESCALE
  781. audio_ref[MC_AUDIO_END] = end_frame
  782. audio_info += f" timeline@end={end_frame:.3f}f"
  783. else:
  784. audio_info += " stock-ref"
  785.  
  786. out = _merge_conditioning(conditioning, keyframes, frame_count, audio_ref)
  787. trim = 0 if first_frame_source else int(span if effective_anchor_mode == "head" else 0)
  788. info = (
  789. f"H3 continuation: {source_info}; anchor={effective_anchor_mode}; "
  790. f"indices={indices[0]}..{indices[-1]}; target={width}x{height}, "
  791. f"{frame_count}f/{latent_t}t; trim={trim}; audio={audio_info}"
  792. )
  793. LOG.info(info)
  794. return out, trim, info
  795.  
  796.  
  797. class MiniMaxH3FirstFrameContext:
  798. """Strengthen a supplied first image using H3's native conditioning rows.
  799.  
  800. This is intentionally separate from continuation context. It keeps any
  801. existing FL2VA first/last keyframes already present in CONDITIONING and
  802. adds extra never-denoised frame-0 context instead of replacing them.
  803. """
  804.  
  805. @classmethod
  806. def INPUT_TYPES(cls):
  807. return {
  808. "required": {
  809. "conditioning": ("CONDITIONING",),
  810. "latent": ("LATENT",),
  811. "vae": ("VAE",),
  812. "first_frame": ("IMAGE",),
  813. "first_frame_mode": (["reinforce", "single", "static_5", "static_22"],
  814. {"default": "reinforce"}),
  815. "reinforce_copies": ("INT", {"default": 2, "min": 1, "max": 8, "step": 1}),
  816. "crop": (["center", "disabled"], {"default": "center"}),
  817. "preserve_existing_keyframes": ("BOOLEAN", {"default": True}),
  818. }
  819. }
  820.  
  821. RETURN_TYPES = ("CONDITIONING", "STRING")
  822. RETURN_NAMES = ("conditioning", "info")
  823. FUNCTION = "apply"
  824. CATEGORY = "MiniMax H3/Continuation"
  825. DESCRIPTION = (
  826. "Strengthens H3 first-frame conditioning. Reinforce duplicates the same "
  827. "never-denoised frame-0 condition without consuming timeline frames; "
  828. "static modes encode one image as a short repeated video context."
  829. )
  830.  
  831. def apply(self, conditioning, latent, vae, first_frame, first_frame_mode,
  832. reinforce_copies=2, crop="center", preserve_existing_keyframes=True):
  833. if not layout_patch_applied():
  834. raise RuntimeError(
  835. "h3_continuation_context: layout patch is not active. Check startup logs."
  836. )
  837.  
  838. target_video = _video(latent)
  839. latent_t = int(target_video.shape[2])
  840. frame_count = _pixel_frames(latent_t)
  841. width = int(target_video.shape[-1]) * 16
  842. height = int(target_video.shape[-2]) * 16
  843.  
  844. if int(first_frame.shape[0]) > 1:
  845. LOG.warning(
  846. "h3_continuation_context: first_frame batch has %d images; using the first",
  847. int(first_frame.shape[0]),
  848. )
  849.  
  850. blocks, indices, span = _first_frame_blocks(
  851. vae, first_frame, target_video, first_frame_mode,
  852. int(reinforce_copies), crop,
  853. )
  854. if span >= frame_count:
  855. raise ValueError(
  856. f"h3_continuation_context: first-frame context span {span} must be "
  857. f"smaller than target clip {frame_count}"
  858. )
  859.  
  860. keyframes = [
  861. {
  862. "resolved_frame_index": 0,
  863. MC_INDEX: int(pixel_index),
  864. "latent": block,
  865. }
  866. for pixel_index, block in zip(indices, blocks)
  867. ]
  868.  
  869. before = _conditioning_keyframe_count(conditioning)
  870. out = _merge_conditioning(
  871. conditioning, keyframes, frame_count, audio_ref=None,
  872. preserve_existing=bool(preserve_existing_keyframes),
  873. )
  874. after = _conditioning_keyframe_count(out)
  875.  
  876. info = (
  877. f"H3 first frame: mode={first_frame_mode}; added={len(keyframes)}; "
  878. f"indices={indices}; existing_before={before}; total_after={after}; "
  879. f"preserve_existing={bool(preserve_existing_keyframes)}; "
  880. f"target={width}x{height}, {frame_count}f/{latent_t}t"
  881. )
  882. if first_frame_mode.startswith("static_"):
  883. info += f"; static_prefix={span}f (not trimmed)"
  884. elif first_frame_mode == "reinforce":
  885. info += "; all added blocks share frame-0 time (no frozen prefix)"
  886. LOG.info(info)
  887. return out, info
  888.  
  889.  
  890. class MiniMaxH3ContinuationTrim:
  891. @classmethod
  892. def INPUT_TYPES(cls):
  893. return {
  894. "required": {
  895. "images": ("IMAGE",),
  896. "trim_frames": ("INT", {"default": 0, "min": 0, "max": 4096}),
  897. },
  898. "optional": {
  899. "audio": ("AUDIO",),
  900. "fps": ("FLOAT", {"default": 24.0, "min": 1.0, "max": 240.0, "step": 0.001}),
  901. "match_tail": ("BOOLEAN", {"default": True}),
  902. },
  903. }
  904. RETURN_TYPES = ("IMAGE", "AUDIO", "STRING")
  905. RETURN_NAMES = ("images", "audio", "info")
  906. FUNCTION = "trim"
  907. CATEGORY = "MiniMax H3/Continuation"
  908.  
  909. def trim(self, images, trim_frames, audio=None, fps=24.0, match_tail=True):
  910. n = max(0, int(trim_frames))
  911. total = int(images.shape[0])
  912. if n >= total:
  913. raise ValueError(f"h3_continuation_context: cannot trim {n} from {total} frames")
  914. out_images = images[n:] if n else images
  915. out_audio = audio
  916. msg = f"trimmed {n} head frames; {total - n} remain"
  917. if audio is not None:
  918. waveform = audio["waveform"]
  919. sr = int(audio["sample_rate"])
  920. cut = int(round(n / float(fps) * sr))
  921. if cut >= waveform.shape[-1]:
  922. raise ValueError("h3_continuation_context: audio head trim would leave no samples")
  923. waveform = waveform[..., cut:]
  924. tail_trimmed = 0
  925. if match_tail:
  926. want = int(round((total - n) / float(fps) * sr))
  927. if waveform.shape[-1] > want:
  928. tail_trimmed = int(waveform.shape[-1] - want)
  929. waveform = waveform[..., :want]
  930. out_audio = {"waveform": waveform, "sample_rate": sr}
  931. msg += f"; audio={waveform.shape[-1]/sr:.4f}s; tail_cut={tail_trimmed} samples"
  932. LOG.info("h3_continuation_context: %s", msg)
  933. return out_images, out_audio, msg
  934.  
  935.  
  936. class MiniMaxH3ContinuationInspect:
  937. @classmethod
  938. def INPUT_TYPES(cls):
  939. return {"required": {"latent": ("LATENT",)}}
  940. RETURN_TYPES = ("LATENT", "STRING")
  941. RETURN_NAMES = ("latent", "info")
  942. FUNCTION = "inspect"
  943. CATEGORY = "MiniMax H3/Continuation"
  944.  
  945. def inspect(self, latent):
  946. v, a = _video(latent), _audio(latent)
  947. frames = _pixel_frames(int(v.shape[2]))
  948. info = (
  949. f"video={tuple(v.shape)} -> RGB {v.shape[-1]*16}x{v.shape[-2]*16}, "
  950. f"frames={frames}; audio={tuple(a.shape)}; "
  951. f"patches(layout/payload)={layout_patch_applied()}/{payload_patch_applied()}"
  952. )
  953. LOG.info("h3_continuation_context: %s", info)
  954. return latent, info
Advertisement
Add Comment
Please, Sign In to add comment