generate_stable_video.py

import torch
from diffusers import UNetSpatioTemporalConditionModel, StableVideoDiffusionPipeline
from diffusers.utils import load_image, export_to_video

base_model_dir = "./stable-video-diffusion-img2vid-xt-1-1" # git clone https://huggingface.co/vdo/stable-video-diffusion-img2vid-xt-1-1
unet_finetuned_dir = "./unet-only-checkpoint-50000/unet" # The folder contains the weight-trained diffusion_pytorch_model.safetensors.
image_path = 'input_image.png' # https://i.imgur.com/uc8wfaX.png

# Load UNet model
unet = UNetSpatioTemporalConditionModel.from_pretrained(
    unet_finetuned_dir,
    subfolder="unet",
    torch_dtype=torch.float16,
    low_cpu_mem_usage=False,
)

# Load Stable Video Diffusion Pipeline
pipe = StableVideoDiffusionPipeline.from_pretrained(
    base_model_dir,
    unet=unet,
    low_cpu_mem_usage=False,
    torch_dtype=torch.float16,
    variant="fp16",
    local_files_only=True,
)

# Move pipeline to CUDA
pipe.to("cuda:0")

# Load and preprocess input image
image = load_image(image_path)
image = image.resize((1024, 576))  # Resize to 1024x576

# Set random seed for reproducibility
generator = torch.manual_seed(-1)

# Generate video frames
with torch.inference_mode():
    frames = pipe(
        image,
        num_frames=14,
        width=1024,
        height=576,
        decode_chunk_size=8,
        generator=generator,
        motion_bucket_id=127,
        fps=8,
        num_inference_steps=30
    ).frames[0]

# Export generated frames to video
export_to_video(frames, "generated.mp4", fps=7)