ZIB_Prodigy

# Change these paths
output_dir =  '/mnt/m/Training/secreLORA/out/zimage'
dataset = '/home/dreamerwsl/newDIFFZIMAGE/diffusion-pipe/examples/dataset.toml'

# training settings
epochs = 1000
micro_batch_size_per_gpu = 1
pipeline_stages = 1
gradient_accumulation_steps = 4
gradient_clipping = 1
#warmup_steps = 100

# eval settings
eval_every_n_epochs = 5
#eval_every_n_steps = 100
eval_before_first_step = true
eval_micro_batch_size_per_gpu = 1
eval_gradient_accumulation_steps = 1

# misc settings
save_every_n_epochs = 5
#save_every_n_steps = 500
#checkpoint_every_n_epochs = 1
checkpoint_every_n_minutes = 120
activation_checkpointing = true
partition_method = 'parameters'
save_dtype = 'bfloat16'
caching_batch_size = 8
steps_per_print = 1
blocks_to_swap = 20

[model]
type = 'z_image'
diffusion_model = '/mnt/m/new_ComfyUI_windows_portable_nvidia_cu121_or_cpu/ComfyUI_windows_portable/ComfyUI/models/diffusion_models/z_image_fp32.safetensors'
vae = '/mnt/m/new_ComfyUI_windows_portable_nvidia_cu121_or_cpu/ComfyUI_windows_portable/ComfyUI/models/vae/ae.safetensors'
text_encoders = [
    {path = '/mnt/m/new_ComfyUI_windows_portable_nvidia_cu121_or_cpu/ComfyUI_windows_portable/ComfyUI/models/clip/qwen_3_4b.safetensors', type = 'lumina2'}
]


dtype = 'float32'


# For models that support full fine tuning, simply delete or comment out the [adapter] table to FFT.
[adapter]
type = 'lora'
rank = 64
# Dtype for the LoRA weights you are training.
dtype = 'float32'
# You can initialize the lora weights from a previously trained lora.
# init_from_existing = '/data/diffusion_pipe_training_runs/wan14b_t2v_video_loras/20250419_13-30-55/epoch60'





# Use if training Z-Image-Turbo

# You can reduce VRAM by having most weights in fp8 (small quality loss).
#diffusion_model_dtype = 'float8'
# Automagic should work fine and requires no tuning, but you can probably get better results using AdamW
# with a properly tuned learning rate.
# [optimizer]
# type = 'automagic'
# weight_decay = 0.01

[optimizer]
type = 'Prodigy'
lr = 1.0
weight_decay = 0.01
use_bias_correction = true
safeguard_warmup = true
betas = [0.9, 0.99]