| base_model_path: ./pretrained_weights/stable-diffusion-v1-5 | |
| checkpointing_steps: 2000 | |
| data: | |
| crop_scale: | |
| - 1 | |
| - 1 | |
| do_center_crop: false | |
| meta_paths: | |
| - /workspace/develop/video/data/ubc_tiktok-dropout0.03/ubc-meta.json | |
| - /workspace/develop/video/data/tiktok-dance/good-meta.json | |
| - /workspace/develop/video/data/20240321/meta.json | |
| - /workspace/develop/video/data/20240327/meta.json | |
| - /workspace/develop/video/data/20240506/meta.json | |
| - /workspace/develop/video/data/20240509/meta.json | |
| n_sample_frames: 24 | |
| ref_augment: | |
| pan: | |
| - 0.04 | |
| - 0.04 | |
| rotate: 2 | |
| scale: | |
| - 0.9 | |
| - 1.0 | |
| sample_rate: 4 | |
| train_bs: 1 | |
| train_height: 960 | |
| train_width: 640 | |
| enable_zero_snr: true | |
| exp_name: stage2-openpg | |
| image_encoder_path: ./pretrained_weights/sd-image-variations-diffusers/image_encoder | |
| mm_path: ./pretrained_weights/mm_sd_v15_v2.ckpt | |
| noise_offset: 0.05 | |
| noise_scheduler_kwargs: | |
| beta_end: 0.012 | |
| beta_schedule: linear | |
| beta_start: 0.00085 | |
| clip_sample: false | |
| num_train_timesteps: 1000 | |
| steps_offset: 1 | |
| openpose_guider: | |
| block_out_channels: | |
| - 96 | |
| - 192 | |
| enable: true | |
| exclude_paf: true | |
| output_dir: /workspace/camus/train | |
| resume_from_checkpoint: '' | |
| save_dir: /workspace/camus/train/20240517-stage2-openpg | |
| save_model_epoch_interval: 1 | |
| seed: 12580 | |
| snr_gamma: 5.0 | |
| solver: | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.999 | |
| adam_epsilon: 1.0e-08 | |
| adam_weight_decay: 0.01 | |
| enable_xformers_memory_efficient_attention: true | |
| gradient_accumulation_steps: 1 | |
| gradient_checkpointing: true | |
| learning_rate: 1.0e-05 | |
| lr_scheduler: constant | |
| lr_warmup_steps: 1 | |
| max_grad_norm: 1.0 | |
| max_train_steps: 160000 | |
| mixed_precision: fp16 | |
| scale_lr: false | |
| use_8bit_adam: true | |
| stage1_ckpt_dir: /workspace/camus/train/20240508-stage1-openpg-nopaf | |
| stage1_ckpt_step: 74712 | |
| uncond_ratio: 0.1 | |
| vae_model_path: ./pretrained_weights/sd-vae-ft-mse | |
| val: | |
| validation_steps: 1000 | |
| validation: | |
| guidance_scale: 2.8 | |
| metric: | |
| generate_frame_range: | |
| - 50 | |
| - 74 | |
| guidance_scale: 2.8 | |
| ref_frame: 29 | |
| seed: 42 | |
| steps: 20 | |
| videos: | |
| - configs/inference/metric/A1ubDo0PbQS.mp4 | |
| - configs/inference/metric/oEtwozJ6AoIBJ6oyK6rAAQGAOiEWIQixF2F2fB.mp4 | |
| - configs/inference/metric/oEOgTIKvy7lAQIfZ37E5BFmCVBID3gIQUQfMdv.mp4 | |
| - configs/inference/metric/ocQoBObnUgBnVskCnPe41sYRiBcAFD5f8AN1Rg.mp4 | |
| pose_range: | |
| - 24 | |
| - 48 | |
| test_cases: | |
| - - ./configs/inference/ref_images/anyone-3.png | |
| - ./configs/inference/metric/A1T-Ea-FlQS.mp4 | |
| - - ./configs/inference/ref_images/anyone-3.png | |
| - ./configs/inference/metric/A1YNmKj0sCS.mp4 | |
| - - ./configs/inference/ref_images/anyone-1.png | |
| - ./configs/inference/metric/oEtwozJ6AoIBJ6oyK6rAAQGAOiEWIQixF2F2fB.mp4 | |
| - - ./configs/inference/ref_images/anyone-5.png | |
| - ./configs/inference/metric/A1YNmKj0sCS.mp4 | |
| - - ./configs/inference/ref_images/anyone-11.png | |
| - ./configs/inference/metric/A1YNmKj0sCS.mp4 | |
| uniform_along_time: false | |
| weight_dtype: fp16 | |