| base_model_path: ./pretrained_weights/stable-diffusion-v1-5 | |
| checkpointing_steps: 2000 | |
| data: | |
| crop_scale: | |
| - 1 | |
| - 1 | |
| do_center_crop: false | |
| meta_paths: | |
| - /workspace/develop/video/data/ubc_tiktok-dropout0.03/ubc-meta.json | |
| - /workspace/develop/video/data/tiktok-dance/good-meta.json | |
| - /workspace/develop/video/data/20240321/meta.json | |
| - /workspace/develop/video/data/20240327/meta.json | |
| - /workspace/develop/video/data/20240506/meta.json | |
| - /workspace/develop/video/data/20240509/meta.json | |
| n_sample_frames: 24 | |
| ref_augment: | |
| pan: | |
| - 0.04 | |
| - 0.04 | |
| rotate: 2 | |
| scale: | |
| - 0.9 | |
| - 1.2 | |
| sample_rate: 4 | |
| train_bs: 1 | |
| train_height: 960 | |
| train_width: 640 | |
| enable_zero_snr: true | |
| exp_name: stage2-9k | |
| image_encoder_path: ./pretrained_weights/sd-image-variations-diffusers/image_encoder | |
| mm_path: ./pretrained_weights/mm_sd_v15_v2.ckpt | |
| noise_offset: 0.05 | |
| noise_scheduler_kwargs: | |
| beta_end: 0.012 | |
| beta_schedule: linear | |
| beta_start: 0.00085 | |
| clip_sample: false | |
| num_train_timesteps: 1000 | |
| steps_offset: 1 | |
| output_dir: /workspace/camus/train | |
| resume_from_checkpoint: latest | |
| save_dir: /workspace/camus/train/20240513-stage2-9k | |
| save_model_epoch_interval: 1 | |
| seed: 12580 | |
| snr_gamma: 5.0 | |
| solver: | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.999 | |
| adam_epsilon: 1.0e-08 | |
| adam_weight_decay: 0.01 | |
| enable_xformers_memory_efficient_attention: true | |
| gradient_accumulation_steps: 1 | |
| gradient_checkpointing: true | |
| learning_rate: 1.0e-05 | |
| lr_scheduler: constant | |
| lr_warmup_steps: 1 | |
| max_grad_norm: 1.0 | |
| max_train_steps: 160000 | |
| mixed_precision: fp16 | |
| scale_lr: false | |
| use_8bit_adam: true | |
| stage1_ckpt_dir: /workspace/camus/train/20240510-stage1-9k | |
| stage1_ckpt_step: 75392 | |
| uncond_ratio: 0.1 | |
| vae_model_path: ./pretrained_weights/sd-vae-ft-mse | |
| val: | |
| validation_steps: 1000 | |
| validation: | |
| metric: | |
| generate_frame_range: | |
| - 30 | |
| - 54 | |
| guidance_scale: 2.8 | |
| ref_frame: 29 | |
| seed: 42 | |
| steps: 30 | |
| videos: | |
| - configs/inference/metric/oATCBbieJIB8u3QAMAUwvMi9ymEOIc1AoDOajA.mp4 | |
| - configs/inference/metric/oonQq0HjAC7ExkJlRSMBBs1q3EIiQgFveLD7fD.mp4 | |
| - configs/inference/metric/os0aLDIkagGgAfAFQsfICCWMuoL8jm3IgJ0Wey.mp4 | |
| - configs/inference/metric/oYflAvAyfAIFRf3yQDrLRDCWcEDoFENF9tBEgg.mp4 | |
| pose_range: | |
| - 0 | |
| - 24 | |
| test_cases: | |
| - - ./configs/inference/ref_images/anyone-3.png | |
| - ./configs/inference/pose_videos/demo18.mp4 | |
| - - ./configs/inference/ref_images/anyone-3-partial.png | |
| - ./configs/inference/pose_videos/demo6.mp4 | |
| - - ./configs/inference/ref_images/anyone-2.png | |
| - ./configs/inference/pose_videos/demo11.mp4 | |
| - - ./configs/inference/ref_images/anyone-1.png | |
| - ./configs/inference/pose_videos/demo11.mp4 | |
| - - ./configs/inference/ref_images/anyone-1.png | |
| - ./configs/inference/pose_videos/demo6.mp4 | |
| - - ./configs/inference/ref_images/anyone-5.png | |
| - ./configs/inference/pose_videos/demo11.mp4 | |
| - - ./configs/inference/ref_images/anyone-11.png | |
| - ./configs/inference/pose_videos/demo15.mp4 | |
| uniform_along_time: false | |
| weight_dtype: fp16 | |