|
|
--- |
|
|
license: apache-2.0 |
|
|
library_name: diffusers |
|
|
--- |
|
|
test under this PR https://github.com/huggingface/diffusers/pull/9672 |
|
|
|
|
|
#### create differential diffusion pipeline |
|
|
|
|
|
```python |
|
|
|
|
|
from diffusers.modular_pipelines import ModularPipeline, ComponentsManager |
|
|
import torch |
|
|
from diffusers.utils import load_image |
|
|
|
|
|
repo_id = "YiYiXu/modular-diffdiff" |
|
|
|
|
|
components = ComponentsManager() |
|
|
|
|
|
diffdiff_pipeline = ModularPipeline.from_pretrained(repo_id, trust_remote_code=True, components_manager=components, collection="diffdiff") |
|
|
diffdiff_pipeline.loader.load(torch_dtype=torch.float16) |
|
|
components.enable_auto_cpu_offload() |
|
|
``` |
|
|
|
|
|
#### basic diff-diff |
|
|
|
|
|
```python |
|
|
|
|
|
image = load_image("https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/20240329211129_4024911930.png?download=true") |
|
|
mask = load_image("https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/gradient_mask.png?download=true") |
|
|
|
|
|
prompt = "a green pear" |
|
|
negative_prompt = "blurry" |
|
|
generator = torch.Generator(device="cuda").manual_seed(42) |
|
|
|
|
|
image = diffdiff_pipeline( |
|
|
prompt=prompt, |
|
|
negative_prompt=negative_prompt, |
|
|
num_inference_steps=25, |
|
|
generator=generator, |
|
|
diffdiff_map=mask, |
|
|
image=image, |
|
|
output="images" |
|
|
)[0] |
|
|
``` |
|
|
|
|
|
 |
|
|
|
|
|
|
|
|
#### ip-adapter |
|
|
|
|
|
```python |
|
|
diffdiff_pipeline.loader.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin") |
|
|
diffdiff_pipeline.loader.set_ip_adapter_scale(0.6) |
|
|
|
|
|
ip_adapter_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_orange.jpeg") |
|
|
image = load_image("https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/20240329211129_4024911930.png?download=true") |
|
|
mask = load_image("https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/gradient_mask.png?download=true") |
|
|
|
|
|
prompt = "a green pear" |
|
|
negative_prompt = "blurry" |
|
|
generator = torch.Generator(device="cuda").manual_seed(42) |
|
|
|
|
|
image = diffdiff_pipeline( |
|
|
prompt=prompt, |
|
|
negative_prompt=negative_prompt, |
|
|
num_inference_steps=25, |
|
|
generator=generator, |
|
|
ip_adapter_image=ip_adapter_image, |
|
|
diffdiff_map=mask, |
|
|
image=image, |
|
|
output="images" |
|
|
)[0] |
|
|
``` |
|
|
|
|
|
 |
|
|
|
|
|
|
|
|
#### controlnet |
|
|
|
|
|
```python |
|
|
diffdiff_pipeline.loader.unload_ip_adapter() |
|
|
|
|
|
control_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_tomato_canny.png") |
|
|
image = load_image("https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/20240329211129_4024911930.png?download=true") |
|
|
mask = load_image("https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/gradient_mask.png?download=true") |
|
|
|
|
|
prompt = "a green pear" |
|
|
negative_prompt = "blurry" |
|
|
generator = torch.Generator(device="cuda").manual_seed(42) |
|
|
|
|
|
image = diffdiff_pipeline( |
|
|
prompt=prompt, |
|
|
negative_prompt=negative_prompt, |
|
|
num_inference_steps=25, |
|
|
generator=generator, |
|
|
control_image=control_image, |
|
|
controlnet_conditioning_scale=0.5, |
|
|
diffdiff_map=mask, |
|
|
image=image, |
|
|
output="images" |
|
|
)[0] |
|
|
|
|
|
``` |
|
|
|
|
|
|
|
|
 |
|
|
|
|
|
|
|
|
|
