Upload diff-diff module
Browse files- __init__.py +9 -0
- __pycache__/__init__.cpython-310.pyc +0 -0
- __pycache__/diff-diff.cpython-310.pyc +0 -0
- diff-diff.py +42 -0
- diffdiff_denoise_mellon_config.json +93 -0
- diffdiff_mellon_config.json +110 -0
- save_mellon_config.py +24 -0
__init__.py
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from diffusers.modular_pipelines.node_utils import ModularNode
|
2 |
+
|
3 |
+
|
4 |
+
|
5 |
+
MODULE_MAP = {
|
6 |
+
"DiffDiff": ModularNode.load_mellon_config("custom/diff-diff/diffdiff_mellon_config.json")["module"],
|
7 |
+
"DiffDiffDenoise": ModularNode.load_mellon_config("custom/diff-diff/diffdiff_denoise_mellon_config.json")["module"],
|
8 |
+
}
|
9 |
+
|
__pycache__/__init__.cpython-310.pyc
ADDED
Binary file (418 Bytes). View file
|
|
__pycache__/diff-diff.cpython-310.pyc
ADDED
Binary file (1.68 kB). View file
|
|
diff-diff.py
ADDED
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import logging
|
2 |
+
from mellon.NodeBase import NodeBase
|
3 |
+
|
4 |
+
# Configure logger
|
5 |
+
logger = logging.getLogger("mellon")
|
6 |
+
logger.setLevel(logging.DEBUG)
|
7 |
+
|
8 |
+
|
9 |
+
|
10 |
+
|
11 |
+
# Initialize components manager
|
12 |
+
# components = ComponentsManager()
|
13 |
+
from custom import components
|
14 |
+
|
15 |
+
|
16 |
+
class DiffDiff(NodeBase):
|
17 |
+
def __init__(self, node_id=None):
|
18 |
+
super().__init__(node_id)
|
19 |
+
from diffusers.modular_pipelines.node_utils import ModularNode
|
20 |
+
self._diffdiff_block = ModularNode.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
|
21 |
+
self._diffdiff_block.setup(components=components)
|
22 |
+
|
23 |
+
def execute(self, **kwargs):
|
24 |
+
return self._diffdiff_block.execute(**kwargs)
|
25 |
+
|
26 |
+
|
27 |
+
class DiffDiffDenoise(NodeBase):
|
28 |
+
def __init__(self, node_id=None):
|
29 |
+
super().__init__(node_id)
|
30 |
+
from diffusers.modular_pipelines.node_utils import ModularNode
|
31 |
+
from diffusers.modular_pipelines import ModularPipelineMixin
|
32 |
+
diffdiff = ModularPipelineMixin.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
|
33 |
+
diffdiff.blocks.pop("text_encoder")
|
34 |
+
diffdiff.blocks.pop("decode")
|
35 |
+
diffdiff.blocks.pop("ip_adapter")
|
36 |
+
self._diffdiff_block = ModularNode(diffdiff)
|
37 |
+
self._diffdiff_block.setup(components=components)
|
38 |
+
|
39 |
+
def execute(self, **kwargs):
|
40 |
+
return self._diffdiff_block.execute(**kwargs)
|
41 |
+
|
42 |
+
|
diffdiff_denoise_mellon_config.json
ADDED
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"module": {
|
3 |
+
"label": "DiffDiffDenoiseBlocks",
|
4 |
+
"category": "Modular Diffusers",
|
5 |
+
"params": {
|
6 |
+
"image": {
|
7 |
+
"label": "Image",
|
8 |
+
"type": "image",
|
9 |
+
"display": "input"
|
10 |
+
},
|
11 |
+
"height": {
|
12 |
+
"label": "Height",
|
13 |
+
"type": "int",
|
14 |
+
"display": "text",
|
15 |
+
"default": 1024,
|
16 |
+
"min": 8,
|
17 |
+
"max": 8192,
|
18 |
+
"step": 8,
|
19 |
+
"group": "dimensions"
|
20 |
+
},
|
21 |
+
"width": {
|
22 |
+
"label": "Width",
|
23 |
+
"type": "int",
|
24 |
+
"display": "text",
|
25 |
+
"default": 1024,
|
26 |
+
"min": 8,
|
27 |
+
"max": 8192,
|
28 |
+
"step": 8,
|
29 |
+
"group": "dimensions"
|
30 |
+
},
|
31 |
+
"num_inference_steps": {
|
32 |
+
"label": "Steps",
|
33 |
+
"type": "int",
|
34 |
+
"default": 25,
|
35 |
+
"min": 1,
|
36 |
+
"max": 1000
|
37 |
+
},
|
38 |
+
"diffdiff_map": {
|
39 |
+
"type": "image",
|
40 |
+
"label": "diffdiff_map",
|
41 |
+
"display": "input"
|
42 |
+
},
|
43 |
+
"prompt_embeddings": {
|
44 |
+
"label": "prompt_embeddings",
|
45 |
+
"type": "prompt_embeddings",
|
46 |
+
"display": "input"
|
47 |
+
},
|
48 |
+
"ip_adapter_embeds": {
|
49 |
+
"label": "ip_adapter_embeds",
|
50 |
+
"type": "ip_adapter_embeds",
|
51 |
+
"display": "input"
|
52 |
+
},
|
53 |
+
"vae": {
|
54 |
+
"label": "vae",
|
55 |
+
"type": "diffusers_auto_model",
|
56 |
+
"display": "input"
|
57 |
+
},
|
58 |
+
"scheduler": {
|
59 |
+
"label": "scheduler",
|
60 |
+
"type": "scheduler",
|
61 |
+
"display": "input"
|
62 |
+
},
|
63 |
+
"guider": {
|
64 |
+
"label": "guider",
|
65 |
+
"type": "guider",
|
66 |
+
"display": "input"
|
67 |
+
},
|
68 |
+
"unet": {
|
69 |
+
"label": "unet",
|
70 |
+
"type": "diffusers_auto_model",
|
71 |
+
"display": "input"
|
72 |
+
},
|
73 |
+
"controlnet": {
|
74 |
+
"label": "controlnet",
|
75 |
+
"type": "diffusers_auto_model",
|
76 |
+
"display": "input"
|
77 |
+
},
|
78 |
+
"latents": {
|
79 |
+
"label": "latents",
|
80 |
+
"type": "latents",
|
81 |
+
"display": "output"
|
82 |
+
}
|
83 |
+
}
|
84 |
+
},
|
85 |
+
"name_mapping": {
|
86 |
+
"prompt_embeds": "prompt_embeddings",
|
87 |
+
"negative_prompt_embeds": "prompt_embeddings",
|
88 |
+
"pooled_prompt_embeds": "prompt_embeddings",
|
89 |
+
"negative_pooled_prompt_embeds": "prompt_embeddings",
|
90 |
+
"ip_adapter_embeds": "ip_adapter_embeds",
|
91 |
+
"negative_ip_adapter_embeds": "ip_adapter_embeds"
|
92 |
+
}
|
93 |
+
}
|
diffdiff_mellon_config.json
ADDED
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"module": {
|
3 |
+
"label": "DiffDiffBlocks",
|
4 |
+
"category": "Modular Diffusers",
|
5 |
+
"params": {
|
6 |
+
"prompt": {
|
7 |
+
"label": "Prompt",
|
8 |
+
"type": "string",
|
9 |
+
"default": "a bear sitting in a chair drinking a milkshake",
|
10 |
+
"display": "textarea"
|
11 |
+
},
|
12 |
+
"negative_prompt": {
|
13 |
+
"label": "Negative Prompt",
|
14 |
+
"type": "string",
|
15 |
+
"default": "deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality",
|
16 |
+
"display": "textarea"
|
17 |
+
},
|
18 |
+
"image": {
|
19 |
+
"label": "Image",
|
20 |
+
"type": "image",
|
21 |
+
"display": "input"
|
22 |
+
},
|
23 |
+
"height": {
|
24 |
+
"label": "Height",
|
25 |
+
"type": "int",
|
26 |
+
"display": "text",
|
27 |
+
"default": 1024,
|
28 |
+
"min": 8,
|
29 |
+
"max": 8192,
|
30 |
+
"step": 8,
|
31 |
+
"group": "dimensions"
|
32 |
+
},
|
33 |
+
"width": {
|
34 |
+
"label": "Width",
|
35 |
+
"type": "int",
|
36 |
+
"display": "text",
|
37 |
+
"default": 1024,
|
38 |
+
"min": 8,
|
39 |
+
"max": 8192,
|
40 |
+
"step": 8,
|
41 |
+
"group": "dimensions"
|
42 |
+
},
|
43 |
+
"num_inference_steps": {
|
44 |
+
"label": "Steps",
|
45 |
+
"type": "int",
|
46 |
+
"default": 25,
|
47 |
+
"min": 1,
|
48 |
+
"max": 1000
|
49 |
+
},
|
50 |
+
"diffdiff_map": {
|
51 |
+
"type": "image",
|
52 |
+
"label": "diffdiff_map",
|
53 |
+
"display": "input"
|
54 |
+
},
|
55 |
+
"ip_adapter_embeds": {
|
56 |
+
"label": "ip_adapter_embeds",
|
57 |
+
"type": "ip_adapter_embeds",
|
58 |
+
"display": "input"
|
59 |
+
},
|
60 |
+
"text_encoders": {
|
61 |
+
"label": "text_encoders",
|
62 |
+
"type": "text_encoders",
|
63 |
+
"display": "input"
|
64 |
+
},
|
65 |
+
"guider": {
|
66 |
+
"label": "guider",
|
67 |
+
"type": "guider",
|
68 |
+
"display": "input"
|
69 |
+
},
|
70 |
+
"image_encoder": {
|
71 |
+
"label": "image_encoder",
|
72 |
+
"type": "diffusers_auto_model",
|
73 |
+
"display": "input"
|
74 |
+
},
|
75 |
+
"unet": {
|
76 |
+
"label": "unet",
|
77 |
+
"type": "diffusers_auto_model",
|
78 |
+
"display": "input"
|
79 |
+
},
|
80 |
+
"vae": {
|
81 |
+
"label": "vae",
|
82 |
+
"type": "diffusers_auto_model",
|
83 |
+
"display": "input"
|
84 |
+
},
|
85 |
+
"scheduler": {
|
86 |
+
"label": "scheduler",
|
87 |
+
"type": "scheduler",
|
88 |
+
"display": "input"
|
89 |
+
},
|
90 |
+
"controlnet": {
|
91 |
+
"label": "controlnet",
|
92 |
+
"type": "diffusers_auto_model",
|
93 |
+
"display": "input"
|
94 |
+
},
|
95 |
+
"images": {
|
96 |
+
"label": "Images",
|
97 |
+
"type": "image",
|
98 |
+
"display": "output"
|
99 |
+
}
|
100 |
+
}
|
101 |
+
},
|
102 |
+
"name_mapping": {
|
103 |
+
"ip_adapter_embeds": "ip_adapter_embeds",
|
104 |
+
"negative_ip_adapter_embeds": "ip_adapter_embeds",
|
105 |
+
"text_encoder": "text_encoders",
|
106 |
+
"text_encoder_2": "text_encoders",
|
107 |
+
"tokenizer": "text_encoders",
|
108 |
+
"tokenizer_2": "text_encoders"
|
109 |
+
}
|
110 |
+
}
|
save_mellon_config.py
ADDED
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# YiYi Notes: use this to generate mellon config
|
2 |
+
# TODO:
|
3 |
+
# 1. the whole process to create a custom node should be automated with something like `node.push_to_hub()`
|
4 |
+
# 2. should be able to edit mellon_config.json
|
5 |
+
|
6 |
+
|
7 |
+
from diffusers.modular_pipelines.node_utils import ModularNode
|
8 |
+
from diffusers.modular_pipelines import ModularPipelineMixin
|
9 |
+
diffdiff = ModularPipelineMixin.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
|
10 |
+
|
11 |
+
# this is the complete pipeline
|
12 |
+
node = ModularNode(diffdiff)
|
13 |
+
node.save_mellon_config("diffdiff_mellon_config.json")
|
14 |
+
|
15 |
+
|
16 |
+
# this is the denoise pipeline, to be used in the modular setting,
|
17 |
+
# remove the text_encoder, decode, and ip_adapter blocks since we already have these nodes in official modular diffusers mellon
|
18 |
+
diffdiff.blocks.pop("text_encoder")
|
19 |
+
diffdiff.blocks.pop("decode")
|
20 |
+
diffdiff.blocks.pop("ip_adapter")
|
21 |
+
|
22 |
+
node = ModularNode(diffdiff, label = "DiffDiffDenoiseBlocks")
|
23 |
+
node.save_mellon_config("diffdiff_denoise_mellon_config.json")
|
24 |
+
|