YiYiXu commited on
Commit
ea17a8d
·
verified ·
1 Parent(s): 904ef62

Upload diff-diff module

Browse files
__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ from diffusers.modular_pipelines.node_utils import ModularNode
2
+
3
+
4
+
5
+ MODULE_MAP = {
6
+ "DiffDiff": ModularNode.load_mellon_config("custom/diff-diff/diffdiff_mellon_config.json")["module"],
7
+ "DiffDiffDenoise": ModularNode.load_mellon_config("custom/diff-diff/diffdiff_denoise_mellon_config.json")["module"],
8
+ }
9
+
__pycache__/__init__.cpython-310.pyc ADDED
Binary file (418 Bytes). View file
 
__pycache__/diff-diff.cpython-310.pyc ADDED
Binary file (1.68 kB). View file
 
diff-diff.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ from mellon.NodeBase import NodeBase
3
+
4
+ # Configure logger
5
+ logger = logging.getLogger("mellon")
6
+ logger.setLevel(logging.DEBUG)
7
+
8
+
9
+
10
+
11
+ # Initialize components manager
12
+ # components = ComponentsManager()
13
+ from custom import components
14
+
15
+
16
+ class DiffDiff(NodeBase):
17
+ def __init__(self, node_id=None):
18
+ super().__init__(node_id)
19
+ from diffusers.modular_pipelines.node_utils import ModularNode
20
+ self._diffdiff_block = ModularNode.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
21
+ self._diffdiff_block.setup(components=components)
22
+
23
+ def execute(self, **kwargs):
24
+ return self._diffdiff_block.execute(**kwargs)
25
+
26
+
27
+ class DiffDiffDenoise(NodeBase):
28
+ def __init__(self, node_id=None):
29
+ super().__init__(node_id)
30
+ from diffusers.modular_pipelines.node_utils import ModularNode
31
+ from diffusers.modular_pipelines import ModularPipelineMixin
32
+ diffdiff = ModularPipelineMixin.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
33
+ diffdiff.blocks.pop("text_encoder")
34
+ diffdiff.blocks.pop("decode")
35
+ diffdiff.blocks.pop("ip_adapter")
36
+ self._diffdiff_block = ModularNode(diffdiff)
37
+ self._diffdiff_block.setup(components=components)
38
+
39
+ def execute(self, **kwargs):
40
+ return self._diffdiff_block.execute(**kwargs)
41
+
42
+
diffdiff_denoise_mellon_config.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "module": {
3
+ "label": "DiffDiffDenoiseBlocks",
4
+ "category": "Modular Diffusers",
5
+ "params": {
6
+ "image": {
7
+ "label": "Image",
8
+ "type": "image",
9
+ "display": "input"
10
+ },
11
+ "height": {
12
+ "label": "Height",
13
+ "type": "int",
14
+ "display": "text",
15
+ "default": 1024,
16
+ "min": 8,
17
+ "max": 8192,
18
+ "step": 8,
19
+ "group": "dimensions"
20
+ },
21
+ "width": {
22
+ "label": "Width",
23
+ "type": "int",
24
+ "display": "text",
25
+ "default": 1024,
26
+ "min": 8,
27
+ "max": 8192,
28
+ "step": 8,
29
+ "group": "dimensions"
30
+ },
31
+ "num_inference_steps": {
32
+ "label": "Steps",
33
+ "type": "int",
34
+ "default": 25,
35
+ "min": 1,
36
+ "max": 1000
37
+ },
38
+ "diffdiff_map": {
39
+ "type": "image",
40
+ "label": "diffdiff_map",
41
+ "display": "input"
42
+ },
43
+ "prompt_embeddings": {
44
+ "label": "prompt_embeddings",
45
+ "type": "prompt_embeddings",
46
+ "display": "input"
47
+ },
48
+ "ip_adapter_embeds": {
49
+ "label": "ip_adapter_embeds",
50
+ "type": "ip_adapter_embeds",
51
+ "display": "input"
52
+ },
53
+ "vae": {
54
+ "label": "vae",
55
+ "type": "diffusers_auto_model",
56
+ "display": "input"
57
+ },
58
+ "scheduler": {
59
+ "label": "scheduler",
60
+ "type": "scheduler",
61
+ "display": "input"
62
+ },
63
+ "guider": {
64
+ "label": "guider",
65
+ "type": "guider",
66
+ "display": "input"
67
+ },
68
+ "unet": {
69
+ "label": "unet",
70
+ "type": "diffusers_auto_model",
71
+ "display": "input"
72
+ },
73
+ "controlnet": {
74
+ "label": "controlnet",
75
+ "type": "diffusers_auto_model",
76
+ "display": "input"
77
+ },
78
+ "latents": {
79
+ "label": "latents",
80
+ "type": "latents",
81
+ "display": "output"
82
+ }
83
+ }
84
+ },
85
+ "name_mapping": {
86
+ "prompt_embeds": "prompt_embeddings",
87
+ "negative_prompt_embeds": "prompt_embeddings",
88
+ "pooled_prompt_embeds": "prompt_embeddings",
89
+ "negative_pooled_prompt_embeds": "prompt_embeddings",
90
+ "ip_adapter_embeds": "ip_adapter_embeds",
91
+ "negative_ip_adapter_embeds": "ip_adapter_embeds"
92
+ }
93
+ }
diffdiff_mellon_config.json ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "module": {
3
+ "label": "DiffDiffBlocks",
4
+ "category": "Modular Diffusers",
5
+ "params": {
6
+ "prompt": {
7
+ "label": "Prompt",
8
+ "type": "string",
9
+ "default": "a bear sitting in a chair drinking a milkshake",
10
+ "display": "textarea"
11
+ },
12
+ "negative_prompt": {
13
+ "label": "Negative Prompt",
14
+ "type": "string",
15
+ "default": "deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality",
16
+ "display": "textarea"
17
+ },
18
+ "image": {
19
+ "label": "Image",
20
+ "type": "image",
21
+ "display": "input"
22
+ },
23
+ "height": {
24
+ "label": "Height",
25
+ "type": "int",
26
+ "display": "text",
27
+ "default": 1024,
28
+ "min": 8,
29
+ "max": 8192,
30
+ "step": 8,
31
+ "group": "dimensions"
32
+ },
33
+ "width": {
34
+ "label": "Width",
35
+ "type": "int",
36
+ "display": "text",
37
+ "default": 1024,
38
+ "min": 8,
39
+ "max": 8192,
40
+ "step": 8,
41
+ "group": "dimensions"
42
+ },
43
+ "num_inference_steps": {
44
+ "label": "Steps",
45
+ "type": "int",
46
+ "default": 25,
47
+ "min": 1,
48
+ "max": 1000
49
+ },
50
+ "diffdiff_map": {
51
+ "type": "image",
52
+ "label": "diffdiff_map",
53
+ "display": "input"
54
+ },
55
+ "ip_adapter_embeds": {
56
+ "label": "ip_adapter_embeds",
57
+ "type": "ip_adapter_embeds",
58
+ "display": "input"
59
+ },
60
+ "text_encoders": {
61
+ "label": "text_encoders",
62
+ "type": "text_encoders",
63
+ "display": "input"
64
+ },
65
+ "guider": {
66
+ "label": "guider",
67
+ "type": "guider",
68
+ "display": "input"
69
+ },
70
+ "image_encoder": {
71
+ "label": "image_encoder",
72
+ "type": "diffusers_auto_model",
73
+ "display": "input"
74
+ },
75
+ "unet": {
76
+ "label": "unet",
77
+ "type": "diffusers_auto_model",
78
+ "display": "input"
79
+ },
80
+ "vae": {
81
+ "label": "vae",
82
+ "type": "diffusers_auto_model",
83
+ "display": "input"
84
+ },
85
+ "scheduler": {
86
+ "label": "scheduler",
87
+ "type": "scheduler",
88
+ "display": "input"
89
+ },
90
+ "controlnet": {
91
+ "label": "controlnet",
92
+ "type": "diffusers_auto_model",
93
+ "display": "input"
94
+ },
95
+ "images": {
96
+ "label": "Images",
97
+ "type": "image",
98
+ "display": "output"
99
+ }
100
+ }
101
+ },
102
+ "name_mapping": {
103
+ "ip_adapter_embeds": "ip_adapter_embeds",
104
+ "negative_ip_adapter_embeds": "ip_adapter_embeds",
105
+ "text_encoder": "text_encoders",
106
+ "text_encoder_2": "text_encoders",
107
+ "tokenizer": "text_encoders",
108
+ "tokenizer_2": "text_encoders"
109
+ }
110
+ }
save_mellon_config.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # YiYi Notes: use this to generate mellon config
2
+ # TODO:
3
+ # 1. the whole process to create a custom node should be automated with something like `node.push_to_hub()`
4
+ # 2. should be able to edit mellon_config.json
5
+
6
+
7
+ from diffusers.modular_pipelines.node_utils import ModularNode
8
+ from diffusers.modular_pipelines import ModularPipelineMixin
9
+ diffdiff = ModularPipelineMixin.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
10
+
11
+ # this is the complete pipeline
12
+ node = ModularNode(diffdiff)
13
+ node.save_mellon_config("diffdiff_mellon_config.json")
14
+
15
+
16
+ # this is the denoise pipeline, to be used in the modular setting,
17
+ # remove the text_encoder, decode, and ip_adapter blocks since we already have these nodes in official modular diffusers mellon
18
+ diffdiff.blocks.pop("text_encoder")
19
+ diffdiff.blocks.pop("decode")
20
+ diffdiff.blocks.pop("ip_adapter")
21
+
22
+ node = ModularNode(diffdiff, label = "DiffDiffDenoiseBlocks")
23
+ node.save_mellon_config("diffdiff_denoise_mellon_config.json")
24
+