diff-diff-mellon / workflow_modular.json
YiYiXu's picture
Upload workflow_modular.json
63332f4 verified
{
"nodes": [
{
"id": "ImncaWQ8VOjefxwYEgaCT",
"type": "custom",
"position": {
"x": -116.1558204391292,
"y": 487.56867449559854
},
"data": {
"label": "Encode Prompt",
"module": "mellon-modular-diffusers.custom",
"action": "EncodePrompt",
"category": "Modular-Diffusers",
"params": {
"text_encoders": {
"label": "Text Encoders",
"display": "input",
"type": "text_encoders"
},
"guider": {
"label": "Guider",
"display": "input",
"type": "guider"
},
"prompt": {
"label": "Prompt",
"type": "string",
"default": "a bear sitting in a chair drinking a milkshake",
"display": "textarea",
"value": "a green pear"
},
"negative_prompt": {
"label": "Negative Prompt",
"type": "string",
"default": "deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality",
"display": "textarea",
"value": "deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality"
},
"embeddings": {
"label": "Embeddings",
"display": "output",
"type": "prompt_embeddings"
},
"guider_out": {
"label": "Guider",
"display": "output",
"type": "guider"
}
},
"groups": {},
"cache": true,
"time": "3.93",
"memory": 0
},
"measured": {
"width": 336,
"height": 461
},
"selected": false,
"dragging": false
},
{
"id": "1odkYiAOIAMJnLYJrfeY3",
"type": "custom",
"position": {
"x": -524.2376560212152,
"y": 460.05421089502954
},
"data": {
"label": "Diffusers Model Loader",
"module": "mellon-modular-diffusers.custom",
"action": "ModelsLoader",
"category": "Modular-Diffusers",
"params": {
"repo_id": {
"label": "Repository ID",
"options": [
"fusing/unet-ldm-dummy-update"
],
"display": "autocomplete",
"no_validation": true,
"default": "YiYiXu/modular-demo-auto",
"value": "YiYiXu/modular-demo-auto"
},
"variant": {
"label": "Variant",
"type": "string",
"options": [
"",
"fp16"
],
"default": "",
"display": "autocomplete",
"no_validation": true,
"value": ""
},
"device": {
"label": "Device",
"type": "string",
"options": {
"cuda:0": {
"index": 0,
"device": "cuda:0",
"host": "",
"label": "cuda:0",
"total_memory": 84987740160
},
"cuda:1": {
"index": 1,
"device": "cuda:1",
"host": "",
"label": "cuda:1",
"total_memory": 84987740160
},
"cuda:2": {
"index": 2,
"device": "cuda:2",
"host": "",
"label": "cuda:2",
"total_memory": 84987740160
},
"cuda:3": {
"index": 3,
"device": "cuda:3",
"host": "",
"label": "cuda:3",
"total_memory": 84987740160
},
"cuda:4": {
"index": 4,
"device": "cuda:4",
"host": "",
"label": "cuda:4",
"total_memory": 4093509632
},
"cpu": {
"index": 0,
"device": "cpu",
"host": "",
"label": "cpu",
"total_memory": null
}
},
"default": "cuda:0",
"value": "cuda:0"
},
"dtype": {
"label": "dtype",
"options": [
"auto",
"float32",
"bfloat16",
"float16"
],
"default": "float16",
"value": "float16"
},
"unet": {
"label": "Unet",
"display": "input",
"type": "diffusers_auto_model"
},
"vae": {
"label": "VAE",
"display": "input",
"type": "diffusers_auto_model"
},
"lora_list": {
"label": "Lora",
"display": "input",
"type": "lora"
},
"text_encoders": {
"label": "Text Encoders",
"display": "output",
"type": "text_encoders"
},
"unet_out": {
"label": "UNet",
"display": "output",
"type": "diffusers_auto_model"
},
"vae_out": {
"label": "VAE",
"display": "output",
"type": "diffusers_auto_model"
},
"scheduler": {
"label": "Scheduler",
"display": "output",
"type": "scheduler"
}
},
"groups": {},
"cache": true,
"time": "0.00",
"memory": 0
},
"measured": {
"width": 336,
"height": 558
},
"selected": false,
"dragging": false
},
{
"id": "HlyuKg2JBy-2BTKJIoFc5",
"type": "custom",
"position": {
"x": 1096.7038494521753,
"y": 527.0055690837924
},
"data": {
"label": "Decode Latents",
"module": "mellon-modular-diffusers.custom",
"action": "DecodeLatents",
"category": "Modular-Diffusers",
"params": {
"vae": {
"label": "VAE",
"display": "input",
"type": "diffusers_auto_model"
},
"latents": {
"label": "Latents",
"type": "latents",
"display": "input"
},
"images": {
"label": "Images",
"type": "image",
"display": "output"
}
},
"groups": {},
"cache": true,
"time": "3.65",
"memory": 0
},
"measured": {
"width": 200,
"height": 206
},
"selected": false,
"dragging": false
},
{
"id": "rrRHxl2qIfOwWYviB86VR",
"type": "custom",
"position": {
"x": 882.6685422383307,
"y": 822.2718728242374
},
"data": {
"label": "Preview Image",
"module": "BasicImage",
"action": "Preview",
"category": "image",
"params": {
"vae": {
"label": "VAE | Pipeline",
"display": "input",
"type": [
"pipeline",
"vae"
],
"description": "VAE to decode latents. Required only if input images are latents."
},
"images": {
"label": "Images | Latents",
"display": "input",
"type": [
"image",
"latent"
]
},
"images_out": {
"label": "Images",
"display": "output",
"type": "image"
},
"width": {
"label": "Width",
"type": "int",
"display": "output"
},
"height": {
"label": "Height",
"type": "int",
"display": "output"
},
"preview": {
"label": "Preview",
"display": "ui",
"source": "images_out",
"type": "image",
"value": [
{
"url": "/view/webp/rrRHxl2qIfOwWYviB86VR/images_out/0?t=1747905887.6735773"
}
]
},
"device": {
"label": "Device",
"type": "string",
"options": {
"cuda:0": {
"index": 0,
"device": "cuda:0",
"host": "",
"label": "cuda:0",
"total_memory": 84987740160
},
"cuda:1": {
"index": 1,
"device": "cuda:1",
"host": "",
"label": "cuda:1",
"total_memory": 84987740160
},
"cuda:2": {
"index": 2,
"device": "cuda:2",
"host": "",
"label": "cuda:2",
"total_memory": 84987740160
},
"cuda:3": {
"index": 3,
"device": "cuda:3",
"host": "",
"label": "cuda:3",
"total_memory": 84987740160
},
"cuda:4": {
"index": 4,
"device": "cuda:4",
"host": "",
"label": "cuda:4",
"total_memory": 4093509632
},
"cpu": {
"index": 0,
"device": "cpu",
"host": "",
"label": "cpu",
"total_memory": null
}
},
"default": "cuda:0",
"value": "cuda:0"
}
},
"groups": {},
"cache": true,
"time": "0.00",
"memory": 0
},
"measured": {
"width": 1044,
"height": 1386
},
"selected": false,
"dragging": false
},
{
"id": "abVoELMdZ0xv594cFncti",
"type": "custom",
"position": {
"x": -123.62972968357649,
"y": 972.6475568956197
},
"data": {
"label": "Load Image",
"module": "BasicImage",
"action": "LoadImage",
"category": "image",
"params": {
"path": {
"label": "Path",
"type": "string",
"display": "filebrowser",
"value": "diffdiff_apple.png"
},
"images": {
"label": "Image",
"display": "output",
"type": "image"
}
},
"groups": {},
"cache": true,
"time": "0.00",
"memory": 0
},
"measured": {
"width": 298,
"height": 186
},
"selected": false,
"dragging": false
},
{
"id": "94KGPHhNdQYt-6l7SmH7F",
"type": "custom",
"position": {
"x": 364.2063909929194,
"y": 974.0149570644614
},
"data": {
"label": "Load Image",
"module": "BasicImage",
"action": "LoadImage",
"category": "image",
"params": {
"path": {
"label": "Path",
"type": "string",
"display": "filebrowser",
"value": "diffdiff_gradient_mask.png"
},
"images": {
"label": "Image",
"display": "output",
"type": "image"
}
},
"groups": {},
"cache": true,
"time": "0.00",
"memory": 0
},
"measured": {
"width": 298,
"height": 186
},
"selected": false,
"dragging": false
},
{
"id": "UOARYm2reAw23CfuewLqN",
"type": "custom",
"position": {
"x": -221.69434529648183,
"y": 1167.1680260947646
},
"data": {
"label": "Preview Image",
"module": "BasicImage",
"action": "Preview",
"category": "image",
"params": {
"vae": {
"label": "VAE | Pipeline",
"display": "input",
"type": [
"pipeline",
"vae"
],
"description": "VAE to decode latents. Required only if input images are latents."
},
"images": {
"label": "Images | Latents",
"display": "input",
"type": [
"image",
"latent"
]
},
"images_out": {
"label": "Images",
"display": "output",
"type": "image"
},
"width": {
"label": "Width",
"type": "int",
"display": "output"
},
"height": {
"label": "Height",
"type": "int",
"display": "output"
},
"preview": {
"label": "Preview",
"display": "ui",
"source": "images_out",
"type": "image",
"value": [
{
"url": "/view/webp/UOARYm2reAw23CfuewLqN/images_out/0?t=1747905887.6861365"
}
]
},
"device": {
"label": "Device",
"type": "string",
"options": {
"cuda:0": {
"index": 0,
"device": "cuda:0",
"host": "",
"label": "cuda:0",
"total_memory": 84987740160
},
"cuda:1": {
"index": 1,
"device": "cuda:1",
"host": "",
"label": "cuda:1",
"total_memory": 84987740160
},
"cuda:2": {
"index": 2,
"device": "cuda:2",
"host": "",
"label": "cuda:2",
"total_memory": 84987740160
},
"cuda:3": {
"index": 3,
"device": "cuda:3",
"host": "",
"label": "cuda:3",
"total_memory": 84987740160
},
"cuda:4": {
"index": 4,
"device": "cuda:4",
"host": "",
"label": "cuda:4",
"total_memory": 4093509632
},
"cpu": {
"index": 0,
"device": "cpu",
"host": "",
"label": "cpu",
"total_memory": null
}
},
"default": "cuda:0",
"value": "cuda:0"
}
},
"groups": {},
"cache": true,
"time": "0.00",
"memory": 0
},
"measured": {
"width": 532,
"height": 874
},
"selected": false,
"dragging": false
},
{
"id": "cvkJ6ydF_r5GFqRXMx7Hf",
"type": "custom",
"position": {
"x": 332.4387648986923,
"y": 1172.593452830556
},
"data": {
"label": "Preview Image",
"module": "BasicImage",
"action": "Preview",
"category": "image",
"params": {
"vae": {
"label": "VAE | Pipeline",
"display": "input",
"type": [
"pipeline",
"vae"
],
"description": "VAE to decode latents. Required only if input images are latents."
},
"images": {
"label": "Images | Latents",
"display": "input",
"type": [
"image",
"latent"
]
},
"images_out": {
"label": "Images",
"display": "output",
"type": "image"
},
"width": {
"label": "Width",
"type": "int",
"display": "output"
},
"height": {
"label": "Height",
"type": "int",
"display": "output"
},
"preview": {
"label": "Preview",
"display": "ui",
"source": "images_out",
"type": "image",
"value": [
{
"url": "/view/webp/cvkJ6ydF_r5GFqRXMx7Hf/images_out/0?t=1747905887.7045686"
}
]
},
"device": {
"label": "Device",
"type": "string",
"options": {
"cuda:0": {
"index": 0,
"device": "cuda:0",
"host": "",
"label": "cuda:0",
"total_memory": 84987740160
},
"cuda:1": {
"index": 1,
"device": "cuda:1",
"host": "",
"label": "cuda:1",
"total_memory": 84987740160
},
"cuda:2": {
"index": 2,
"device": "cuda:2",
"host": "",
"label": "cuda:2",
"total_memory": 84987740160
},
"cuda:3": {
"index": 3,
"device": "cuda:3",
"host": "",
"label": "cuda:3",
"total_memory": 84987740160
},
"cuda:4": {
"index": 4,
"device": "cuda:4",
"host": "",
"label": "cuda:4",
"total_memory": 4093509632
},
"cpu": {
"index": 0,
"device": "cpu",
"host": "",
"label": "cpu",
"total_memory": null
}
},
"default": "cuda:0",
"value": "cuda:0"
}
},
"groups": {},
"cache": true,
"time": "0.00",
"memory": 0
},
"measured": {
"width": 532,
"height": 874
},
"selected": false,
"dragging": false
},
{
"id": "HWujR4q0GVmR2yIcMVvXo",
"type": "custom",
"position": {
"x": -961.8902257530846,
"y": 1169.791484193087
},
"data": {
"label": "Resize",
"module": "BasicImage",
"action": "Resize",
"category": "image",
"params": {
"images": {
"label": "Images",
"display": "input",
"type": "image"
},
"images_out": {
"label": "Images",
"display": "output",
"type": "image"
},
"new_width": {
"label": "Width",
"display": "output",
"type": "int"
},
"new_height": {
"label": "Height",
"display": "output",
"type": "int"
},
"width": {
"label": "Width",
"type": "int",
"default": 1024,
"min": 0,
"value": "512"
},
"height": {
"label": "Height",
"type": "int",
"default": 1024,
"min": 0,
"value": "512"
},
"method": {
"label": "Method",
"type": "string",
"options": [
"stretch",
"keep aspect ratio",
"fit",
"pad"
],
"default": "stretch",
"value": "stretch"
},
"resample": {
"label": "Resample",
"type": "string",
"options": [
"nearest",
"box",
"bilinear",
"hamming",
"bicubic",
"lanczos"
],
"default": "bicubic",
"value": "bicubic"
}
},
"groups": {},
"cache": true,
"time": "0.01",
"memory": 0
},
"measured": {
"width": 346,
"height": 445
},
"selected": false,
"dragging": false
},
{
"id": "np8HTMZhkiQF0AMzrReBS",
"type": "custom",
"position": {
"x": -960.149595888946,
"y": 648.5190722365428
},
"data": {
"label": "Resize",
"module": "BasicImage",
"action": "Resize",
"category": "image",
"params": {
"images": {
"label": "Images",
"display": "input",
"type": "image"
},
"images_out": {
"label": "Images",
"display": "output",
"type": "image"
},
"new_width": {
"label": "Width",
"display": "output",
"type": "int"
},
"new_height": {
"label": "Height",
"display": "output",
"type": "int"
},
"width": {
"label": "Width",
"type": "int",
"default": 1024,
"min": 0,
"value": "512"
},
"height": {
"label": "Height",
"type": "int",
"default": 1024,
"min": 0,
"value": "512"
},
"method": {
"label": "Method",
"type": "string",
"options": [
"stretch",
"keep aspect ratio",
"fit",
"pad"
],
"default": "stretch",
"value": "stretch"
},
"resample": {
"label": "Resample",
"type": "string",
"options": [
"nearest",
"box",
"bilinear",
"hamming",
"bicubic",
"lanczos"
],
"default": "bicubic",
"value": "bicubic"
}
},
"groups": {},
"cache": true,
"time": "0.02",
"memory": 0
},
"measured": {
"width": 346,
"height": 445
},
"selected": false,
"dragging": false
},
{
"id": "zmiFFVgdKfia0ro96IqbA",
"type": "custom",
"position": {
"x": 313.20150040562226,
"y": 338.5448950893251
},
"data": {
"label": "DiffDiffDenoiseBlocks",
"module": "diff-diff.custom",
"action": "DiffDiffDenoise",
"category": "Modular-Diffusers",
"params": {
"image": {
"label": "Image",
"type": "image",
"display": "input"
},
"height": {
"label": "Height",
"type": "int",
"display": "text",
"default": 1024,
"min": 8,
"max": 8192,
"step": 8,
"group": "dimensions",
"value": 1024
},
"width": {
"label": "Width",
"type": "int",
"display": "text",
"default": 1024,
"min": 8,
"max": 8192,
"step": 8,
"group": "dimensions",
"value": 1024
},
"num_inference_steps": {
"label": "Steps",
"type": "int",
"default": 25,
"min": 1,
"max": 1000,
"value": 25
},
"diffdiff_map": {
"type": "image",
"label": "diffdiff_map",
"display": "input"
},
"prompt_embeddings": {
"label": "prompt_embeddings",
"type": "prompt_embeddings",
"display": "input"
},
"ip_adapter_embeds": {
"label": "ip_adapter_embeds",
"type": "ip_adapter_embeds",
"display": "input"
},
"vae": {
"label": "vae",
"type": "diffusers_auto_model",
"display": "input"
},
"scheduler": {
"label": "scheduler",
"type": "scheduler",
"display": "input"
},
"guider": {
"label": "guider",
"type": "guider",
"display": "input"
},
"unet": {
"label": "unet",
"type": "diffusers_auto_model",
"display": "input"
},
"controlnet": {
"label": "controlnet",
"type": "diffusers_auto_model",
"display": "input"
},
"latents": {
"label": "latents",
"type": "latents",
"display": "output"
}
},
"groups": {},
"cache": true,
"time": "13.23",
"memory": 0
},
"measured": {
"width": 492,
"height": 560
},
"selected": false,
"dragging": false
}
],
"edges": [
{
"source": "1odkYiAOIAMJnLYJrfeY3",
"sourceHandle": "text_encoders",
"target": "ImncaWQ8VOjefxwYEgaCT",
"targetHandle": "text_encoders",
"id": "ASfSkl9OLvXzswi3xQndJ",
"style": {
"stroke": "rgb(100, 100, 100)"
}
},
{
"source": "HlyuKg2JBy-2BTKJIoFc5",
"sourceHandle": "images",
"target": "rrRHxl2qIfOwWYviB86VR",
"targetHandle": "images",
"id": "AKlv3fk61FiAWjZ3H7kUG",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "abVoELMdZ0xv594cFncti",
"sourceHandle": "images",
"target": "HWujR4q0GVmR2yIcMVvXo",
"targetHandle": "images",
"id": "5Zlggtwk8QXkH5HNDeEey",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "HWujR4q0GVmR2yIcMVvXo",
"sourceHandle": "images_out",
"target": "UOARYm2reAw23CfuewLqN",
"targetHandle": "images",
"id": "Ex5xWfrHVAQr4pTmJHdqM",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "94KGPHhNdQYt-6l7SmH7F",
"sourceHandle": "images",
"target": "np8HTMZhkiQF0AMzrReBS",
"targetHandle": "images",
"id": "HKIK1dnHpZxZnCgn01UYy",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "np8HTMZhkiQF0AMzrReBS",
"sourceHandle": "images_out",
"target": "cvkJ6ydF_r5GFqRXMx7Hf",
"targetHandle": "images",
"id": "wMuCz3R-41E1Ddt2ww8yp",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "1odkYiAOIAMJnLYJrfeY3",
"sourceHandle": "vae_out",
"target": "HlyuKg2JBy-2BTKJIoFc5",
"targetHandle": "vae",
"id": "2oFcPjoDDbu73dd4vElcM",
"style": {
"stroke": "rgb(100, 100, 100)"
}
},
{
"source": "abVoELMdZ0xv594cFncti",
"sourceHandle": "images",
"target": "zmiFFVgdKfia0ro96IqbA",
"targetHandle": "image",
"id": "oiiC1abo4LqRH4Tws2vIV",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "94KGPHhNdQYt-6l7SmH7F",
"sourceHandle": "images",
"target": "zmiFFVgdKfia0ro96IqbA",
"targetHandle": "diffdiff_map",
"id": "x91SVExM-cVFAj2Yx8-ME",
"style": {
"stroke": "rgb(21, 95, 160)"
}
},
{
"source": "ImncaWQ8VOjefxwYEgaCT",
"sourceHandle": "embeddings",
"target": "zmiFFVgdKfia0ro96IqbA",
"targetHandle": "prompt_embeddings",
"id": "LQRDUqYHerw9PgCSaNteC",
"style": {
"stroke": "rgb(100, 100, 100)"
}
},
{
"source": "1odkYiAOIAMJnLYJrfeY3",
"sourceHandle": "unet_out",
"target": "zmiFFVgdKfia0ro96IqbA",
"targetHandle": "unet",
"id": "5jZYD6TYUR6EnjFA3ZLV4",
"style": {
"stroke": "rgb(100, 100, 100)"
}
},
{
"source": "1odkYiAOIAMJnLYJrfeY3",
"sourceHandle": "scheduler",
"target": "zmiFFVgdKfia0ro96IqbA",
"targetHandle": "scheduler",
"id": "c3oO10mCBAEsgyJ5DBsAj",
"style": {
"stroke": "rgb(100, 100, 100)"
}
},
{
"source": "1odkYiAOIAMJnLYJrfeY3",
"sourceHandle": "vae_out",
"target": "zmiFFVgdKfia0ro96IqbA",
"targetHandle": "vae",
"id": "heZQrlr5HGeWYbUyAJOsV",
"style": {
"stroke": "rgb(100, 100, 100)"
}
},
{
"source": "zmiFFVgdKfia0ro96IqbA",
"sourceHandle": "latents",
"target": "HlyuKg2JBy-2BTKJIoFc5",
"targetHandle": "latents",
"id": "HP5zFiOwyPrSVa-_glz9G",
"style": {
"stroke": "rgb(100, 100, 100)"
}
}
],
"viewport": {
"x": 311.7971183820332,
"y": -151.77704812858502,
"zoom": 0.5207725795536197
}
}