#!/usr/bin/python3 # -*- coding: utf-8 -*- """ Business logic functions for IC-Custom application. """ import numpy as np import torch import cv2 import gradio as gr from PIL import Image from datetime import datetime import json import os from scipy.ndimage import binary_dilation, binary_erosion from constants import ( DEFAULT_BACKGROUND_BLEND_THRESHOLD, DEFAULT_SEED, DEFAULT_NUM_IMAGES, DEFAULT_GUIDANCE, DEFAULT_TRUE_GS, DEFAULT_NUM_STEPS, DEFAULT_ASPECT_RATIO, DEFAULT_DILATION_KERNEL_SIZE, DEFAULT_MARKER_SIZE, DEFAULT_MARKER_THICKNESS, DEFAULT_MASK_ALPHA, DEFAULT_COLOR_ALPHA, TIMESTAMP_FORMAT, SEGMENTATION_COLORS, SEGMENTATION_MARKERS ) from utils import run_vlm, construct_vlm_gen_prompt, construct_vlm_polish_prompt device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Global holder for SAM mobile predictor injected from the app layer MOBILE_PREDICTOR = None BEN2_MODEL = None # ben2 model injected from the app layer def set_mobile_predictor(predictor): """Inject SAM mobile predictor into this module without changing function signatures.""" global MOBILE_PREDICTOR MOBILE_PREDICTOR = predictor def set_ben2_model(ben2_model): """Inject ben2 model into this module without changing function signatures.""" global BEN2_MODEL BEN2_MODEL = ben2_model def set_vlm_processor(vlm_processor): """Inject vlm processor into this module without changing function signatures.""" global VLM_PROCESSOR VLM_PROCESSOR = vlm_processor def set_vlm_model(vlm_model): """Inject vlm model into this module without changing function signatures.""" global VLM_MODEL VLM_MODEL = vlm_model def init_image_target_1(target_image): """Initialize UI state when a target image is uploaded.""" # Handle both PIL Image (image_target_1) and ImageEditor dict (image_target_2) try: if isinstance(target_image, dict) and 'composite' in target_image: # ImageEditor format (user-drawn mask) image_target_state = np.array(target_image['composite'].convert("RGB")) else: # PIL Image format (precise mask) image_target_state = np.array(target_image.convert("RGB")) except Exception as e: # If there's an error processing the image, skip initialization return ( gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.update(value="-1") ) selected_points = [] mask_target_state = None prompt = None mask_gallery = [] result_gallery = [] use_background_preservation = False background_blend_threshold = DEFAULT_BACKGROUND_BLEND_THRESHOLD seed = DEFAULT_SEED num_images_per_prompt = DEFAULT_NUM_IMAGES guidance = DEFAULT_GUIDANCE true_gs = DEFAULT_TRUE_GS num_steps = DEFAULT_NUM_STEPS aspect_ratio_val = gr.update(value=DEFAULT_ASPECT_RATIO) return (image_target_state, selected_points, mask_target_state, prompt, mask_gallery, result_gallery, use_background_preservation, background_blend_threshold, seed, num_images_per_prompt, guidance, true_gs, num_steps, aspect_ratio_val) def init_image_target_2(target_image): """Initialize UI state when a target image is uploaded.""" # Handle both PIL Image (image_target_1) and ImageEditor dict (image_target_2) try: if isinstance(target_image, dict) and 'composite' in target_image: # ImageEditor format (user-drawn mask) image_target_state = np.array(target_image['composite'].convert("RGB")) else: # PIL Image format (precise mask) image_target_state = np.array(target_image.convert("RGB")) except Exception as e: # If there's an error processing the image, skip initialization return ( gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.update(value="-1") ) selected_points = gr.skip() mask_target_state = gr.skip() prompt = gr.skip() mask_gallery = gr.skip() result_gallery = gr.skip() use_background_preservation = gr.skip() background_blend_threshold = gr.skip() seed = gr.skip() num_images_per_prompt = gr.skip() guidance = gr.skip() true_gs = gr.skip() num_steps = gr.skip() aspect_ratio_val = gr.skip() return (image_target_state, selected_points, mask_target_state, prompt, mask_gallery, result_gallery, use_background_preservation, background_blend_threshold, seed, num_images_per_prompt, guidance, true_gs, num_steps, aspect_ratio_val) def init_image_reference(image_reference): """Initialize all UI states when a reference image is uploaded.""" image_reference_state = np.array(image_reference.convert("RGB")) image_reference_ori_state = image_reference_state image_reference_rmbg_state = None image_target_state = None mask_target_state = None prompt = None mask_gallery = [] result_gallery = [] image_target_1_val = None image_target_2_val = None selected_points = [] input_mask_mode_val = gr.update(value="Precise mask") seg_ref_mode_val = gr.update(value="Full Ref") move_to_center = False use_background_preservation = False background_blend_threshold = DEFAULT_BACKGROUND_BLEND_THRESHOLD seed = DEFAULT_SEED num_images_per_prompt = DEFAULT_NUM_IMAGES guidance = DEFAULT_GUIDANCE true_gs = DEFAULT_TRUE_GS num_steps = DEFAULT_NUM_STEPS aspect_ratio_val = gr.update(value=DEFAULT_ASPECT_RATIO) return ( image_reference_ori_state, image_reference_rmbg_state, image_target_state, mask_target_state, prompt, mask_gallery, result_gallery, image_target_1_val, image_target_2_val, selected_points, input_mask_mode_val, seg_ref_mode_val, move_to_center, use_background_preservation, background_blend_threshold, seed, num_images_per_prompt, guidance, true_gs, num_steps, aspect_ratio_val, ) def undo_seg_points(orig_img, sel_pix): """Remove the latest segmentation point and recompute the preview mask.""" if len(sel_pix) != 0: temp = orig_img.copy() sel_pix.pop() # Online show seg mask if len(sel_pix) != 0: temp, output_mask = segmentation(temp, sel_pix, MOBILE_PREDICTOR, SEGMENTATION_COLORS, SEGMENTATION_MARKERS) output_mask_pil = Image.fromarray(output_mask.astype("uint8")) masked_img_pil = Image.fromarray(np.where(output_mask > 0, orig_img, 0).astype("uint8")) mask_gallery = [masked_img_pil, output_mask_pil] else: output_mask = None mask_gallery = [] return temp.astype(np.uint8), output_mask, mask_gallery else: gr.Warning("Nothing to Undo") return orig_img, None, [] def segmentation(img, sel_pix, mobile_predictor, colors, markers): """Run SAM-based segmentation given selected points and return previews.""" points = [] labels = [] for p, l in sel_pix: points.append(p) labels.append(l) mobile_predictor.set_image(img if isinstance(img, np.ndarray) else np.array(img)) with torch.no_grad(): masks, _, _ = mobile_predictor.predict( point_coords=np.array(points), point_labels=np.array(labels), multimask_output=False ) output_mask = np.ones((masks.shape[1], masks.shape[2], 3)) * 255 for i in range(3): output_mask[masks[0] == True, i] = 0.0 mask_all = np.ones((masks.shape[1], masks.shape[2], 3)) color_mask = np.random.random((1, 3)).tolist()[0] for i in range(3): mask_all[masks[0] == True, i] = color_mask[i] masked_img = img / 255 * DEFAULT_MASK_ALPHA + mask_all * DEFAULT_COLOR_ALPHA masked_img = masked_img * 255 # Draw points for point, label in sel_pix: cv2.drawMarker( masked_img, point, colors[label], markerType=markers[label], markerSize=DEFAULT_MARKER_SIZE, thickness=DEFAULT_MARKER_THICKNESS ) return masked_img, output_mask def get_point(img, sel_pix, evt: gr.SelectData): """Handle a user click on the target image to add a foreground point.""" if evt is None or not hasattr(evt, 'index'): gr.Warning(f"Event object missing index attribute. Event type: {type(evt)}") return img, None, [] sel_pix.append((evt.index, 1)) # append the foreground_point # Online show seg mask global MOBILE_PREDICTOR masked_img_seg, output_mask = segmentation(img, sel_pix, MOBILE_PREDICTOR, SEGMENTATION_COLORS, SEGMENTATION_MARKERS) # Apply dilation to output_mask output_mask = 1 - output_mask kernel = np.ones((DEFAULT_DILATION_KERNEL_SIZE, DEFAULT_DILATION_KERNEL_SIZE), np.uint8) output_mask = cv2.dilate(output_mask, kernel, iterations=1) output_mask = 1 - output_mask output_mask_binary = output_mask / 255 masked_img_seg = masked_img_seg.astype("uint8") output_mask = output_mask.astype("uint8") masked_img = img * output_mask_binary masked_img_pil = Image.fromarray(masked_img.astype("uint8")) output_mask_pil = Image.fromarray(output_mask.astype("uint8")) outputs_gallery = [masked_img_pil, output_mask_pil] return masked_img_seg, output_mask, outputs_gallery def get_brush(img): """Extract a mask from ImageEditor brush layers or composite/background diff.""" if img is None or not isinstance(img, dict): return gr.skip(), gr.skip() layers = img.get("layers", []) background = img.get('background', None) composite = img.get('composite', None) output_mask = None if layers and layers[0] is not None and background is not None: output_mask = 255 - np.array(layers[0].convert("RGB")).astype(np.uint8) elif composite is not None and background is not None: comp_rgb = np.array(composite.convert("RGB")).astype(np.int16) bg_rgb = np.array(background.convert("RGB")).astype(np.int16) diff = np.abs(comp_rgb - bg_rgb) painted = (diff.sum(axis=2) > 0).astype(np.uint8) output_mask = (1 - painted) * 255 output_mask = np.repeat(output_mask[:, :, None], 3, axis=2).astype(np.uint8) else: return gr.skip(), gr.skip() if len(np.unique(output_mask)) == 1: return gr.skip(), gr.skip() img = np.array(background.convert("RGB")).astype(np.uint8) output_mask_binary = output_mask / 255 masked_img = img * output_mask_binary masked_img_pil = Image.fromarray(masked_img.astype("uint8")) output_mask_pil = Image.fromarray(output_mask.astype("uint8")) mask_gallery = [masked_img_pil, output_mask_pil] return output_mask, mask_gallery def random_mask_func(mask, dilation_type='square', dilation_size=20): """Utility to dilate/erode/box/ellipse expand a binary mask.""" binary_mask = mask[:,:,0] < 128 if dilation_type == 'square_dilation': structure = np.ones((dilation_size, dilation_size), dtype=bool) dilated_mask = binary_dilation(binary_mask, structure=structure) elif dilation_type == 'square_erosion': structure = np.ones((dilation_size, dilation_size), dtype=bool) dilated_mask = binary_erosion(binary_mask, structure=structure) elif dilation_type == 'bounding_box': # Find the most left top and left bottom point rows, cols = np.where(binary_mask) if len(rows) == 0 or len(cols) == 0: return mask # return original mask if no valid points min_row, max_row = np.min(rows), np.max(rows) min_col, max_col = np.min(cols), np.max(cols) # Create a bounding box dilated_mask = np.zeros_like(binary_mask, dtype=bool) dilated_mask[min_row:max_row + 1, min_col:max_col + 1] = True elif dilation_type == 'bounding_ellipse': # Find the most left top and left bottom point rows, cols = np.where(binary_mask) if len(rows) == 0 or len(cols) == 0: return mask # return original mask if no valid points min_row, max_row = np.min(rows), np.max(rows) min_col, max_col = np.min(cols), np.max(cols) # Calculate the center and axis length of the ellipse center = ((min_col + max_col) // 2, (min_row + max_row) // 2) a = (max_col - min_col) // 2 # half long axis b = (max_row - min_row) // 2 # half short axis # Create a bounding ellipse y, x = np.ogrid[:mask.shape[0], :mask.shape[1]] ellipse_mask = ((x - center[0])**2 / a**2 + (y - center[1])**2 / b**2) <= 1 dilated_mask = np.zeros_like(binary_mask, dtype=bool) dilated_mask[ellipse_mask] = True else: raise ValueError("dilation_type must be 'square', 'ellipse', 'bounding_box', or 'bounding_ellipse'") # Use binary dilation dilated_mask = 1 - dilated_mask dilated_mask = np.uint8(dilated_mask[:,:,np.newaxis]) * 255 dilated_mask = np.concatenate([dilated_mask, dilated_mask, dilated_mask], axis=2) return dilated_mask def dilate_mask(mask, image): """Dilate the target mask for robustness and preview the result.""" if mask is None: gr.Warning("Please input the target mask first") return None, None mask = random_mask_func(mask, dilation_type='square_dilation', dilation_size=DEFAULT_DILATION_KERNEL_SIZE) masked_img = image * (mask > 0) return mask, [masked_img, mask] def erode_mask(mask, image): """Erode the target mask and preview the result.""" if mask is None: gr.Warning("Please input the target mask first") return None, None mask = random_mask_func(mask, dilation_type='square_erosion', dilation_size=DEFAULT_DILATION_KERNEL_SIZE) masked_img = image * (mask > 0) return mask, [masked_img, mask] def bounding_box(mask, image): """Create bounding box mask and preview the result.""" if mask is None: gr.Warning("Please input the target mask first") return None, None mask = random_mask_func(mask, dilation_type='bounding_box', dilation_size=DEFAULT_DILATION_KERNEL_SIZE) masked_img = image * (mask > 0) return mask, [masked_img, mask] def change_input_mask_mode(input_mask_mode, custmization_mode): """Change visibility of input mask mode components.""" if custmization_mode == "Position-free": return ( gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), ) elif input_mask_mode.lower() == "precise mask": return ( gr.update(visible=True), gr.update(visible=False), gr.update(visible=True), ) elif input_mask_mode.lower() == "user-drawn mask": return ( gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), ) else: gr.Warning("Invalid input mask mode") return ( gr.skip(), gr.skip(), gr.skip() ) def change_custmization_mode(custmization_mode, input_mask_mode): """Change visibility and interactivity based on customization mode.""" if custmization_mode.lower() == "position-free": return (gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False), gr.update(value="Select a input mask mode", visible=False), gr.update(value="Input target image & mask (Iterate clicking or brushing until the target is covered)", visible=False), gr.update(value="View or modify the target mask", visible=False), gr.update(value="3\. Input text prompt (necessary)"), gr.update(value="4\. Submit and view the output"), gr.update(visible=False), gr.update(visible=False), ) else: if input_mask_mode.lower() == "precise mask": return (gr.update(interactive=True, visible=True), gr.update(interactive=True, visible=False), gr.update(interactive=True, visible=True), gr.update(interactive=True, visible=True), gr.update(interactive=True, visible=True), gr.update(interactive=True, visible=True), gr.update(value="3\. Select a input mask mode", visible=True), gr.update(value="4\. Input target image & mask (Iterate clicking or brushing until the target is covered)", visible=True), gr.update(value="6\. View or modify the target mask", visible=True), gr.update(value="5\. Input text prompt (optional)", visible=True), gr.update(value="7\. Submit and view the output", visible=True), gr.update(visible=True, value="Precise mask"), gr.update(visible=True), ) elif input_mask_mode.lower() == "user-drawn mask": return (gr.update(interactive=True, visible=False), gr.update(interactive=True, visible=True), gr.update(interactive=False, visible=False), gr.update(interactive=True, visible=True), gr.update(interactive=True, visible=True), gr.update(interactive=True, visible=True), gr.update(value="3\. Select a input mask mode", visible=True), gr.update(value="4\. Input target image & mask (Iterate clicking or brushing until the target is covered)", visible=True), gr.update(value="6\. View or modify the target mask", visible=True), gr.update(value="5\. Input text prompt (optional)", visible=True), gr.update(value="7\. Submit and view the output", visible=True), gr.update(visible=True, value="User-drawn mask"), gr.update(visible=True), ) def change_seg_ref_mode(seg_ref_mode, image_reference_state, move_to_center): """Change segmentation reference mode and handle background removal.""" if image_reference_state is None: gr.Warning("Please upload the reference image first") return None, None global BEN2_MODEL if seg_ref_mode == "Full Ref": return image_reference_state, None else: if BEN2_MODEL is None: gr.Warning("Please enable ben2 for mask reference first") return gr.skip(), gr.skip() image_reference_pil = Image.fromarray(image_reference_state) image_reference_pil_rmbg = BEN2_MODEL.inference(image_reference_pil, move_to_center=move_to_center) image_reference_rmbg = np.array(image_reference_pil_rmbg) return image_reference_rmbg, image_reference_rmbg def vlm_auto_generate(image_target_state, image_reference_state, mask_target_state, custmization_mode): """Auto-generate prompt using VLM.""" global VLM_PROCESSOR, VLM_MODEL if custmization_mode == "Position-aware": if image_target_state is None or mask_target_state is None: gr.Warning("Please upload the target image and get mask first") return None if image_reference_state is None: gr.Warning("Please upload the reference image first") return None if VLM_PROCESSOR is None or VLM_MODEL is None: gr.Warning("Please enable vlm for prompt first") return None messages = construct_vlm_gen_prompt(image_target_state, image_reference_state, mask_target_state, custmization_mode) output_text = run_vlm(VLM_PROCESSOR, VLM_MODEL, messages, device=device) return output_text def vlm_auto_polish(prompt, custmization_mode): """Auto-polish prompt using VLM.""" global VLM_PROCESSOR, VLM_MODEL if prompt is None: gr.Warning("Please input the text prompt first") return None if custmization_mode == "Position-aware": gr.Warning("Polishing only works in position-free mode") return prompt if VLM_PROCESSOR is None or VLM_MODEL is None: gr.Warning("Please enable vlm for prompt first") return prompt messages = construct_vlm_polish_prompt(prompt) output_text = run_vlm(VLM_PROCESSOR, VLM_MODEL, messages, device=device) return output_text def save_results(output_img, image_reference, image_target, mask_target, prompt, custmization_mode, input_mask_mode, seg_ref_mode, seed, guidance, num_steps, num_images_per_prompt, use_background_preservation, background_blend_threshold, true_gs, assets_cache_dir): """Save generated results and metadata.""" save_name = datetime.now().strftime(TIMESTAMP_FORMAT) results = [] for i in range(num_images_per_prompt): save_dir = os.path.join(assets_cache_dir, save_name) os.makedirs(save_dir, exist_ok=True) output_img[i].save(os.path.join(save_dir, f"img_gen_{i}.png")) image_reference.save(os.path.join(save_dir, f"img_ref_{i}.png")) image_target.save(os.path.join(save_dir, f"img_target_{i}.png")) mask_target.save(os.path.join(save_dir, f"mask_target_{i}.png")) with open(os.path.join(save_dir, f"hyper_params_{i}.json"), "w") as f: json.dump({ "prompt": prompt, "custmization_mode": custmization_mode, "input_mask_mode": input_mask_mode, "seg_ref_mode": seg_ref_mode, "seed": seed, "guidance": guidance, "num_steps": num_steps, "num_images_per_prompt": num_images_per_prompt, "use_background_preservation": use_background_preservation, "background_blend_threshold": background_blend_threshold, "true_gs": true_gs, }, f) results.append(output_img[i]) return results