updates

app.py

@@ -1,103 +1,80 @@
-"""
-Gradio demo for UI‑TARS 1.5‑7B (image‑text‑to‑text) on Hugging Face Spaces.
-Save this file as **app.py** and add a *requirements.txt* with the packages
-listed below.  Then create a new **Python** Space, upload both files and
-commit — the Space will build and serve the app automatically.
-
-requirements.txt (suggested versions)
--------------------------------------
-transformers==4.41.0
-accelerate>=0.29.0
-torch>=2.2
-sentencepiece  # needed for many multilingual models
-bitsandbytes    # optional: enables 4‑bit quantization if Space has GPU
-pillow
-gradio>=4.33
-"""
-
-from __future__ import annotations
-
-from typing import List, Dict, Any
-
+# app.py
+import spaces
+import ast
+import torch
+from PIL import Image, ImageDraw
 import gradio as gr
-from PIL import Image
-from transformers import pipeline
-import base64
 
-def load_model():
-    """Load the UI‑TARS multimodal pipeline once at startup."""
-    print("Loading UI‑TARS 1.5‑7B… this may take a while the first time.")
-    return pipeline(
-        "image-text-to-text",
-        model="ByteDance-Seed/UI-TARS-1.5-7B",
-        device_map="auto",  # automatically use GPU if available
-    )
-
-
-pipe = load_model()
+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+from qwen_vl_utils import process_vision_info  # include this file in your repo if not pip-installable
 
+# ---- model & processor loaded on CPU ----
+model = Qwen2VLForConditionalGeneration.from_pretrained(
+    "ByteDance-Seed/UI-TARS-1.5-7B",
+    device_map="auto",
+    torch_dtype=torch.float16,    # CPU-friendly
+)
+processor = AutoProcessor.from_pretrained(
+    "ByteDance-Seed/UI-TARS-1.5-7B",
+    size={"shortest_edge": 256 * 28 * 28, "longest_edge": 1344 * 28 * 28},
+    use_fast=True,
 
-def answer_question(image: Image.Image, question: str) -> str:
-    """Run the model on the provided image & question and return its answer."""
-    if image is None or not question.strip():
-        return "Please supply **both** an image and a question."
-    
-    base64_image = base64.b64encode(image.tobytes()).decode('utf-8')
+)
 
-    # Compose a messages list in the expected multimodal chat format.
-    messages: List[Dict[str, Any]] = [
-        {
-            "role": "user",
-            "content": [
-                {"type": "text", "text": f"You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n```\nThought: ...\nAction: ...\n```\n\n## Action Space\n\nclick(start_box='<|box_start|>(x1, y1)<|box_end|>')\nleft_double(start_box='<|box_start|>(x1, y1)<|box_end|>')\nright_single(start_box='<|box_start|>(x1, y1)<|box_end|>')\ndrag(start_box='<|box_start|>(x1, y1)<|box_end|>', end_box='<|box_start|>(x3, y3)<|box_end|>')\nhotkey(key='')\ntype(content='') #If you want to submit your input, use \"\\n\" at the end of `content`.\nscroll(start_box='<|box_start|>(x1, y1)<|box_end|>', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use Chinese in `Thought` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.\n\n## User Instruction\n{question.strip()}"},
-            ],
-        },
-        {
-            "role":"user",
-            "content": [
-                {"type": "image_url", 
-                "image_url": base64_image},
-            ],
-        }
+def draw_point(image: Image.Image, point=None, radius=5):
+    img = image.copy()
+    if point:
+        x, y = point[0] * img.width, point[1] * img.height
+        ImageDraw.Draw(img).ellipse(
+            (x - radius, y - radius, x + radius, y + radius), fill='red'
+        )
+    return img
+
+@spaces.GPU
+def navigate(image, task, platform):
+    messages = [
+        {"role": "user", "content": f"You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n```\nThought: ...\nAction: ...\n```\n\n## Action Space\n\nclick(start_box='<|box_start|>(x1, y1)<|box_end|>')\nleft_double(start_box='<|box_start|>(x1, y1)<|box_end|>')\nright_single(start_box='<|box_start|>(x1, y1)<|box_end|>')\ndrag(start_box='<|box_start|>(x1, y1)<|box_end|>', end_box='<|box_start|>(x3, y3)<|box_end|>')\nhotkey(key='')\ntype(content='') #If you want to submit your input, use \"\\n\" at the end of `content`.\nscroll(start_box='<|box_start|>(x1, y1)<|box_end|>', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use Chinese in `Thought` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.\n\n## User Instruction\n{task}"},
+        {"role":"user", "content": [
+            {"type": "image_url", "image_url": {"url":image}}
+        ]}
     ]
-
-    # The pipeline returns a list with one dict when `messages` is passed via
-    # the `text` keyword. We extract the generated text robustly.
-    outputs = pipe(text=messages)
-
-    if isinstance(outputs, list):
-        first = outputs[0]
-        if isinstance(first, dict) and "generated_text" in first:
-            return first["generated_text"].strip()
-        return str(first)
-
-    return str(outputs)
+    # prepare inputs
+    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    images, videos = process_vision_info(messages)
+    inputs = processor(text=[text], images=images, videos=videos, padding=True, return_tensors="pt")
+    inputs = inputs.to("cuda")
+
+    # generate
+    generated = model.generate(**inputs, max_new_tokens=128)
+    trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated)]
+    out = processor.batch_decode(trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+
+    # optionally parse JSON and draw point
+    try:
+        actions = ast.literal_eval(out)
+        for act in actions if isinstance(actions, list) else [actions]:
+            pos = act.get('position')
+            if pos and isinstance(pos, list) and len(pos)==2:
+                image = draw_point(image, pos)
+        return image, out
+    except:
+        return image, out
 
 
 demo = gr.Interface(
-    fn=answer_question,
+    fn=navigate,
     inputs=[
-        gr.Image(type="pil", label="Upload image"),
-        gr.Textbox(label="Ask a question about the image", placeholder="e.g. What animal is on the candy?"),
-    ],
-    outputs=gr.Textbox(label="UI‑TARS answer"),
-    title="UI‑TARS 1.5‑7B – Visual Q&A",
-    description=(
-        "Upload an image and ask a question. The **UI‑TARS 1.5‑7B** model will "
-        "answer based on the visual content. Runs completely on‑device in this Space."
-    ),
-    examples=[
-        [
-            "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG",
-            "What animal is on the candy?",
-        ]
+        gr.Image(type="pil", label="Screenshot"),
+        gr.Textbox(lines=1, placeholder="e.g. Search the weather for New York", label="Task"),
+        gr.Dropdown(choices=["web", "phone"], value="web", label="Platform"),
     ],
-    cache_examples=True,
-    allow_flagging="never",
+    outputs=[gr.Image(label="With Click Point"), gr.Textbox(label="Raw Action JSON")],
+    title="ShowUI-2B Navigation Demo",
 )
 
-
-if __name__ == "__main__":
-    # Spaces automatically call `demo.launch()`, but running locally this
-    # guard lets you execute `python app.py` for quick tests.
-    demo.launch()
+demo.launch(
+    server_name="0.0.0.0",
+    server_port=7860,
+    share=False,      # or True if you need a public link
+    ssr_mode=False,   # turn off experimental SSR so the process blocks
+)

app1.py

@@ -0,0 +1,103 @@
+"""
+Gradio demo for UI‑TARS 1.5‑7B (image‑text‑to‑text) on Hugging Face Spaces.
+Save this file as **app.py** and add a *requirements.txt* with the packages
+listed below.  Then create a new **Python** Space, upload both files and
+commit — the Space will build and serve the app automatically.
+
+requirements.txt (suggested versions)
+-------------------------------------
+transformers==4.41.0
+accelerate>=0.29.0
+torch>=2.2
+sentencepiece  # needed for many multilingual models
+bitsandbytes    # optional: enables 4‑bit quantization if Space has GPU
+pillow
+gradio>=4.33
+"""
+
+from __future__ import annotations
+
+from typing import List, Dict, Any
+
+import gradio as gr
+from PIL import Image
+from transformers import pipeline
+import base64
+
+def load_model():
+    """Load the UI‑TARS multimodal pipeline once at startup."""
+    print("Loading UI‑TARS 1.5‑7B… this may take a while the first time.")
+    return pipeline(
+        "image-text-to-text",
+        model="ByteDance-Seed/UI-TARS-1.5-7B",
+        device_map="auto",  # automatically use GPU if available
+    )
+
+
+pipe = load_model()
+
+
+def answer_question(image: Image.Image, question: str) -> str:
+    """Run the model on the provided image & question and return its answer."""
+    if image is None or not question.strip():
+        return "Please supply **both** an image and a question."
+    
+    base64_image = base64.b64encode(image.tobytes()).decode('utf-8')
+
+    # Compose a messages list in the expected multimodal chat format.
+    messages: List[Dict[str, Any]] = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": f"You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n```\nThought: ...\nAction: ...\n```\n\n## Action Space\n\nclick(start_box='<|box_start|>(x1, y1)<|box_end|>')\nleft_double(start_box='<|box_start|>(x1, y1)<|box_end|>')\nright_single(start_box='<|box_start|>(x1, y1)<|box_end|>')\ndrag(start_box='<|box_start|>(x1, y1)<|box_end|>', end_box='<|box_start|>(x3, y3)<|box_end|>')\nhotkey(key='')\ntype(content='') #If you want to submit your input, use \"\\n\" at the end of `content`.\nscroll(start_box='<|box_start|>(x1, y1)<|box_end|>', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use Chinese in `Thought` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.\n\n## User Instruction\n{question.strip()}"},
+            ],
+        },
+        {
+            "role":"user",
+            "content": [
+                {"type": "image_url", 
+                "image_url": base64_image},
+            ],
+        }
+    ]
+
+    # The pipeline returns a list with one dict when `messages` is passed via
+    # the `text` keyword. We extract the generated text robustly.
+    outputs = pipe(text=messages)
+
+    if isinstance(outputs, list):
+        first = outputs[0]
+        if isinstance(first, dict) and "generated_text" in first:
+            return first["generated_text"].strip()
+        return str(first)
+
+    return str(outputs)
+
+
+demo = gr.Interface(
+    fn=answer_question,
+    inputs=[
+        gr.Image(type="pil", label="Upload image"),
+        gr.Textbox(label="Ask a question about the image", placeholder="e.g. What animal is on the candy?"),
+    ],
+    outputs=gr.Textbox(label="UI‑TARS answer"),
+    title="UI‑TARS 1.5‑7B – Visual Q&A",
+    description=(
+        "Upload an image and ask a question. The **UI‑TARS 1.5‑7B** model will "
+        "answer based on the visual content. Runs completely on‑device in this Space."
+    ),
+    examples=[
+        [
+            "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG",
+            "What animal is on the candy?",
+        ]
+    ],
+    cache_examples=True,
+    allow_flagging="never",
+)
+
+
+if __name__ == "__main__":
+    # Spaces automatically call `demo.launch()`, but running locally this
+    # guard lets you execute `python app.py` for quick tests.
+    demo.launch()