|
|
import gradio as gr |
|
|
import spaces |
|
|
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor |
|
|
from qwen_vl_utils import process_vision_info |
|
|
import torch |
|
|
from PIL import Image |
|
|
import subprocess |
|
|
from datetime import datetime |
|
|
import numpy as np |
|
|
import os |
|
|
|
|
|
|
|
|
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True) |
|
|
|
|
|
|
|
|
MODEL_ID = "Qwen/Qwen2-VL-7B-Instruct" |
|
|
model = Qwen2VLForConditionalGeneration.from_pretrained(MODEL_ID, trust_remote_code=True, torch_dtype=torch.float16).to("cuda").eval() |
|
|
processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True) |
|
|
|
|
|
DESCRIPTION = "[Qwen2-VL-7B Demo](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct)" |
|
|
|
|
|
@spaces.GPU |
|
|
def qwen_inference(media_path, text_input=None): |
|
|
|
|
|
image_extensions = Image.registered_extensions() |
|
|
if media_path.endswith(tuple([i for i, f in image_extensions.items()])): |
|
|
media_type = "image" |
|
|
elif media_path.endswith(("avi", "mp4", "mov", "mkv", "flv", "wmv", "mjpeg", "wav", "gif", "webm", "m4v", "3gp")): |
|
|
media_type = "video" |
|
|
else: |
|
|
raise ValueError("Unsupported media type. Please upload an image or video.") |
|
|
|
|
|
messages = [ |
|
|
{ |
|
|
"role": "user", |
|
|
"content": [ |
|
|
{ |
|
|
"type": media_type, |
|
|
media_type: media_path, |
|
|
**({"fps": 8.0} if media_type == "video" else {}), |
|
|
}, |
|
|
{"type": "text", "text": text_input}, |
|
|
], |
|
|
} |
|
|
] |
|
|
|
|
|
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) |
|
|
image_inputs, video_inputs = process_vision_info(messages) |
|
|
inputs = processor( |
|
|
text=[text], |
|
|
images=image_inputs, |
|
|
videos=video_inputs, |
|
|
padding=True, |
|
|
return_tensors="pt", |
|
|
).to("cuda") |
|
|
|
|
|
generated_ids = model.generate(**inputs, max_new_tokens=1024) |
|
|
generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)] |
|
|
output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] |
|
|
|
|
|
return output_text |
|
|
|
|
|
css = """ |
|
|
#output { |
|
|
height: 500px; |
|
|
overflow: auto; |
|
|
border: 1px solid #ccc; |
|
|
} |
|
|
""" |
|
|
|
|
|
with gr.Blocks(css=css) as demo: |
|
|
gr.Markdown(DESCRIPTION) |
|
|
|
|
|
with gr.Tab(label="Image/Video Input"): |
|
|
with gr.Row(): |
|
|
with gr.Column(): |
|
|
input_media = gr.File(label="Upload Image or Video", type="filepath") |
|
|
text_input = gr.Textbox(label="Question") |
|
|
submit_btn = gr.Button(value="Submit") |
|
|
with gr.Column(): |
|
|
output_text = gr.Textbox(label="Output Text") |
|
|
|
|
|
submit_btn.click(qwen_inference, [input_media, text_input], [output_text]) |
|
|
|
|
|
demo.launch(debug=True) |
