import os
import json
import re
import hashlib
import gc
from io import BytesIO
from collections import OrderedDict
from PIL import Image, UnidentifiedImageError
import torch
from transformers import AutoProcessor, BitsAndBytesConfig
from transformers.models.qwen2_5_vl import Qwen2_5_VLForConditionalGeneration
from pdf2image import convert_from_bytes
import gradio as gr
import fitz
# --- CONFIGURATION ---
MODEL_ID = "prithivMLmods/Camel-Doc-OCR-062825"
CACHE_MAX_SIZE = 128
DPI = 300 # Giữ vừa đủ, không quá cao
IMAGE_MAX_DIM = None # Không resize nếu không cần
JPEG_QUALITY = 80
GPU_MEMORY_FRACTION = 0.8
# --- 1. Device ---
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.backends.cudnn.benchmark = True
if device.type == 'cuda':
torch.cuda.set_per_process_memory_fraction(GPU_MEMORY_FRACTION, device=0)
# --- 2. Load model ---
# from transformers import AutoProcessor, BitsAndBytesConfig
# from transformers.models.qwen2_5_vl import Qwen2_5_VLForConditionalGeneration
# bnb = BitsAndBytesConfig(
# load_in_4bit=True,
# bnb_4bit_use_double_quant=True,
# bnb_4bit_quant_type="nf4",
# bnb_4bit_compute_dtype=torch.float16
# )
# processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
# model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
# MODEL_ID,
# quantization_config=bnb,
# device_map="auto",
# trust_remote_code=True
# ).eval()
# processor.tokenizer.pad_token_id = processor.tokenizer.eos_token_id
# --- 8. File handler ---
import traceback
from concurrent.futures import ThreadPoolExecutor
# def handle_file(file, prompt, extra_prompt, max_new_tokens, progress=gr.Progress()):
# try:
# file_path = file.name if hasattr(file, "name") else file
# filename = os.path.basename(file_path)
# ext = filename.lower().split('.')[-1]
# full_prompt = (prompt + "\n" + extra_prompt).strip() or ""
# print(f"[INFO] handle_file → {filename} (.{ext})")
# if ext == "pdf":
# try:
# with open(file_path, "rb") as f:
# pdf_bytes = f.read()
# print(f"[INFO] Read PDF bytes: {len(pdf_bytes)} bytes")
# doc = fitz.open(stream=pdf_bytes, filetype="pdf")
# pages = []
# zoom = DPI
# mat = fitz.Matrix(zoom, zoom)
# for i, page in enumerate(doc):
# pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
# img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
# if max(img.size) > 3072:
# img.thumbnail((3072, 3072), Image.Resampling.LANCZOS)
# pages.append(img)
# print(f"[INFO] Converted PDF → {len(pages)} pages")
# except Exception as e:
# traceback.print_exc()
# return filename, f"[ERROR] PDF conversion failed: {e}"
# outputs = []
# with ThreadPoolExecutor(max_workers=4) as executor:
# futures = [executor.submit(run_inference, img, full_prompt, max_new_tokens) for img in pages]
# for idx, future in enumerate(futures):
# try:
# out = future.result()
# except Exception as e:
# traceback.print_exc()
# out = f"[ERROR] Inference page {idx+1} failed: {e}"
# outputs.append(out)
# progress((idx) / len(pages), desc=f"Page {idx+1}/{len(pages)}")
# result = "\n\n--- Page Break ---\n\n".join(outputs)
# print("[INFO] handle_file done")
# return filename, result
# else:
# try:
# img = Image.open(file_path)
# print(f"[INFO] Opened image: {img.mode}, {img.size}")
# except Exception as e:
# traceback.print_exc()
# return filename, f"[ERROR] Image open failed: {e}"
# return filename, run_inference(img, full_prompt, max_new_tokens)
# except Exception as e:
# traceback.print_exc()
# return "error", f"[ERROR] handle_file unexpected: {e}"
import time
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
def handle_file(file, prompt, extra_prompt, max_new_tokens, progress=None):
try:
file_path = file.name if hasattr(file, "name") else file
filename = os.path.basename(file_path)
ext = filename.lower().split('.')[-1]
full_prompt = (prompt + "\n" + extra_prompt).strip() or ""
start_total = time.perf_counter()
if ext == "pdf":
# --- Chuyển PDF sang ảnh ---
start_convert = time.perf_counter()
with open(file_path, "rb") as f:
pdf_bytes = f.read()
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
scale = DPI / 72
mat = fitz.Matrix(scale, scale)
pages = []
for page in doc:
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
if max(img.size) > 3072:
img.thumbnail((3072, 3072), Image.Resampling.LANCZOS)
pages.append(img)
end_convert = time.perf_counter()
# --- Inference từng trang ---
start_infer = time.perf_counter()
outputs = []
for idx, img in enumerate(pages):
out = run_inference(img, full_prompt, max_new_tokens)
outputs.append(out)
if progress:
progress(idx / len(pages), desc=f"Page {idx+1}/{len(pages)}")
end_infer = time.perf_counter()
# --- Tổng hợp kết quả ---
combined_text = "\n\n".join(outputs) # hoặc json.dumps(outputs) tuỳ mục đích
# --- Gọi model tổng hợp cuối ---
start_agg = time.perf_counter()
final_result = run_inference_on_text(combined_text, max_new_tokens) # bạn cần implement hàm này
end_agg = time.perf_counter()
total_time = end_agg - start_total
return filename, (
f"OKE (total time: {total_time:.3f}s, convert: {end_convert - start_convert:.3f}s, "
f"infer per page: {end_infer - start_infer:.3f}s, aggregate: {end_agg - start_agg:.3f}s)\n"
f"{final_result}"
)
else:
# Xử lý ảnh đơn
start_img = time.perf_counter()
img = Image.open(file_path)
if img.mode != "RGB":
img = img.convert("RGB")
end_img = time.perf_counter()
start_infer = time.perf_counter()
result = run_inference(img, full_prompt, max_new_tokens)
end_infer = time.perf_counter()
total_time = end_infer - start_img
return filename, f"OKE (time: {total_time:.3f}s)\n{result}"
except Exception as e:
import traceback
traceback.print_exc()
return "error", f"[ERROR] handle_file failed: {e}"
# def run_inference(img: Image.Image, prompt: str = "", max_new_tokens: int = 512) -> str:
# if img.mode != "RGB":
# img = img.convert("RGB")
# prompt_text = prompt.strip()
# messages = [{
# "role": "user",
# "content": [
# {"type": "image", "image": img},
# {"type": "text", "text": prompt_text}
# ]
# }]
# text_prompt = processor.apply_chat_template(
# messages, tokenize=False, add_generation_prompt=True
# )
# inputs = processor(
# text=[text_prompt], images=[img], return_tensors="pt", padding=True
# ).to(device)
# with torch.inference_mode(), torch.cuda.amp.autocast(enabled=(device.type == 'cuda')):
# gen = model.generate(
# **inputs,
# max_new_tokens=max_new_tokens,
# do_sample=False,
# eos_token_id=processor.tokenizer.eos_token_id
# )
# trimmed = [o[len(i):] for i, o in zip(inputs['input_ids'], gen)]
# result = processor.tokenizer.batch_decode(
# trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=True
# )[0].strip()
# return result
def run_inference(img: Image.Image, prompt: str = "", max_new_tokens: int = 512) -> str:
if img.mode != "RGB":
img = img.convert("RGB")
return f"[DEBUG] Ảnh nhận: size={img.size}, prompt='{prompt[:30]}...'"
def run_inference_on_text(img: Image.Image, prompt: str = "", max_new_tokens: int = 512) -> str:
return f"[DEBUG] Total..."
# --- 9. Prompt templates & JSON export ---
prompt_templates = {
"Electrolux": """Extract all structured information from the delivery order document image.
You must return the result as a valid XML block that strictly follows the structure below.
STRICT INSTRUCTIONS – read carefully and follow EXACTLY:
1. Return **ONLY** the XML block – nothing before or after it.
2. DO NOT add, remove, rename, or reorder any XML tags.
3. DO NOT include explanations, markdown, notes, comments, or extra spacing outside the XML block.
4. For every tag, fill in the exact value read from the image.
• NEVER copy or repeat the label/placeholder text.
• NEVER guess or invent values.
5. If a value is missing or unreadable, leave the tag EMPTY (e.g. ).
6. DO NOT include Vietnamese text or translations inside tag values.
7. The output MUST start with the root tag and end with its correct closing tag; all tags must be well-formed.
8. Dates must be in YYYY-MM-DD format.
9. Boolean tags must be exactly true or false (lower-case, no quotes).
✔ √ Yes Passed ⇒ true | ✘ X No Fail ⇒ false
10. **Inside each value**
• Replace every internal line-break with “, ” (comma + space).
• Trim leading/trailing whitespace.
• Escape XML special characters: & → &, < → <, > → >.
11. **Phone / contact fields** – digits, “+”, “–”, spaces only; if multiple numbers, separate with “, ”.
12. **Signature fields** – fill ONLY if the signature appears as legible text; if it is handwritten, leave the tag empty.
13. Ignore any information not represented by the tags below.
Số lệnh giao nhận hàng
Số đơn hàng
Mã số khách hàng
Mã đơn khách hàng
Ngày đặt hàng của khách
Ngày giao hàng
Ngày giao hàng yêu cầu
Số hóa đơn
Tên công ty gửi hàng
Địa chỉ gửi hàng
Số điện thoại
Số fax
Mã số thuế
Mã khách hàng
Tên công ty nhận hàng
Địa chỉ nhận hàng chi tiết
Tỉnh/Thành phố
Mã bưu chính
Họ tên người lập phiếu
Ngày lập phiếu
Đã ký hay chưa (true hoặc false)
""",
"Jotun": """Extract all structured information from the delivery order document.
You must return the result as a valid XML block that strictly follows the structure below.
STRICT INSTRUCTIONS – read carefully and follow EXACTLY:
1. Return **ONLY** the XML block – nothing before or after it.
2. DO NOT add, remove, rename, or reorder any XML tags.
3. DO NOT include explanations, markdown, notes, comments, or extra spacing outside the XML block.
4. For every tag, fill in the exact value read from the image.
• NEVER copy or repeat the label/placeholder text.
• NEVER guess or invent values.
5. If a value is missing or unreadable, leave the tag EMPTY (e.g. ).
6. DO NOT include Vietnamese text or translations inside tag values.
7. The output MUST start with the root tag and end with its correct closing tag; all tags must be well-formed.
8. Dates must be in YYYY-MM-DD format.
9. Boolean tags must be exactly true or false (lower-case, no quotes).
✔ √ Yes Passed ⇒ true | ✘ X No Fail ⇒ false
10. **Inside each value**
• Replace every internal line-break with “, ” (comma + space).
• Trim leading/trailing whitespace.
• Escape XML special characters: & → &, < → <, > → >.
11. **Phone / contact fields** – digits, “+”, “–”, spaces only; if multiple numbers, separate with “, ”.
12. **Signature fields** – fill ONLY if the signature appears as legible text; if it is handwritten, leave the tag empty.
13. Ignore any information not represented by the tags below.
Số lệnh giao hàng
Số lệnh giao hàng số
Mã khách hàng
Tên khách hàng
Địa chỉ khách hàng
Điện thoại khách hàng
Tên người nhận hóa đơn
Địa chỉ người nhận hóa đơn
Số đơn đặt hàng
Ngày đặt hàng
Số đơn hàng
Ngày giao hàng
Đã ký hay chưa (true hoặc false)
""",
"MAWB": """Extract all structured information from the Master Air Waybill (MAWB) document.
You must return the result as a valid XML block that strictly follows the structure below.
STRICT INSTRUCTIONS – read carefully and follow EXACTLY:
1. Return **ONLY** the XML block – nothing before or after it.
2. DO NOT add, remove, rename, or reorder any XML tags.
3. DO NOT include explanations, markdown, notes, comments, or extra spacing outside the XML block.
4. For every tag, fill in the exact value read from the image.
• NEVER copy or repeat the label/placeholder text.
• NEVER guess or invent values.
5. If a value is missing or unreadable, leave the tag EMPTY (e.g. ).
6. DO NOT include Vietnamese text or translations inside tag values.
7. The output MUST start with the root tag and end with its correct closing tag; all tags must be well-formed.
8. Dates must be in YYYY-MM-DD format.
9. Boolean tags must be exactly true or false (lower-case, no quotes).
✔ √ Yes Passed ⇒ true | ✘ X No Fail ⇒ false
10. **Inside each value**
• Replace every internal line-break with “, ” (comma + space).
• Trim leading/trailing whitespace.
• Escape XML special characters: & → &, < → <, > → >.
11. **Phone / contact fields** – digits, “+”, “–”, spaces only; if multiple numbers, separate with “, ”.
12. **Signature fields** – fill ONLY if the signature appears as legible text; if it is handwritten, leave the tag empty.
13. Ignore any information not represented by the tags below.
Số MAWB
Tên người gửi hàng
Địa chỉ người gửi hàng
Mã tài khoản người gửi
Tên người nhận hàng
Địa chỉ người nhận hàng
Mã tài khoản người nhận
Ghi chú hàng nguy hiểm (true or false)
Chữ ký người gửi
""",
"Phiếu Cân": """Extract all structured information from the document 'PHIẾU CÂN / SHIPPER’S LETTER OF INSTRUCTIONS'.
You must return the result as a valid XML block that strictly follows the structure below.
STRICT INSTRUCTIONS – read carefully and follow EXACTLY:
1. Return **ONLY** the XML block – nothing before or after it.
2. DO NOT add, remove, rename, or reorder any XML tags.
3. DO NOT include explanations, markdown, notes, comments, or extra spacing outside the XML block.
4. For every tag, fill in the exact value read from the image.
• NEVER copy or repeat the label/placeholder text.
• NEVER guess or invent values.
5. If a value is missing or unreadable, leave the tag EMPTY (e.g. ).
6. DO NOT include Vietnamese text or translations inside tag values.
7. The output MUST start with the root tag and end with its correct closing tag; all tags must be well-formed.
8. Dates must be in YYYY-MM-DD format.
9. Boolean tags must be exactly true or false (lower-case, no quotes).
✔ √ Yes Passed ⇒ true | ✘ X No Fail ⇒ false
10. **Inside each value**
• Replace every internal line-break with “, ” (comma + space).
• Trim leading/trailing whitespace.
• Escape XML special characters: & → &, < → <, > → >.
11. **Phone / contact fields** – digits, “+”, “–”, spaces only; if multiple numbers, separate with “, ”.
12. **Signature fields** – fill ONLY if the signature appears as legible text; if it is handwritten, leave the tag empty.
13. Ignore any information not represented by the tags below.
Số AWB
Tên người gửi hàng
Địa chỉ người gửi hàng
Số điện thoại người gửi
Tên người nhận hàng
Địa chỉ người nhận hàng
Tên hàng hóa
Đã kiểm tra an ninh (true/false)
Tên nhân viên tiếp nhận
Chữ ký nhân viên tiếp nhận
""",
"PC 3U": """Extract all structured information from the PC 3U air cargo instruction document.
You must return the result as a valid XML block that strictly follows the structure below.
STRICT INSTRUCTIONS – read carefully and follow EXACTLY:
1. Return **ONLY** the XML block – nothing before or after it.
2. DO NOT add, remove, rename, or reorder any XML tags.
3. DO NOT include explanations, markdown, notes, comments, or extra spacing outside the XML block.
4. For every tag, fill in the exact value read from the image.
• NEVER copy or repeat the label/placeholder text.
• NEVER guess or invent values.
5. If a value is missing or unreadable, leave the tag EMPTY (e.g. ).
6. DO NOT include Vietnamese text or translations inside tag values.
7. The output MUST start with the root tag and end with its correct closing tag; all tags must be well-formed.
8. Dates must be in YYYY-MM-DD format.
9. Boolean tags must be exactly true or false (lower-case, no quotes).
✔ √ Yes Passed ⇒ true | ✘ X No Fail ⇒ false
10. **Inside each value**
• Replace every internal line-break with “, ” (comma + space).
• Trim leading/trailing whitespace.
• Escape XML special characters: & → &, < → <, > → >.
11. **Phone / contact fields** – digits, “+”, “–”, spaces only; if multiple numbers, separate with “, ”.
12. **Signature fields** – fill ONLY if the signature appears as legible text; if it is handwritten, leave the tag empty.
13. Ignore any information not represented by the tags below.
Số AWB
Mã dịch vụ
Tên người gửi
Địa chỉ người gửi
Thông tin liên hệ người gửi
Người thanh toán
Mã số thuế người thanh toán
Tên người nhận
Địa chỉ người nhận
Thông tin liên hệ người nhận
Chữ ký người gửi
Chữ ký nhân viên tiếp nhận
""",
"SLIS-AVS DAD": """Extract all structured information from the document 'TỜ KHAI GỬI HÀNG - SHIPPER’S LETTER OF INSTRUCTION'.
You must return the result as a valid XML block that strictly follows the structure below.
STRICT INSTRUCTIONS – read carefully and follow EXACTLY:
1. Return **ONLY** the XML block – nothing before or after it.
2. DO NOT add, remove, rename, or reorder any XML tags.
3. DO NOT include explanations, markdown, notes, comments, or extra spacing outside the XML block.
4. For every tag, fill in the exact value read from the image.
• NEVER copy or repeat the label/placeholder text.
• NEVER guess or invent values.
5. If a value is missing or unreadable, leave the tag EMPTY (e.g. ).
6. DO NOT include Vietnamese text or translations inside tag values.
7. The output MUST start with the root tag and end with its correct closing tag; all tags must be well-formed.
8. Dates must be in YYYY-MM-DD format.
9. Boolean tags must be exactly true or false (lower-case, no quotes).
✔ √ Yes Passed ⇒ true | ✘ X No Fail ⇒ false
10. **Inside each value**
• Replace every internal line-break with “, ” (comma + space).
• Trim leading/trailing whitespace.
• Escape XML special characters: & → &, < → <, > → >.
11. **Phone / contact fields** – digits, “+”, “–”, spaces only; if multiple numbers, separate with “, ”.
12. **Signature fields** – fill ONLY if the signature appears as legible text; if it is handwritten, leave the tag empty.
13. Ignore any information not represented by the tags below.
Số AWB
Mã biểu mẫu
Tên người gửi
Địa chỉ người gửi
Điện thoại người gửi
Email người gửi
Mã số thuế người gửi
Tên người nhận
Địa chỉ người nhận
Điện thoại người nhận
Email người nhận
Nơi đi
Nơi đến
Tên nhân viên tiếp nhận
Chữ ký nhân viên tiếp nhận
Thời điểm tiếp nhận
Chữ ký người gửi
Ngày ký người gửi
"""
}
def insert_template(name):
return prompt_templates.get(name, "")
def sanitize_filename(name):
return re.sub(r'[^a-zA-Z0-9_\-\.]', '_', name)
def clean_text(text):
text = re.sub(r'<[^<> ]+?>', lambda m: m.group(0).strip(), text)
text = re.sub(r'<[^<>]+?>[^<>]*?<[^<>]+?>', lambda m: m.group(0).strip(), text)
return text.strip()
def export_json(image_name, result_text):
try:
clean_name = sanitize_filename(image_name)
content = {"image": image_name, "text_sequence": clean_text(result_text)}
path = f"/tmp/{clean_name}.json"
with open(path, "w", encoding="utf-8") as f:
json.dump(content, f, ensure_ascii=False, indent=2)
return path, json.dumps(content, ensure_ascii=False, indent=2)
except Exception as e:
return "", f"[Export JSON Failed]: {e}"
# --- 10. Gradio UI ---
css = """
.gradio-textbox textarea {
font-size: 13px !important;
line-height: 1.3 !important;
padding: 6px 8px !important;
}
.gradio-textbox label {
font-size: 13px !important;
font-weight: 600 !important;
margin-bottom: 4px !important;
}
.gradio-button {
font-size: 12px !important;
padding: 4px 8px !important;
height: 28px !important;
min-height: 28px !important;
margin: 2px !important;
}
.gradio-button[data-variant="primary"] {
height: 36px !important;
font-size: 13px !important;
padding: 8px 16px !important;
}
.gradio-file {
font-size: 13px !important;
}
.gradio-file .file-upload {
padding: 8px !important;
min-height: 80px !important;
}
.gradio-markdown h3 {
font-size: 14px !important;
margin: 8px 0 4px 0 !important;
}
.gradio-markdown h2 {
font-size: 18px !important;
margin: 8px 0 !important;
}
.gradio-code {
font-size: 12px !important;
}
"""
with gr.Blocks(title="Camel-Doc-OCR", css=css) as demo:
gr.Markdown("## 🧾 Camel-Doc-OCR (Qwen2.5-VL, 4-bit)")
# --- Main Layout: 2 Columns ---
with gr.Row():
# === LEFT COLUMN: Input ===
with gr.Column(scale=1):
gr.Markdown("### 📥 INPUT")
# File Input
# file_input = gr.File(
# label="📤 Tải ảnh hoặc PDF",
# file_types=[".jpg", ".jpeg", ".png", ".pdf"],
# height=100
# )
# Cho phép tất cả file:
file_input = gr.File(
label="📤 Tải ảnh hoặc PDF",
file_types=None, # ← fix ở đây
height=100
)
# Prompt Input
prompt_input = gr.Textbox(
label="Prompt thuần",
lines=2,
placeholder="Nhập prompt tùy chỉnh...",
max_lines=3
)
# JSON Config
config_input = gr.Textbox(
label="JSON Prompt",
lines=6,
placeholder="Cấu hình JSON sẽ xuất hiện ở đây...",
max_lines=8
)
# Max New Tokens Radio
max_new_tokens_input = gr.Radio(
choices=[128, 256, 512, 1024, 1536, 2048],
value=512,
label="🔢 Chọn max_new_tokens (giới hạn độ dài đầu ra)",
info="Chọn độ dài tối đa cho đầu ra của mô hình"
)
# Prompt Templates
gr.Markdown("### 📑 Mẫu:")
with gr.Row():
for key in list(prompt_templates.keys()): # All buttons in one row
gr.Button(f"{key}", size="sm", scale=1).click(
fn=lambda *, k=key: insert_template(k),
inputs=[],
outputs=config_input
)
# Run Button
run_btn = gr.Button("🚀 Chạy OCR", variant="primary")
# === RIGHT COLUMN: Output ===
with gr.Column(scale=1):
gr.Markdown("### 📤 OUTPUT")
# Result Output
result_output = gr.Textbox(
label="Kết quả trích xuất",
lines=10,
placeholder="Kết quả sẽ hiển thị ở đây sau khi chạy OCR...",
max_lines=12
)
# Export Section
with gr.Row():
export_btn = gr.Button("📦 Xuất JSON", visible=False, variant="secondary", size="sm")
# JSON Output
json_text = gr.Code(
label="JSON Output",
language="json",
lines=6,
visible=False
)
# Download File
json_file = gr.File(
label="File JSON để tải",
visible=False,
file_types=[".json"]
)
# --- Hidden Fields ---
hidden_name = gr.Textbox(visible=False)
# --- Event Handlers ---
# Run Inference
run_btn.click(
fn=handle_file,
inputs=[file_input, prompt_input, config_input, max_new_tokens_input],
outputs=[hidden_name, result_output]
)
# Export JSON
export_btn.click(
fn=export_json,
inputs=[hidden_name, result_output],
outputs=[json_file, json_text]
)
export_btn.click(fn=lambda: gr.update(visible=True), outputs=[json_file])
export_btn.click(fn=lambda: gr.update(visible=True), outputs=[json_text])
if __name__ == "__main__":
demo.launch()