Spaces:
Sleeping
Sleeping
File size: 3,922 Bytes
81e56fe d7c5b46 81e56fe 6224b77 81e56fe 6224b77 81e56fe 6224b77 d7c5b46 6224b77 81e56fe d7c5b46 81e56fe d7c5b46 81e56fe d7c5b46 81e56fe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
import gradio as gr
import os
import subprocess
from huggingface_hub import snapshot_download, login, HfApi
import tempfile
import shutil
# Function to download, convert, and quantize model to GGUF
def convert_to_gguf(model_id, quantization_method, hf_token=None):
try:
# Create temporary directories for model files and output
temp_dir = tempfile.mkdtemp()
output_dir = tempfile.mkdtemp()
# Log in to Hugging Face if token is provided (for gated models)
if hf_token:
login(hf_token)
# Download the model from Hugging Face
model_path = snapshot_download(repo_id=model_id, local_dir=temp_dir, local_dir_use_symlinks=False)
# Clone llama.cpp repository if not already present
llama_cpp_dir = "/tmp/llama.cpp"
if not os.path.exists(llama_cpp_dir):
subprocess.run(["git", "clone", "https://github.com/ggerganov/llama.cpp.git", llama_cpp_dir], check=True)
# Install llama.cpp Python dependencies
subprocess.run(["pip", "install", "-r", f"{llama_cpp_dir}/requirements.txt"], check=True)
# Create build directory for CMake
build_dir = os.path.join(llama_cpp_dir, "build")
os.makedirs(build_dir, exist_ok=True)
# Run CMake to configure and build
subprocess.run(["cmake", ".."], cwd=build_dir, check=True)
subprocess.run(["cmake", "--build", ".", "--config", "Release"], cwd=build_dir, check=True)
# Convert model to GGUF with f16 precision
intermediate_gguf = os.path.join(output_dir, f"{model_id.replace('/', '-')}-f16.gguf")
convert_cmd = [
"python", f"{llama_cpp_dir}/convert_hf_to_gguf.py",
model_path,
"--outfile", intermediate_gguf,
"--outtype", "f16"
]
subprocess.run(convert_cmd, check=True)
# Quantize the GGUF file to the selected method
output_gguf = os.path.join(output_dir, f"{model_id.replace('/', '-')}-{quantization_method}.gguf")
quantize_cmd = [
os.path.join(build_dir, "bin", "quantize"),
intermediate_gguf,
output_gguf,
quantization_method
]
subprocess.run(quantize_cmd, check=True)
# Return the path to the quantized GGUF file
return output_gguf, f"Model converted and quantized successfully! Download the GGUF file: {output_gguf}"
except Exception as e:
return None, f"Error: {str(e)}"
finally:
# Clean up temporary directories
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
if os.path.exists(llama_cpp_dir):
shutil.rmtree(llama_cpp_dir)
# Gradio interface
def gradio_app():
with gr.Blocks() as demo:
gr.Markdown("# Hugging Face to GGUF Converter")
gr.Markdown("Enter a Hugging Face model ID and select a quantization method to convert the model to GGUF format.")
with gr.Row():
model_id_input = gr.Textbox(label="Hugging Face Model ID", placeholder="e.g., meta-llama/Llama-3.2-3B-Instruct")
hf_token_input = gr.Textbox(label="Hugging Face Token (optional for gated models)", type="password", placeholder="Enter your HF token")
with gr.Row():
quantization_method = gr.Dropdown(
choices=["q8_0", "q4_k_m", "q4_0", "q5_0", "q5_k_m", "q6_k", "q3_k_m"],
label="Quantization Method",
value="q4_k_m"
)
convert_button = gr.Button("Convert to GGUF")
output_file = gr.File(label="Download GGUF File")
output_message = gr.Textbox(label="Status")
convert_button.click(
fn=convert_to_gguf,
inputs=[model_id_input, quantization_method, hf_token_input],
outputs=[output_file, output_message]
)
return demo
if __name__ == "__main__":
gradio_app().launch() |