Trying ONNX models on CPU.

app.py

@@ -4,9 +4,10 @@ import gradio
 import json
 import time
 from datetime import datetime
+from transformers import pipeline
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from optimum.pipelines import pipeline
+from optimum.onnxruntime import ORTModelForSequenceClassification
 
 # CORS Config - This isn't actually working; instead, I am taking a gross approach to origin whitelisting within the service.
 app = FastAPI()
@@ -24,12 +25,17 @@ print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")
 
 # "xenova/mobilebert-uncased-mnli" "typeform/mobilebert-uncased-mnli" Fast but small--same as bundled in Statosphere
 
-# model_name = "MoritzLaurer/deberta-v3-base-zeroshot-v2.0"
-# tokenizer_name = "MoritzLaurer/deberta-v3-base-zeroshot-v2.0"
-model_name = "MoritzLaurer/ModernBERT-large-zeroshot-v2.0"
+model_name = "MoritzLaurer/deberta-v3-base-zeroshot-v2.0"
+tokenizer_name = "MoritzLaurer/deberta-v3-base-zeroshot-v2.0"
+file_name = "onnx/model.onnx"
 
-classifier_cpu = pipeline(task="zero-shot-classification", model=model_name, provider="CPUExecutionProvider")
-classifier_gpu = pipeline(task="zero-shot-classification", model=model_name, provider="CUDAExecutionProvider", device="cuda:0")
+model_name_cpu = "MoritzLaurer/ModernBERT-large-zeroshot-v2.0"
+
+model_cpu = ORTModelForSequenceClassification.from_pretrained(model_id=model_name_cpu, file_name=file_name)
+tokenizer_cpu = AutoTokenizer.from_pretrained(model_name_cpu)
+
+classifier_cpu = pipeline(task="zero-shot-classification", model=model_cpu, tokenizer=tokenizer_cpu)
+classifier_gpu = pipeline(task="zero-shot-classification", model=model_name, tokenizer=tokenizer_name, device="cuda:0")
 
 def classify(data_string, request: gradio.Request):
     if request:

requirements.txt

@@ -4,5 +4,5 @@ huggingface_hub==0.32.4
 json5==0.9.25
 numpy
 uvicorn
-optimum[pipelines,exporters,onnxruntime-gpu]==1.25.3
+optimum[exporters,onnxruntime,onnxruntime-gpu]==1.25.3
 transformers==4.51.3