Commit
·
d8359af
1
Parent(s):
c083a98
printing extracted text7
Browse files
main.py
CHANGED
@@ -121,6 +121,30 @@ logging.basicConfig(
|
|
121 |
)
|
122 |
logger = logging.getLogger(__name__)
|
123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
124 |
def initialize_resources():
|
125 |
"""Initialize all required resources"""
|
126 |
try:
|
@@ -766,30 +790,6 @@ def allowed_file(filename, allowed_extensions):
|
|
766 |
return '.' in filename and \
|
767 |
filename.rsplit('.', 1)[1].lower() in allowed_extensions
|
768 |
|
769 |
-
def get_or_load_model(model_name):
|
770 |
-
"""Get a model from cache or load it if not present"""
|
771 |
-
if model_name not in global_models:
|
772 |
-
try:
|
773 |
-
if model_name == 'fasttext':
|
774 |
-
from gensim.models import KeyedVectors
|
775 |
-
log_print(f"Loading {model_name} model...")
|
776 |
-
model_path = os.path.join(gensim_data_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
|
777 |
-
if not os.path.exists(model_path):
|
778 |
-
from gensim.downloader import load
|
779 |
-
log_print("Downloading fasttext model...")
|
780 |
-
global_models[model_name] = load('fasttext-wiki-news-subwords-300')
|
781 |
-
else:
|
782 |
-
global_models[model_name] = KeyedVectors.load_word2vec_format(model_path)
|
783 |
-
log_print(f"Successfully loaded {model_name} model")
|
784 |
-
elif model_name == 'llm':
|
785 |
-
# Implement LLM model loading here
|
786 |
-
log_print("LLM model loading not implemented", "WARNING")
|
787 |
-
return None
|
788 |
-
except Exception as e:
|
789 |
-
log_print(f"Error loading {model_name} model: {e}", "ERROR")
|
790 |
-
raise
|
791 |
-
return global_models.get(model_name)
|
792 |
-
|
793 |
def wait_for_initialization():
|
794 |
"""Wait for initialization to complete"""
|
795 |
initialization_complete.wait()
|
|
|
121 |
)
|
122 |
logger = logging.getLogger(__name__)
|
123 |
|
124 |
+
def get_or_load_model(model_name):
|
125 |
+
"""Get a model from cache or load it if not present"""
|
126 |
+
if model_name not in global_models:
|
127 |
+
try:
|
128 |
+
if model_name == 'fasttext':
|
129 |
+
from gensim.models import KeyedVectors
|
130 |
+
log_print(f"Loading {model_name} model...")
|
131 |
+
model_path = os.path.join(gensim_data_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
|
132 |
+
if not os.path.exists(model_path):
|
133 |
+
from gensim.downloader import load
|
134 |
+
log_print("Downloading fasttext model...")
|
135 |
+
global_models[model_name] = load('fasttext-wiki-news-subwords-300')
|
136 |
+
else:
|
137 |
+
global_models[model_name] = KeyedVectors.load_word2vec_format(model_path)
|
138 |
+
log_print(f"Successfully loaded {model_name} model")
|
139 |
+
elif model_name == 'llm':
|
140 |
+
# Implement LLM model loading here
|
141 |
+
log_print("LLM model loading not implemented", "WARNING")
|
142 |
+
return None
|
143 |
+
except Exception as e:
|
144 |
+
log_print(f"Error loading {model_name} model: {e}", "ERROR")
|
145 |
+
raise
|
146 |
+
return global_models.get(model_name)
|
147 |
+
|
148 |
def initialize_resources():
|
149 |
"""Initialize all required resources"""
|
150 |
try:
|
|
|
790 |
return '.' in filename and \
|
791 |
filename.rsplit('.', 1)[1].lower() in allowed_extensions
|
792 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
793 |
def wait_for_initialization():
|
794 |
"""Wait for initialization to complete"""
|
795 |
initialization_complete.wait()
|