yamanavijayavardhan commited on
Commit
d8359af
·
1 Parent(s): c083a98

printing extracted text7

Browse files
Files changed (1) hide show
  1. main.py +24 -24
main.py CHANGED
@@ -121,6 +121,30 @@ logging.basicConfig(
121
  )
122
  logger = logging.getLogger(__name__)
123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  def initialize_resources():
125
  """Initialize all required resources"""
126
  try:
@@ -766,30 +790,6 @@ def allowed_file(filename, allowed_extensions):
766
  return '.' in filename and \
767
  filename.rsplit('.', 1)[1].lower() in allowed_extensions
768
 
769
- def get_or_load_model(model_name):
770
- """Get a model from cache or load it if not present"""
771
- if model_name not in global_models:
772
- try:
773
- if model_name == 'fasttext':
774
- from gensim.models import KeyedVectors
775
- log_print(f"Loading {model_name} model...")
776
- model_path = os.path.join(gensim_data_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
777
- if not os.path.exists(model_path):
778
- from gensim.downloader import load
779
- log_print("Downloading fasttext model...")
780
- global_models[model_name] = load('fasttext-wiki-news-subwords-300')
781
- else:
782
- global_models[model_name] = KeyedVectors.load_word2vec_format(model_path)
783
- log_print(f"Successfully loaded {model_name} model")
784
- elif model_name == 'llm':
785
- # Implement LLM model loading here
786
- log_print("LLM model loading not implemented", "WARNING")
787
- return None
788
- except Exception as e:
789
- log_print(f"Error loading {model_name} model: {e}", "ERROR")
790
- raise
791
- return global_models.get(model_name)
792
-
793
  def wait_for_initialization():
794
  """Wait for initialization to complete"""
795
  initialization_complete.wait()
 
121
  )
122
  logger = logging.getLogger(__name__)
123
 
124
+ def get_or_load_model(model_name):
125
+ """Get a model from cache or load it if not present"""
126
+ if model_name not in global_models:
127
+ try:
128
+ if model_name == 'fasttext':
129
+ from gensim.models import KeyedVectors
130
+ log_print(f"Loading {model_name} model...")
131
+ model_path = os.path.join(gensim_data_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
132
+ if not os.path.exists(model_path):
133
+ from gensim.downloader import load
134
+ log_print("Downloading fasttext model...")
135
+ global_models[model_name] = load('fasttext-wiki-news-subwords-300')
136
+ else:
137
+ global_models[model_name] = KeyedVectors.load_word2vec_format(model_path)
138
+ log_print(f"Successfully loaded {model_name} model")
139
+ elif model_name == 'llm':
140
+ # Implement LLM model loading here
141
+ log_print("LLM model loading not implemented", "WARNING")
142
+ return None
143
+ except Exception as e:
144
+ log_print(f"Error loading {model_name} model: {e}", "ERROR")
145
+ raise
146
+ return global_models.get(model_name)
147
+
148
  def initialize_resources():
149
  """Initialize all required resources"""
150
  try:
 
790
  return '.' in filename and \
791
  filename.rsplit('.', 1)[1].lower() in allowed_extensions
792
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
793
  def wait_for_initialization():
794
  """Wait for initialization to complete"""
795
  initialization_complete.wait()