saranimje commited on
Commit
29e6a30
·
verified ·
1 Parent(s): 1580613

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +1 -0
utils.py CHANGED
@@ -214,6 +214,7 @@ def fetch_news_data(company_name: str, article_number: int):
214
 
215
  # remove unwanted unicode characters (like \u2018, \u2019, etc.)
216
  unicode_patterns = re.compile(r'[\u2018\u2019\u2020\u2021\u2014]') # Add more if needed
 
217
  df['article_text'] = df['article_text'].apply(lambda x: unicode_patterns.sub('', x))
218
 
219
  custom_stop_words = list(custom_stop_words)
 
214
 
215
  # remove unwanted unicode characters (like \u2018, \u2019, etc.)
216
  unicode_patterns = re.compile(r'[\u2018\u2019\u2020\u2021\u2014]') # Add more if needed
217
+ df['article_text'] = df['article_text'].fillna('').astype(str)
218
  df['article_text'] = df['article_text'].apply(lambda x: unicode_patterns.sub('', x))
219
 
220
  custom_stop_words = list(custom_stop_words)