DD0101 commited on
Commit
f8e5290
·
1 Parent(s): 5bcc751

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -0
app.py CHANGED
@@ -14,6 +14,7 @@ rdrsegmenter = py_vncorenlp.VnCoreNLP(annotators=["wseg"], save_dir='/home/user/
14
  class MyPipeline(TokenClassificationPipeline):
15
  def preprocess(self, sentence, offset_mapping=None):
16
  truncation = True if self.tokenizer.model_max_length and self.tokenizer.model_max_length > 0 else False
 
17
  model_inputs = self.tokenizer(
18
  sentence,
19
  return_tensors=self.framework,
 
14
  class MyPipeline(TokenClassificationPipeline):
15
  def preprocess(self, sentence, offset_mapping=None):
16
  truncation = True if self.tokenizer.model_max_length and self.tokenizer.model_max_length > 0 else False
17
+ sentence = rdrsegmenter.word_segment(sentence)
18
  model_inputs = self.tokenizer(
19
  sentence,
20
  return_tensors=self.framework,