Spaces:
Running
Running
| """ adapted from https://github.com/keithito/tacotron """ | |
| ''' | |
| Cleaners are transformations that run over the input text at both training and eval time. | |
| Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners" | |
| hyperparameter. Some cleaners are English-specific. You'll typically want to use: | |
| 1. "english_cleaners" for English text | |
| 2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using | |
| the Unidecode library (https://pypi.python.org/pypi/Unidecode) | |
| 3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update | |
| the symbols in symbols.py to match your data). | |
| ''' | |
| import re | |
| from .abbreviations import normalize_abbreviations | |
| from .acronyms import normalize_acronyms, spell_acronyms | |
| from .datestime import normalize_datestime | |
| from .letters_and_numbers import normalize_letters_and_numbers | |
| from .numerical import normalize_numbers | |
| from .unidecoder import unidecoder | |
| # Regular expression matching whitespace: | |
| _whitespace_re = re.compile(r'\s+') | |
| def expand_abbreviations(text): | |
| return normalize_abbreviations(text) | |
| def expand_numbers(text): | |
| return normalize_numbers(text) | |
| def expand_acronyms(text): | |
| return normalize_acronyms(text) | |
| def expand_datestime(text): | |
| return normalize_datestime(text) | |
| def expand_letters_and_numbers(text): | |
| return normalize_letters_and_numbers(text) | |
| def lowercase(text): | |
| return text.lower() | |
| def collapse_whitespace(text): | |
| return re.sub(_whitespace_re, ' ', text) | |
| def separate_acronyms(text): | |
| text = re.sub(r"([0-9]+)([a-zA-Z]+)", r"\1 \2", text) | |
| text = re.sub(r"([a-zA-Z]+)([0-9]+)", r"\1 \2", text) | |
| return text | |
| def convert_to_ascii(text): | |
| return unidecoder(text) | |
| def basic_cleaners(text): | |
| '''Basic pipeline that collapses whitespace without transliteration.''' | |
| # text = lowercase(text) | |
| text = collapse_whitespace(text) | |
| return text | |
| def transliteration_cleaners(text): | |
| '''Pipeline for non-English text that transliterates to ASCII.''' | |
| text = convert_to_ascii(text) | |
| text = lowercase(text) | |
| text = collapse_whitespace(text) | |
| return text | |
| def english_cleaners(text): | |
| '''Pipeline for English text, with number and abbreviation expansion.''' | |
| text = convert_to_ascii(text) | |
| text = lowercase(text) | |
| text = expand_numbers(text) | |
| text = expand_abbreviations(text) | |
| text = collapse_whitespace(text) | |
| return text | |
| def english_cleaners_v2(text): | |
| text = convert_to_ascii(text) | |
| text = expand_datestime(text) | |
| text = expand_letters_and_numbers(text) | |
| text = expand_numbers(text) | |
| text = expand_abbreviations(text) | |
| text = spell_acronyms(text) | |
| text = lowercase(text) | |
| text = collapse_whitespace(text) | |
| # compatibility with basic_english symbol set | |
| text = re.sub(r'/+', ' ', text) | |
| return text | |