moved tokenizer loading inside of g2p function for code integrity.

This commit is contained in:
gordon0414
2024-05-10 11:55:38 +09:00
parent 3018309199
commit 7de367b76f

View File

@@ -17,7 +17,6 @@ ARPA = {
}
_g2p = G2p()
eng_dict = get_dict()
tokenizer = bert_models.load_tokenizer(Languages.EN)
def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
phones = []
@@ -115,6 +114,7 @@ def __distribute_phone(n_phone: int, n_word: int) -> list[int]:
def __text_to_words(text: str) -> list[list[str]]:
tokenizer = bert_models.load_tokenizer(Languages.EN)
tokens = tokenizer.tokenize(text)
words = []
for idx, t in enumerate(tokens):