tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Update japanese.py

Browse Source
This commit is contained in:
Stardust·减
2023-09-23 20:20:17 +08:00
committed by GitHub
parent e95292dfd6
commit bdaff5708e
1 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
text/japanese.py
View File

@@ -2,11 +2,12 @@
# compatible with Julius https://github.com/julius-speech/segmentation-kit # compatible with Julius https://github.com/julius-speech/segmentation-kit
import re import re
import unicodedata import unicodedata
import sys
from transformers import AutoTokenizer from transformers import AutoTokenizer
from text import punctuation, symbols from text import punctuation, symbols
from typing import TextIO
import pyopenjtalk import pyopenjtalk
from num2words import num2words from num2words import num2words
@@ -359,6 +360,8 @@ _NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
def japanese_convert_numbers_to_words(text: str) -> str: def japanese_convert_numbers_to_words(text: str) -> str:
res = text res = text
for x in _CURRENCY_MAP.keys():
res = res.replace(x, _CURRENCY_MAP[x])
return res return res
@@ -377,17 +380,11 @@ rep_map = {
def replace_punctuation(text): def replace_punctuation(text):
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys())) replaced_text = text
for x in rep_map.keys():
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) replaced_text = replaced_text.replace(x, rep_map[x])
replaced_text = re.sub(r'[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF' + "".join(punctuation) + r']+', '',
replaced_text)
#print('AFTER1:', replaced_text)
return replaced_text return replaced_text
def text_normalize(text): def text_normalize(text):
res = unicodedata.normalize("NFKC", text) res = unicodedata.normalize("NFKC", text)
res = japanese_convert_numbers_to_words(res) res = japanese_convert_numbers_to_words(res)
@@ -403,15 +400,18 @@ def g2p(norm_text):
tokenized = tokenizer.tokenize(norm_text) tokenized = tokenizer.tokenize(norm_text)
st = [x.replace("#", "") for x in tokenized] st = [x.replace("#", "") for x in tokenized]
word2ph = [] word2ph = []
phs = [] phs = pyopenjtalk.g2p(norm_text).split(" ") # Directly use the entire norm_text sequence.
for sub in st: for sub in st: # the following code is only for calculating word2ph
wph = 0 wph = 0
for x in sub: for x in sub:
phonemes = pyopenjtalk.g2p(x) sys.stdout.flush()
if x not in ['?', '.', '!', '…', ',']: # This will throw warnings.
phonemes = pyopenjtalk.g2p(x)
else:
phonemes = 'pau'
# for x in range(repeat): # for x in range(repeat):
wph += len(phonemes.split(' ')) wph += len(phonemes.split(' '))
# print(f'{x}-->:{phones}') # print(f'{x}-->:{phones}')
phs += phonemes.split(" ")
word2ph.append(wph) word2ph.append(wph)
phonemes = ['_'] + phs + ['_'] phonemes = ['_'] + phs + ['_']
tones = [0 for i in phonemes] tones = [0 for i in phonemes]