Fmt only (maybe)

2024-05-25 18:15:36 +09:00
parent acf93b80ce
commit 2274087da4
33 changed files with 216 additions and 166 deletions
--- a/style_bert_vits2/nlp/chinese/g2p.py
+++ b/style_bert_vits2/nlp/chinese/g2p.py
@@ -8,7 +8,7 @@ from style_bert_vits2.nlp.chinese.tone_sandhi import ToneSandhi
 from style_bert_vits2.nlp.symbols import PUNCTUATIONS


-with open(Path(__file__).parent / "opencpop-strict.txt", "r", encoding="utf-8") as f:
+with open(Path(__file__).parent / "opencpop-strict.txt", encoding="utf-8") as f:
    __PINYIN_TO_SYMBOL_MAP = {
        line.split("\t")[0]: line.strip().split("\t")[1] for line in f.readlines()
    }
@@ -73,7 +73,7 @@ def __g2p(segments: list[str]) -> tuple[list[str], list[int], list[int]]:
                        "iou": "iu",
                        "uen": "un",
                    }
-                    if v_without_tone in v_rep_map.keys():
+                    if v_without_tone in v_rep_map:
                        pinyin = c + v_rep_map[v_without_tone]
                else:
                    # 单音节
@@ -83,7 +83,7 @@ def __g2p(segments: list[str]) -> tuple[list[str], list[int], list[int]]:
                        "in": "yin",
                        "u": "wu",
                    }
-                    if pinyin in pinyin_rep_map.keys():
+                    if pinyin in pinyin_rep_map:
                        pinyin = pinyin_rep_map[pinyin]
                    else:
                        single_rep_map = {
@@ -92,10 +92,10 @@ def __g2p(segments: list[str]) -> tuple[list[str], list[int], list[int]]:
                            "i": "y",
                            "u": "w",
                        }
-                        if pinyin[0] in single_rep_map.keys():
+                        if pinyin[0] in single_rep_map:
                            pinyin = single_rep_map[pinyin[0]] + pinyin[1:]

-                assert pinyin in __PINYIN_TO_SYMBOL_MAP.keys(), (
+                assert pinyin in __PINYIN_TO_SYMBOL_MAP, (
                    pinyin,
                    seg,
                    raw_pinyin,
--- a/style_bert_vits2/nlp/chinese/normalizer.py
+++ b/style_bert_vits2/nlp/chinese/normalizer.py
@@ -51,7 +51,7 @@ def normalize_text(text: str) -> str:
 def replace_punctuation(text: str) -> str:

    text = text.replace("嗯", "恩").replace("呣", "母")
-    pattern = re.compile("|".join(re.escape(p) for p in __REPLACE_MAP.keys()))
+    pattern = re.compile("|".join(re.escape(p) for p in __REPLACE_MAP))

    replaced_text = pattern.sub(lambda x: __REPLACE_MAP[x.group()], text)

--- a/style_bert_vits2/nlp/chinese/tone_sandhi.py
+++ b/style_bert_vits2/nlp/chinese/tone_sandhi.py
@@ -471,26 +471,27 @@ class ToneSandhi:
            ):
                finals[j] = finals[j][:-1] + "5"
        ge_idx = word.find("个")
-        if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
-            finals[-1] = finals[-1][:-1] + "5"
-        elif len(word) >= 1 and word[-1] in "的地得":
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 走了, 看着, 去过
-        # elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
-        #     finals[-1] = finals[-1][:-1] + "5"
-        elif (
-            len(word) > 1
-            and word[-1] in "们子"
-            and pos in {"r", "n"}
-            and word not in self.must_not_neural_tone_words
+        if (
+            len(word) >= 1
+            and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶"
+            or len(word) >= 1
+            and word[-1] in "的地得"
+            or (
+                (
+                    len(word) > 1
+                    and word[-1] in "们子"
+                    and pos in {"r", "n"}
+                    and word not in self.must_not_neural_tone_words
+                )
+                or len(word) > 1
+                and word[-1] in "上下里"
+                and pos in {"s", "l", "f"}
+            )
+            or len(word) > 1
+            and word[-1] in "来去"
+            and word[-2] in "上下进出回过起开"
        ):
            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 桌上, 地下, 家里
-        elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 上来, 下去
-        elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
-            finals[-1] = finals[-1][:-1] + "5"
        # 个做量词
        elif (
            ge_idx >= 1
@@ -500,12 +501,11 @@ class ToneSandhi:
            )
        ) or word == "个":
            finals[ge_idx] = finals[ge_idx][:-1] + "5"
-        else:
-            if (
-                word in self.must_neural_tone_words
-                or word[-2:] in self.must_neural_tone_words
-            ):
-                finals[-1] = finals[-1][:-1] + "5"
+        elif (
+            word in self.must_neural_tone_words
+            or word[-2:] in self.must_neural_tone_words
+        ):
+            finals[-1] = finals[-1][:-1] + "5"

        word_list = self._split_word(word)
        finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
@@ -549,10 +549,8 @@ class ToneSandhi:
                    if finals[i + 1][-1] == "4":
                        finals[i] = finals[i][:-1] + "2"
                    # "一" before non-tone4 should be yi4, e.g. 一天
-                    else:
-                        # "一" 后面如果是标点，还读一声
-                        if word[i + 1] not in self.punc:
-                            finals[i] = finals[i][:-1] + "4"
+                    elif word[i + 1] not in self.punc:
+                        finals[i] = finals[i][:-1] + "4"
        return finals

    def _split_word(self, word: str) -> list[str]:
--- a/style_bert_vits2/nlp/english/cmudict.py
+++ b/style_bert_vits2/nlp/english/cmudict.py
@@ -20,7 +20,7 @@ def get_dict() -> dict[str, list[list[str]]]:
 def read_dict() -> dict[str, list[list[str]]]:
    g2p_dict = {}
    start_line = 49
-    with open(CMU_DICT_PATH, "r", encoding="utf-8") as f:
+    with open(CMU_DICT_PATH, encoding="utf-8") as f:
        line = f.readline()
        line_index = 1
        while line:
--- a/style_bert_vits2/nlp/english/g2p.py
+++ b/style_bert_vits2/nlp/english/g2p.py
@@ -1,23 +1,91 @@
 import re
+
 from g2p_en import G2p
+
 from style_bert_vits2.constants import Languages
 from style_bert_vits2.nlp import bert_models
 from style_bert_vits2.nlp.english.cmudict import get_dict
 from style_bert_vits2.nlp.symbols import PUNCTUATIONS, SYMBOLS

+
 # Initialize global variables once
 ARPA = {
-    "AH0", "S", "AH1", "EY2", "AE2", "EH0", "OW2", "UH0", "NG", "B", "G", "AY0",
-    "M", "AA0", "F", "AO0", "ER2", "UH1", "IY1", "AH2", "DH", "IY0", "EY1",
-    "IH0", "K", "N", "W", "IY2", "T", "AA1", "ER1", "EH2", "OY0", "UH2", "UW1",
-    "Z", "AW2", "AW1", "V", "UW2", "AA2", "ER", "AW0", "UW0", "R", "OW1", "EH1",
-    "ZH", "AE0", "IH2", "IH", "Y", "JH", "P", "AY1", "EY0", "OY2", "TH", "HH",
-    "D", "ER0", "CH", "AO1", "AE1", "AO2", "OY1", "AY2", "IH1", "OW0", "L",
-    "SH"
+    "AH0",
+    "S",
+    "AH1",
+    "EY2",
+    "AE2",
+    "EH0",
+    "OW2",
+    "UH0",
+    "NG",
+    "B",
+    "G",
+    "AY0",
+    "M",
+    "AA0",
+    "F",
+    "AO0",
+    "ER2",
+    "UH1",
+    "IY1",
+    "AH2",
+    "DH",
+    "IY0",
+    "EY1",
+    "IH0",
+    "K",
+    "N",
+    "W",
+    "IY2",
+    "T",
+    "AA1",
+    "ER1",
+    "EH2",
+    "OY0",
+    "UH2",
+    "UW1",
+    "Z",
+    "AW2",
+    "AW1",
+    "V",
+    "UW2",
+    "AA2",
+    "ER",
+    "AW0",
+    "UW0",
+    "R",
+    "OW1",
+    "EH1",
+    "ZH",
+    "AE0",
+    "IH2",
+    "IH",
+    "Y",
+    "JH",
+    "P",
+    "AY1",
+    "EY0",
+    "OY2",
+    "TH",
+    "HH",
+    "D",
+    "ER0",
+    "CH",
+    "AO1",
+    "AE1",
+    "AO2",
+    "OY1",
+    "AY2",
+    "IH1",
+    "OW0",
+    "L",
+    "SH",
 }
 _g2p = G2p()
 eng_dict = get_dict()

+
 def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
    phones = []
    tones = []
@@ -51,7 +119,7 @@ def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
                        tns.append(0)
                temp_phones += [__post_replace_ph(i) for i in phns]
                temp_tones += tns
-                
+
        phones += temp_phones
        tones += temp_tones
        phone_len.append(len(temp_phones))
@@ -72,9 +140,19 @@ def g2p(text: str) -> tuple[list[str], list[int], list[int]]:

 def __post_replace_ph(ph: str) -> str:
    REPLACE_MAP = {
-        "：": ",", "；": ",", "，": ",", "。": ".", "！": "!", "？": "?",
-        "\n": ".", "·": ",", "、": ",", "…": "...", "···": "...",
-        "・・・": "...", "v": "V"
+        "：": ",",
+        "；": ",",
+        "，": ",",
+        "。": ".",
+        "！": "!",
+        "？": "?",
+        "\n": ".",
+        "·": ",",
+        "、": ",",
+        "…": "...",
+        "···": "...",
+        "・・・": "...",
+        "v": "V",
    }
    if ph in REPLACE_MAP:
        ph = REPLACE_MAP[ph]
@@ -120,21 +198,22 @@ def __text_to_words(text: str) -> list[list[str]]:
    for idx, t in enumerate(tokens):
        if t.startswith("▁"):
            words.append([t[1:]])
-        else:
-            if t in PUNCTUATIONS:
-                if idx == len(tokens) - 1:
-                    words.append([f"{t}"])
-                else:
-                    if not tokens[idx + 1].startswith("▁") and tokens[idx + 1] not in PUNCTUATIONS:
-                        if idx == 0:
-                            words.append([])
-                        words[-1].append(f"{t}")
-                    else:
-                        words.append([f"{t}"])
-            else:
+        elif t in PUNCTUATIONS:
+            if idx == len(tokens) - 1:
+                words.append([f"{t}"])
+            elif (
+                not tokens[idx + 1].startswith("▁")
+                and tokens[idx + 1] not in PUNCTUATIONS
+            ):
                if idx == 0:
                    words.append([])
                words[-1].append(f"{t}")
+            else:
+                words.append([f"{t}"])
+        else:
+            if idx == 0:
+                words.append([])
+            words[-1].append(f"{t}")
    return words


@@ -149,4 +228,3 @@ if __name__ == "__main__":
    #         for ph in group:
    #             all_phones.add(ph)
    # print(all_phones)
-
--- a/style_bert_vits2/nlp/english/normalizer.py
+++ b/style_bert_vits2/nlp/english/normalizer.py
@@ -58,7 +58,7 @@ def replace_punctuation(text: str) -> str:
        "「": "'",
        "」": "'",
    }
-    pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
+    pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP))
    replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
    # replaced_text = re.sub(
    #     r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
--- a/style_bert_vits2/nlp/japanese/g2p.py
+++ b/style_bert_vits2/nlp/japanese/g2p.py
@@ -719,5 +719,3 @@ class YomiError(Exception):
    基本的に「学習の前処理のテキスト処理時」には発生させ、そうでない場合は、
    ignore_yomi_error=True にしておいて、この例外を発生させないようにする。
    """
-
-    pass
--- a/style_bert_vits2/nlp/japanese/normalizer.py
+++ b/style_bert_vits2/nlp/japanese/normalizer.py
@@ -60,7 +60,7 @@ __REPLACE_MAP = {
    "」": "'",
 }
 # 記号類の正規化パターン
-__REPLACE_PATTERN = re.compile("|".join(re.escape(p) for p in __REPLACE_MAP.keys()))
+__REPLACE_PATTERN = re.compile("|".join(re.escape(p) for p in __REPLACE_MAP))
 # 句読点等の正規化パターン
 __PUNCTUATION_CLEANUP_PATTERN = re.compile(
    # ↓ ひらがな、カタカナ、漢字
--- a/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/init.py
+++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/init.py
@@ -88,7 +88,7 @@ def initialize_worker(port: int = WORKER_PORT) -> None:
    client = None
    try:
        client = WorkerClient(port)
-    except (socket.timeout, socket.error):
+    except (OSError, socket.timeout):
        logger.debug("try starting pyopenjtalk worker server")
        import os
        import subprocess
@@ -120,7 +120,7 @@ def initialize_worker(port: int = WORKER_PORT) -> None:
            try:
                client = WorkerClient(port)
                break
-            except socket.error:
+            except OSError:
                time.sleep(0.5)
                count += 1
                # 20: max number of retries
--- a/style_bert_vits2/nlp/japanese/user_dict/word_model.py
+++ b/style_bert_vits2/nlp/japanese/user_dict/word_model.py
@@ -114,7 +114,7 @@ class PartOfSpeechDetail(BaseModel):
    part_of_speech_detail_2: str = Field(title="品詞細分類2")
    part_of_speech_detail_3: str = Field(title="品詞細分類3")
    # context_idは辞書の左・右文脈IDのこと
-    # https://github.com/VOICEVOX/open_jtalk/blob/427cfd761b78efb6094bea3c5bb8c968f0d711ab/src/mecab-naist-jdic/_left-id.def # noqa
+    # https://github.com/VOICEVOX/open_jtalk/blob/427cfd761b78efb6094bea3c5bb8c968f0d711ab/src/mecab-naist-jdic/_left-id.def
    context_id: int = Field(title="文脈ID")
    cost_candidates: List[int] = Field(title="コストのパーセンタイル")
    accent_associative_rules: List[str] = Field(title="アクセント結合規則の一覧")