Refactor: rename text_processing to nlp

"text_processing" is clearer, but the import statement is longer.
"nlp" is shorter and makes it clear that it is natural language processing.

style_bert_vits2/nlp/__init__.py

@@ -0,0 +1,107 @@
+import torch
+from typing import Optional
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp.symbols import (
+    LANGUAGE_ID_MAP,
+    LANGUAGE_TONE_START_MAP,
+    SYMBOLS,
+)
+
+
+__symbol_to_id = {s: i for i, s in enumerate(SYMBOLS)}
+
+
+def extract_bert_feature(
+    text: str,
+    word2ph: list[int],
+    language: Languages,
+    device: torch.device | str,
+    assist_text: Optional[str] = None,
+    assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+    """
+    テキストから BERT の特徴量を抽出する
+
+    Args:
+        text (str): テキスト
+        word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+        language (Languages): テキストの言語
+        device (torch.device | str): 推論に利用するデバイス
+        assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+        assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+    Returns:
+        torch.Tensor: BERT の特徴量
+    """
+
+    if language == Languages.JP:
+        from style_bert_vits2.nlp.japanese.bert_feature import extract_bert_feature
+    elif language == Languages.EN:
+        from style_bert_vits2.nlp.english.bert_feature import extract_bert_feature
+    elif language == Languages.ZH:
+        from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature
+    else:
+        raise ValueError(f"Language {language} not supported")
+
+    return extract_bert_feature(text, word2ph, device, assist_text, assist_text_weight)
+
+
+def clean_text(
+    text: str,
+    language: Languages,
+    use_jp_extra: bool = True,
+    raise_yomi_error: bool = False,
+) -> tuple[str, list[str], list[int], list[int]]:
+    """
+    テキストをクリーニングし、音素に変換する
+
+    Args:
+        text (str): クリーニングするテキスト
+        language (Languages): テキストの言語
+        use_jp_extra (bool, optional): テキストが日本語の場合に JP-Extra モデルを利用するかどうか。Defaults to True.
+        raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
+
+    Returns:
+        tuple[str, list[str], list[int], list[int]]: クリーニングされたテキストと、音素・アクセント・元のテキストの各文字に音素が何個割り当てられるかのリスト
+    """
+
+    # Changed to import inside if condition to avoid unnecessary import
+    if language == Languages.JP:
+        from style_bert_vits2.nlp.japanese import g2p, normalize_text
+        norm_text = normalize_text(text)
+        phones, tones, word2ph = g2p(norm_text, use_jp_extra, raise_yomi_error)
+    elif language == Languages.EN:
+        from style_bert_vits2.nlp.english import g2p, normalize_text
+        norm_text = normalize_text(text)
+        phones, tones, word2ph = g2p(norm_text)
+    elif language == Languages.ZH:
+        from style_bert_vits2.nlp.chinese import g2p, normalize_text
+        norm_text = normalize_text(text)
+        phones, tones, word2ph = g2p(norm_text)
+    else:
+        raise ValueError(f"Language {language} not supported")
+
+    return norm_text, phones, tones, word2ph
+
+
+def cleaned_text_to_sequence(cleaned_phones: list[str], tones: list[int], language: Languages) -> tuple[list[int], list[int], list[int]]:
+    """
+    テキスト文字列を、テキスト内の記号に対応する一連の ID に変換する
+
+    Args:
+        cleaned_phones (list[str]): clean_text() でクリーニングされた音素のリスト (?)
+        tones (list[int]): 各音素のアクセント
+        language (Languages): テキストの言語
+
+    Returns:
+        tuple[list[int], list[int], list[int]]: List of integers corresponding to the symbols in the text
+    """
+
+    phones = [__symbol_to_id[symbol] for symbol in cleaned_phones]
+    tone_start = LANGUAGE_TONE_START_MAP[language]
+    tones = [i + tone_start for i in tones]
+    lang_id = LANGUAGE_ID_MAP[language]
+    lang_ids = [lang_id for i in phones]
+
+    return phones, tones, lang_ids

style_bert_vits2/nlp/bert_models.py

@@ -0,0 +1,177 @@
+"""
+Style-Bert-VITS2 の学習・推論に必要な各言語ごとの BERT モデルをロード/取得するためのモジュール。
+
+オリジナルの Bert-VITS2 では各言語ごとの BERT モデルが初回インポート時にハードコードされたパスから「暗黙的に」ロードされているが、
+場合によっては多重にロードされて非効率なほか、BERT モデルのロード元のパスがハードコードされているためライブラリ化ができない。
+
+そこで、ライブラリの利用前に、音声合成に利用する言語の BERT モデルだけを「明示的に」ロードできるようにした。
+一度 load_model/tokenizer() で当該言語の BERT モデルがロードされていれば、ライブラリ内部のどこからでもロード済みのモデル/トークナイザーを取得できる。
+"""
+
+import gc
+from typing import cast, Optional
+
+import torch
+from transformers import (
+    AutoModelForMaskedLM,
+    AutoTokenizer,
+    DebertaV2Model,
+    DebertaV2Tokenizer,
+    PreTrainedModel,
+    PreTrainedTokenizer,
+    PreTrainedTokenizerFast,
+)
+
+from style_bert_vits2.constants import DEFAULT_BERT_TOKENIZER_PATHS, Languages
+from style_bert_vits2.logging import logger
+
+
+# 各言語ごとのロード済みの BERT モデルを格納する辞書
+__loaded_models: dict[Languages, PreTrainedModel | DebertaV2Model] = {}
+
+# 各言語ごとのロード済みの BERT トークナイザーを格納する辞書
+__loaded_tokenizers: dict[Languages, PreTrainedTokenizer | PreTrainedTokenizerFast | DebertaV2Tokenizer] = {}
+
+
+def load_model(
+    language: Languages,
+    pretrained_model_name_or_path: Optional[str] = None,
+) -> PreTrainedModel | DebertaV2Model:
+    """
+    指定された言語の BERT モデルをロードし、ロード済みの BERT モデルを返す。
+    一度ロードされていれば、ロード済みの BERT モデルを即座に返す。
+    ライブラリ利用時は常に pretrain_model_name_or_path (Hugging Face のリポジトリ名 or ローカルのファイルパス) を指定する必要がある。
+    ロードにはそれなりに時間がかかるため、ライブラリ利用前に明示的に pretrained_model_name_or_path を指定してロードしておくべき。
+
+    Style-Bert-VITS2 では、BERT モデルに下記の 3 つが利用されている。
+    これ以外の BERT モデルを指定した場合は正常に動作しない可能性が高い。
+    - 日本語: ku-nlp/deberta-v2-large-japanese-char-wwm
+    - 英語: microsoft/deberta-v3-large
+    - 中国語: hfl/chinese-roberta-wwm-ext-large
+
+    Args:
+        language (Languages): ロードする学習済みモデルの対象言語
+        pretrained_model_name_or_path (Optional[str]): ロードする学習済みモデルの名前またはパス。指定しない場合はデフォルトのパスが利用される (デフォルト: None)
+
+    Returns:
+        PreTrainedModel | DebertaV2Model: ロード済みの BERT モデル
+    """
+
+    # すでにロード済みの場合はそのまま返す
+    if language in __loaded_models:
+        return __loaded_models[language]
+
+    # pretrained_model_name_or_path が指定されていない場合はデフォルトのパスを利用
+    if pretrained_model_name_or_path is None:
+        assert DEFAULT_BERT_TOKENIZER_PATHS[language].exists(), \
+            f"The default {language} BERT model does not exist on the file system. Please specify the path to the pre-trained model."
+        pretrained_model_name_or_path = str(DEFAULT_BERT_TOKENIZER_PATHS[language])
+
+    # BERT モデルをロードし、辞書に格納して返す
+    ## 英語のみ DebertaV2Model でロードする必要がある
+    if language == Languages.EN:
+        model = cast(DebertaV2Model, DebertaV2Model.from_pretrained(pretrained_model_name_or_path))
+    else:
+        model = AutoModelForMaskedLM.from_pretrained(pretrained_model_name_or_path)
+    __loaded_models[language] = model
+    logger.info(f"Loaded the {language} BERT model from {pretrained_model_name_or_path}")
+
+    return model
+
+
+def load_tokenizer(
+    language: Languages,
+    pretrained_model_name_or_path: Optional[str] = None,
+) -> PreTrainedTokenizer | PreTrainedTokenizerFast | DebertaV2Tokenizer:
+    """
+    指定された言語の BERT モデルをロードし、ロード済みの BERT トークナイザーを返す。
+    一度ロードされていれば、ロード済みの BERT トークナイザーを即座に返す。
+    ライブラリ利用時は常に pretrain_model_name_or_path (Hugging Face のリポジトリ名 or ローカルのファイルパス) を指定する必要がある。
+    ロードにはそれなりに時間がかかるため、ライブラリ利用前に明示的に pretrained_model_name_or_path を指定してロードしておくべき。
+
+    Style-Bert-VITS2 では、BERT モデルに下記の 3 つが利用されている。
+    これ以外の BERT モデルを指定した場合は正常に動作しない可能性が高い。
+    - 日本語: ku-nlp/deberta-v2-large-japanese-char-wwm
+    - 英語: microsoft/deberta-v3-large
+    - 中国語: hfl/chinese-roberta-wwm-ext-large
+
+    Args:
+        language (Languages): ロードする学習済みモデルの対象言語
+        pretrained_model_name_or_path (Optional[str]): ロードする学習済みモデルの名前またはパス。指定しない場合はデフォルトのパスが利用される (デフォルト: None)
+
+    Returns:
+        PreTrainedTokenizer | PreTrainedTokenizerFast | DebertaV2Tokenizer: ロード済みの BERT トークナイザー
+    """
+
+    # すでにロード済みの場合はそのまま返す
+    if language in __loaded_tokenizers:
+        return __loaded_tokenizers[language]
+
+    # pretrained_model_name_or_path が指定されていない場合はデフォルトのパスを利用
+    if pretrained_model_name_or_path is None:
+        assert DEFAULT_BERT_TOKENIZER_PATHS[language].exists(), \
+            f"The default {language} BERT tokenizer does not exist on the file system. Please specify the path to the pre-trained model."
+        pretrained_model_name_or_path = str(DEFAULT_BERT_TOKENIZER_PATHS[language])
+
+    # BERT トークナイザーをロードし、辞書に格納して返す
+    ## 英語のみ DebertaV2Tokenizer でロードする必要がある
+    if language == Languages.EN:
+        tokenizer = DebertaV2Tokenizer.from_pretrained(pretrained_model_name_or_path)
+    else:
+        tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path)
+    __loaded_tokenizers[language] = tokenizer
+    logger.info(f"Loaded the {language} BERT tokenizer from {pretrained_model_name_or_path}")
+
+    return tokenizer
+
+
+def unload_model(language: Languages) -> None:
+    """
+    指定された言語の BERT モデルをアンロードする。
+
+    Args:
+        language (Languages): アンロードする BERT モデルの言語
+    """
+
+    if language in __loaded_models:
+        del __loaded_models[language]
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        logger.info(f"Unloaded the {language} BERT model")
+
+
+def unload_tokenizer(language: Languages) -> None:
+    """
+    指定された言語の BERT トークナイザーをアンロードする。
+
+    Args:
+        language (Languages): アンロードする BERT トークナイザーの言語
+    """
+
+    if language in __loaded_tokenizers:
+        del __loaded_tokenizers[language]
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        logger.info(f"Unloaded the {language} BERT tokenizer")
+
+
+def unload_all_models() -> None:
+    """
+    すべての BERT モデルをアンロードする。
+    """
+
+    for language in list(__loaded_models.keys()):
+        unload_model(language)
+    logger.info("Unloaded all BERT models")
+
+
+def unload_all_tokenizers() -> None:
+    """
+    すべての BERT トークナイザーをアンロードする。
+    """
+
+    for language in list(__loaded_tokenizers.keys()):
+        unload_tokenizer(language)
+    logger.info("Unloaded all BERT tokenizers")

style_bert_vits2/nlp/chinese/__init__.py

@@ -0,0 +1,193 @@
+import os
+import re
+
+import cn2an
+import jieba.posseg as psg
+from pypinyin import lazy_pinyin, Style
+
+from style_bert_vits2.nlp.chinese.tone_sandhi import ToneSandhi
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+current_file_path = os.path.dirname(__file__)
+pinyin_to_symbol_map = {
+    line.split("\t")[0]: line.strip().split("\t")[1]
+    for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
+}
+
+
+rep_map = {
+    "：": ",",
+    "；": ",",
+    "，": ",",
+    "。": ".",
+    "！": "!",
+    "？": "?",
+    "\n": ".",
+    "·": ",",
+    "、": ",",
+    "...": "…",
+    "$": ".",
+    "“": "'",
+    "”": "'",
+    '"': "'",
+    "‘": "'",
+    "’": "'",
+    "（": "'",
+    "）": "'",
+    "(": "'",
+    ")": "'",
+    "《": "'",
+    "》": "'",
+    "【": "'",
+    "】": "'",
+    "[": "'",
+    "]": "'",
+    "—": "-",
+    "～": "-",
+    "~": "-",
+    "「": "'",
+    "」": "'",
+}
+
+tone_modifier = ToneSandhi()
+
+
+def replace_punctuation(text):
+    text = text.replace("嗯", "恩").replace("呣", "母")
+    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
+
+    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
+
+    replaced_text = re.sub(
+        r"[^\u4e00-\u9fa5" + "".join(PUNCTUATIONS) + r"]+", "", replaced_text
+    )
+
+    return replaced_text
+
+
+def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
+    pattern = r"(?<=[{0}])\s*".format("".join(PUNCTUATIONS))
+    sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
+    phones, tones, word2ph = _g2p(sentences)
+    assert sum(word2ph) == len(phones)
+    assert len(word2ph) == len(text)  # Sometimes it will crash,you can add a try-catch.
+    phones = ["_"] + phones + ["_"]
+    tones = [0] + tones + [0]
+    word2ph = [1] + word2ph + [1]
+    return phones, tones, word2ph
+
+
+def _get_initials_finals(word):
+    initials = []
+    finals = []
+    orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
+    orig_finals = lazy_pinyin(
+        word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
+    )
+    for c, v in zip(orig_initials, orig_finals):
+        initials.append(c)
+        finals.append(v)
+    return initials, finals
+
+
+def _g2p(segments):
+    phones_list = []
+    tones_list = []
+    word2ph = []
+    for seg in segments:
+        # Replace all English words in the sentence
+        seg = re.sub("[a-zA-Z]+", "", seg)
+        seg_cut = psg.lcut(seg)
+        initials = []
+        finals = []
+        seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
+        for word, pos in seg_cut:
+            if pos == "eng":
+                continue
+            sub_initials, sub_finals = _get_initials_finals(word)
+            sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
+            initials.append(sub_initials)
+            finals.append(sub_finals)
+
+            # assert len(sub_initials) == len(sub_finals) == len(word)
+        initials = sum(initials, [])
+        finals = sum(finals, [])
+        #
+        for c, v in zip(initials, finals):
+            raw_pinyin = c + v
+            # NOTE: post process for pypinyin outputs
+            # we discriminate i, ii and iii
+            if c == v:
+                assert c in PUNCTUATIONS
+                phone = [c]
+                tone = "0"
+                word2ph.append(1)
+            else:
+                v_without_tone = v[:-1]
+                tone = v[-1]
+
+                pinyin = c + v_without_tone
+                assert tone in "12345"
+
+                if c:
+                    # 多音节
+                    v_rep_map = {
+                        "uei": "ui",
+                        "iou": "iu",
+                        "uen": "un",
+                    }
+                    if v_without_tone in v_rep_map.keys():
+                        pinyin = c + v_rep_map[v_without_tone]
+                else:
+                    # 单音节
+                    pinyin_rep_map = {
+                        "ing": "ying",
+                        "i": "yi",
+                        "in": "yin",
+                        "u": "wu",
+                    }
+                    if pinyin in pinyin_rep_map.keys():
+                        pinyin = pinyin_rep_map[pinyin]
+                    else:
+                        single_rep_map = {
+                            "v": "yu",
+                            "e": "e",
+                            "i": "y",
+                            "u": "w",
+                        }
+                        if pinyin[0] in single_rep_map.keys():
+                            pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
+
+                assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
+                phone = pinyin_to_symbol_map[pinyin].split(" ")
+                word2ph.append(len(phone))
+
+            phones_list += phone
+            tones_list += [int(tone)] * len(phone)
+    return phones_list, tones_list, word2ph
+
+
+def normalize_text(text: str) -> str:
+    numbers = re.findall(r"\d+(?:\.?\d+)?", text)
+    for number in numbers:
+        text = text.replace(number, cn2an.an2cn(number), 1)
+    text = replace_punctuation(text)
+    return text
+
+
+if __name__ == "__main__":
+    from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature
+
+    text = "啊！但是《原神》是由,米哈\游自主，  [研发]的一款全.新开放世界.冒险游戏"
+    text = normalize_text(text)
+    print(text)
+    phones, tones, word2ph = g2p(text)
+    bert = extract_bert_feature(text, word2ph, 'cuda')
+
+    print(phones, tones, word2ph, bert.shape)
+
+
+# # 示例用法
+# text = "这是一个示例文本：,你好！这是一个测试...."
+# print(g2p_paddle(text))  # 输出: 这是一个示例文本你好这是一个测试

style_bert_vits2/nlp/chinese/bert_feature.py

@@ -0,0 +1,139 @@
+import sys
+from typing import Optional
+
+import torch
+from transformers import PreTrainedModel
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+
+
+__models: dict[torch.device | str, PreTrainedModel] = {}
+
+
+def extract_bert_feature(
+    text: str,
+    word2ph: list[int],
+    device: torch.device | str,
+    assist_text: Optional[str] = None,
+    assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+    """
+    中国語のテキストから BERT の特徴量を抽出する
+
+    Args:
+        text (str): 中国語のテキスト
+        word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+        device (torch.device | str): 推論に利用するデバイス
+        assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+        assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+    Returns:
+        torch.Tensor: BERT の特徴量
+    """
+
+    if (
+        sys.platform == "darwin"
+        and torch.backends.mps.is_available()
+        and device == "cpu"
+    ):
+        device = "mps"
+    if not device:
+        device = "cuda"
+    if device == "cuda" and not torch.cuda.is_available():
+        device = "cpu"
+    if device not in __models.keys():
+        __models[device] = bert_models.load_model(Languages.ZH).to(device)  # type: ignore
+
+    style_res_mean = None
+    with torch.no_grad():
+        tokenizer = bert_models.load_tokenizer(Languages.ZH)
+        inputs = tokenizer(text, return_tensors="pt")
+        for i in inputs:
+            inputs[i] = inputs[i].to(device)  # type: ignore
+        res = __models[device](**inputs, output_hidden_states=True)
+        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if assist_text:
+            style_inputs = tokenizer(assist_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)  # type: ignore
+            style_res = __models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
+
+    assert len(word2ph) == len(text) + 2
+    word2phone = word2ph
+    phone_level_feature = []
+    for i in range(len(word2phone)):
+        if assist_text:
+            assert style_res_mean is not None
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
+        phone_level_feature.append(repeat_feature)
+
+    phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+    return phone_level_feature.T
+
+
+if __name__ == "__main__":
+    word_level_feature = torch.rand(38, 1024)  # 12个词,每个词1024维特征
+    word2phone = [
+        1,
+        2,
+        1,
+        2,
+        2,
+        1,
+        2,
+        2,
+        1,
+        2,
+        2,
+        1,
+        2,
+        2,
+        2,
+        2,
+        2,
+        1,
+        1,
+        2,
+        2,
+        1,
+        2,
+        2,
+        2,
+        2,
+        1,
+        2,
+        2,
+        2,
+        2,
+        2,
+        1,
+        2,
+        2,
+        2,
+        2,
+        1,
+    ]
+
+    # 计算总帧数
+    total_frames = sum(word2phone)
+    print(word_level_feature.shape)
+    print(word2phone)
+    phone_level_feature = []
+    for i in range(len(word2phone)):
+        print(word_level_feature[i].shape)
+
+        # 对每个词重复word2phone[i]次
+        repeat_feature = word_level_feature[i].repeat(word2phone[i], 1)
+        phone_level_feature.append(repeat_feature)
+
+    phone_level_feature = torch.cat(phone_level_feature, dim=0)
+    print(phone_level_feature.shape)  # torch.Size([36, 1024])

style_bert_vits2/nlp/chinese/tone_sandhi.py

@@ -0,0 +1,774 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jieba
+from pypinyin import lazy_pinyin
+from pypinyin import Style
+
+
+class ToneSandhi:
+    def __init__(self):
+        self.must_neural_tone_words = {
+            "麻烦",
+            "麻利",
+            "鸳鸯",
+            "高粱",
+            "骨头",
+            "骆驼",
+            "马虎",
+            "首饰",
+            "馒头",
+            "馄饨",
+            "风筝",
+            "难为",
+            "队伍",
+            "阔气",
+            "闺女",
+            "门道",
+            "锄头",
+            "铺盖",
+            "铃铛",
+            "铁匠",
+            "钥匙",
+            "里脊",
+            "里头",
+            "部分",
+            "那么",
+            "道士",
+            "造化",
+            "迷糊",
+            "连累",
+            "这么",
+            "这个",
+            "运气",
+            "过去",
+            "软和",
+            "转悠",
+            "踏实",
+            "跳蚤",
+            "跟头",
+            "趔趄",
+            "财主",
+            "豆腐",
+            "讲究",
+            "记性",
+            "记号",
+            "认识",
+            "规矩",
+            "见识",
+            "裁缝",
+            "补丁",
+            "衣裳",
+            "衣服",
+            "衙门",
+            "街坊",
+            "行李",
+            "行当",
+            "蛤蟆",
+            "蘑菇",
+            "薄荷",
+            "葫芦",
+            "葡萄",
+            "萝卜",
+            "荸荠",
+            "苗条",
+            "苗头",
+            "苍蝇",
+            "芝麻",
+            "舒服",
+            "舒坦",
+            "舌头",
+            "自在",
+            "膏药",
+            "脾气",
+            "脑袋",
+            "脊梁",
+            "能耐",
+            "胳膊",
+            "胭脂",
+            "胡萝",
+            "胡琴",
+            "胡同",
+            "聪明",
+            "耽误",
+            "耽搁",
+            "耷拉",
+            "耳朵",
+            "老爷",
+            "老实",
+            "老婆",
+            "老头",
+            "老太",
+            "翻腾",
+            "罗嗦",
+            "罐头",
+            "编辑",
+            "结实",
+            "红火",
+            "累赘",
+            "糨糊",
+            "糊涂",
+            "精神",
+            "粮食",
+            "簸箕",
+            "篱笆",
+            "算计",
+            "算盘",
+            "答应",
+            "笤帚",
+            "笑语",
+            "笑话",
+            "窟窿",
+            "窝囊",
+            "窗户",
+            "稳当",
+            "稀罕",
+            "称呼",
+            "秧歌",
+            "秀气",
+            "秀才",
+            "福气",
+            "祖宗",
+            "砚台",
+            "码头",
+            "石榴",
+            "石头",
+            "石匠",
+            "知识",
+            "眼睛",
+            "眯缝",
+            "眨巴",
+            "眉毛",
+            "相声",
+            "盘算",
+            "白净",
+            "痢疾",
+            "痛快",
+            "疟疾",
+            "疙瘩",
+            "疏忽",
+            "畜生",
+            "生意",
+            "甘蔗",
+            "琵琶",
+            "琢磨",
+            "琉璃",
+            "玻璃",
+            "玫瑰",
+            "玄乎",
+            "狐狸",
+            "状元",
+            "特务",
+            "牲口",
+            "牙碜",
+            "牌楼",
+            "爽快",
+            "爱人",
+            "热闹",
+            "烧饼",
+            "烟筒",
+            "烂糊",
+            "点心",
+            "炊帚",
+            "灯笼",
+            "火候",
+            "漂亮",
+            "滑溜",
+            "溜达",
+            "温和",
+            "清楚",
+            "消息",
+            "浪头",
+            "活泼",
+            "比方",
+            "正经",
+            "欺负",
+            "模糊",
+            "槟榔",
+            "棺材",
+            "棒槌",
+            "棉花",
+            "核桃",
+            "栅栏",
+            "柴火",
+            "架势",
+            "枕头",
+            "枇杷",
+            "机灵",
+            "本事",
+            "木头",
+            "木匠",
+            "朋友",
+            "月饼",
+            "月亮",
+            "暖和",
+            "明白",
+            "时候",
+            "新鲜",
+            "故事",
+            "收拾",
+            "收成",
+            "提防",
+            "挖苦",
+            "挑剔",
+            "指甲",
+            "指头",
+            "拾掇",
+            "拳头",
+            "拨弄",
+            "招牌",
+            "招呼",
+            "抬举",
+            "护士",
+            "折腾",
+            "扫帚",
+            "打量",
+            "打算",
+            "打点",
+            "打扮",
+            "打听",
+            "打发",
+            "扎实",
+            "扁担",
+            "戒指",
+            "懒得",
+            "意识",
+            "意思",
+            "情形",
+            "悟性",
+            "怪物",
+            "思量",
+            "怎么",
+            "念头",
+            "念叨",
+            "快活",
+            "忙活",
+            "志气",
+            "心思",
+            "得罪",
+            "张罗",
+            "弟兄",
+            "开通",
+            "应酬",
+            "庄稼",
+            "干事",
+            "帮手",
+            "帐篷",
+            "希罕",
+            "师父",
+            "师傅",
+            "巴结",
+            "巴掌",
+            "差事",
+            "工夫",
+            "岁数",
+            "屁股",
+            "尾巴",
+            "少爷",
+            "小气",
+            "小伙",
+            "将就",
+            "对头",
+            "对付",
+            "寡妇",
+            "家伙",
+            "客气",
+            "实在",
+            "官司",
+            "学问",
+            "学生",
+            "字号",
+            "嫁妆",
+            "媳妇",
+            "媒人",
+            "婆家",
+            "娘家",
+            "委屈",
+            "姑娘",
+            "姐夫",
+            "妯娌",
+            "妥当",
+            "妖精",
+            "奴才",
+            "女婿",
+            "头发",
+            "太阳",
+            "大爷",
+            "大方",
+            "大意",
+            "大夫",
+            "多少",
+            "多么",
+            "外甥",
+            "壮实",
+            "地道",
+            "地方",
+            "在乎",
+            "困难",
+            "嘴巴",
+            "嘱咐",
+            "嘟囔",
+            "嘀咕",
+            "喜欢",
+            "喇嘛",
+            "喇叭",
+            "商量",
+            "唾沫",
+            "哑巴",
+            "哈欠",
+            "哆嗦",
+            "咳嗽",
+            "和尚",
+            "告诉",
+            "告示",
+            "含糊",
+            "吓唬",
+            "后头",
+            "名字",
+            "名堂",
+            "合同",
+            "吆喝",
+            "叫唤",
+            "口袋",
+            "厚道",
+            "厉害",
+            "千斤",
+            "包袱",
+            "包涵",
+            "匀称",
+            "勤快",
+            "动静",
+            "动弹",
+            "功夫",
+            "力气",
+            "前头",
+            "刺猬",
+            "刺激",
+            "别扭",
+            "利落",
+            "利索",
+            "利害",
+            "分析",
+            "出息",
+            "凑合",
+            "凉快",
+            "冷战",
+            "冤枉",
+            "冒失",
+            "养活",
+            "关系",
+            "先生",
+            "兄弟",
+            "便宜",
+            "使唤",
+            "佩服",
+            "作坊",
+            "体面",
+            "位置",
+            "似的",
+            "伙计",
+            "休息",
+            "什么",
+            "人家",
+            "亲戚",
+            "亲家",
+            "交情",
+            "云彩",
+            "事情",
+            "买卖",
+            "主意",
+            "丫头",
+            "丧气",
+            "两口",
+            "东西",
+            "东家",
+            "世故",
+            "不由",
+            "不在",
+            "下水",
+            "下巴",
+            "上头",
+            "上司",
+            "丈夫",
+            "丈人",
+            "一辈",
+            "那个",
+            "菩萨",
+            "父亲",
+            "母亲",
+            "咕噜",
+            "邋遢",
+            "费用",
+            "冤家",
+            "甜头",
+            "介绍",
+            "荒唐",
+            "大人",
+            "泥鳅",
+            "幸福",
+            "熟悉",
+            "计划",
+            "扑腾",
+            "蜡烛",
+            "姥爷",
+            "照顾",
+            "喉咙",
+            "吉他",
+            "弄堂",
+            "蚂蚱",
+            "凤凰",
+            "拖沓",
+            "寒碜",
+            "糟蹋",
+            "倒腾",
+            "报复",
+            "逻辑",
+            "盘缠",
+            "喽啰",
+            "牢骚",
+            "咖喱",
+            "扫把",
+            "惦记",
+        }
+        self.must_not_neural_tone_words = {
+            "男子",
+            "女子",
+            "分子",
+            "原子",
+            "量子",
+            "莲子",
+            "石子",
+            "瓜子",
+            "电子",
+            "人人",
+            "虎虎",
+        }
+        self.punc = "：，；。？！“”‘’':,;.?!"
+
+    # the meaning of jieba pos tag: https://blog.csdn.net/weixin_44174352/article/details/113731041
+    # e.g.
+    # word: "家里"
+    # pos: "s"
+    # finals: ['ia1', 'i3']
+    def _neural_sandhi(self, word: str, pos: str, finals: list[str]) -> list[str]:
+        # reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
+        for j, item in enumerate(word):
+            if (
+                j - 1 >= 0
+                and item == word[j - 1]
+                and pos[0] in {"n", "v", "a"}
+                and word not in self.must_not_neural_tone_words
+            ):
+                finals[j] = finals[j][:-1] + "5"
+        ge_idx = word.find("个")
+        if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
+            finals[-1] = finals[-1][:-1] + "5"
+        elif len(word) >= 1 and word[-1] in "的地得":
+            finals[-1] = finals[-1][:-1] + "5"
+        # e.g. 走了, 看着, 去过
+        # elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
+        #     finals[-1] = finals[-1][:-1] + "5"
+        elif (
+            len(word) > 1
+            and word[-1] in "们子"
+            and pos in {"r", "n"}
+            and word not in self.must_not_neural_tone_words
+        ):
+            finals[-1] = finals[-1][:-1] + "5"
+        # e.g. 桌上, 地下, 家里
+        elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
+            finals[-1] = finals[-1][:-1] + "5"
+        # e.g. 上来, 下去
+        elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
+            finals[-1] = finals[-1][:-1] + "5"
+        # 个做量词
+        elif (
+            ge_idx >= 1
+            and (
+                word[ge_idx - 1].isnumeric()
+                or word[ge_idx - 1] in "几有两半多各整每做是"
+            )
+        ) or word == "个":
+            finals[ge_idx] = finals[ge_idx][:-1] + "5"
+        else:
+            if (
+                word in self.must_neural_tone_words
+                or word[-2:] in self.must_neural_tone_words
+            ):
+                finals[-1] = finals[-1][:-1] + "5"
+
+        word_list = self._split_word(word)
+        finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
+        for i, word in enumerate(word_list):
+            # conventional neural in Chinese
+            if (
+                word in self.must_neural_tone_words
+                or word[-2:] in self.must_neural_tone_words
+            ):
+                finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
+        finals = sum(finals_list, [])
+        return finals
+
+    def _bu_sandhi(self, word: str, finals: list[str]) -> list[str]:
+        # e.g. 看不懂
+        if len(word) == 3 and word[1] == "不":
+            finals[1] = finals[1][:-1] + "5"
+        else:
+            for i, char in enumerate(word):
+                # "不" before tone4 should be bu2, e.g. 不怕
+                if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
+                    finals[i] = finals[i][:-1] + "2"
+        return finals
+
+    def _yi_sandhi(self, word: str, finals: list[str]) -> list[str]:
+        # "一" in number sequences, e.g. 一零零, 二一零
+        if word.find("一") != -1 and all(
+            [item.isnumeric() for item in word if item != "一"]
+        ):
+            return finals
+        # "一" between reduplication words should be yi5, e.g. 看一看
+        elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
+            finals[1] = finals[1][:-1] + "5"
+        # when "一" is ordinal word, it should be yi1
+        elif word.startswith("第一"):
+            finals[1] = finals[1][:-1] + "1"
+        else:
+            for i, char in enumerate(word):
+                if char == "一" and i + 1 < len(word):
+                    # "一" before tone4 should be yi2, e.g. 一段
+                    if finals[i + 1][-1] == "4":
+                        finals[i] = finals[i][:-1] + "2"
+                    # "一" before non-tone4 should be yi4, e.g. 一天
+                    else:
+                        # "一" 后面如果是标点，还读一声
+                        if word[i + 1] not in self.punc:
+                            finals[i] = finals[i][:-1] + "4"
+        return finals
+
+    def _split_word(self, word: str) -> list[str]:
+        word_list = jieba.cut_for_search(word)
+        word_list = sorted(word_list, key=lambda i: len(i), reverse=False)  # type: ignore
+        first_subword = word_list[0]
+        first_begin_idx = word.find(first_subword)
+        if first_begin_idx == 0:
+            second_subword = word[len(first_subword) :]
+            new_word_list = [first_subword, second_subword]
+        else:
+            second_subword = word[: -len(first_subword)]
+            new_word_list = [second_subword, first_subword]
+        return new_word_list
+
+    def _three_sandhi(self, word: str, finals: list[str]) -> list[str]:
+        if len(word) == 2 and self._all_tone_three(finals):
+            finals[0] = finals[0][:-1] + "2"
+        elif len(word) == 3:
+            word_list = self._split_word(word)
+            if self._all_tone_three(finals):
+                #  disyllabic + monosyllabic, e.g. 蒙古/包
+                if len(word_list[0]) == 2:
+                    finals[0] = finals[0][:-1] + "2"
+                    finals[1] = finals[1][:-1] + "2"
+                #  monosyllabic + disyllabic, e.g. 纸/老虎
+                elif len(word_list[0]) == 1:
+                    finals[1] = finals[1][:-1] + "2"
+            else:
+                finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
+                if len(finals_list) == 2:
+                    for i, sub in enumerate(finals_list):
+                        # e.g. 所有/人
+                        if self._all_tone_three(sub) and len(sub) == 2:
+                            finals_list[i][0] = finals_list[i][0][:-1] + "2"
+                        # e.g. 好/喜欢
+                        elif (
+                            i == 1
+                            and not self._all_tone_three(sub)
+                            and finals_list[i][0][-1] == "3"
+                            and finals_list[0][-1][-1] == "3"
+                        ):
+                            finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
+                        finals = sum(finals_list, [])
+        # split idiom into two words who's length is 2
+        elif len(word) == 4:
+            finals_list = [finals[:2], finals[2:]]
+            finals = []
+            for sub in finals_list:
+                if self._all_tone_three(sub):
+                    sub[0] = sub[0][:-1] + "2"
+                finals += sub
+
+        return finals
+
+    def _all_tone_three(self, finals: list[str]) -> bool:
+        return all(x[-1] == "3" for x in finals)
+
+    # merge "不" and the word behind it
+    # if don't merge, "不" sometimes appears alone according to jieba, which may occur sandhi error
+    def _merge_bu(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        new_seg = []
+        last_word = ""
+        for word, pos in seg:
+            if last_word == "不":
+                word = last_word + word
+            if word != "不":
+                new_seg.append((word, pos))
+            last_word = word[:]
+        if last_word == "不":
+            new_seg.append((last_word, "d"))
+            last_word = ""
+        return new_seg
+
+    # function 1: merge "一" and reduplication words in it's left and right, e.g. "听","一","听" ->"听一听"
+    # function 2: merge single  "一" and the word behind it
+    # if don't merge, "一" sometimes appears alone according to jieba, which may occur sandhi error
+    # e.g.
+    # input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
+    # output seg: [['听一听', 'v']]
+    def _merge_yi(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        new_seg = [] * len(seg)
+        # function 1
+        i = 0
+        while i < len(seg):
+            word, pos = seg[i]
+            if (
+                i - 1 >= 0
+                and word == "一"
+                and i + 1 < len(seg)
+                and seg[i - 1][0] == seg[i + 1][0]
+                and seg[i - 1][1] == "v"
+            ):
+                new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
+                i += 2
+            else:
+                if (
+                    i - 2 >= 0
+                    and seg[i - 1][0] == "一"
+                    and seg[i - 2][0] == word
+                    and pos == "v"
+                ):
+                    continue
+                else:
+                    new_seg.append([word, pos])
+                i += 1
+        seg = [i for i in new_seg if len(i) > 0]
+        new_seg = []
+        # function 2
+        for i, (word, pos) in enumerate(seg):
+            if new_seg and new_seg[-1][0] == "一":
+                new_seg[-1][0] = new_seg[-1][0] + word
+            else:
+                new_seg.append([word, pos])
+        return new_seg
+
+    # the first and the second words are all_tone_three
+    def _merge_continuous_three_tones(
+        self, seg: list[tuple[str, str]]
+    ) -> list[tuple[str, str]]:
+        new_seg = []
+        sub_finals_list = [
+            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
+            for (word, pos) in seg
+        ]
+        assert len(sub_finals_list) == len(seg)
+        merge_last = [False] * len(seg)
+        for i, (word, pos) in enumerate(seg):
+            if (
+                i - 1 >= 0
+                and self._all_tone_three(sub_finals_list[i - 1])
+                and self._all_tone_three(sub_finals_list[i])
+                and not merge_last[i - 1]
+            ):
+                # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
+                if (
+                    not self._is_reduplication(seg[i - 1][0])
+                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
+                ):
+                    new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
+                    merge_last[i] = True
+                else:
+                    new_seg.append([word, pos])
+            else:
+                new_seg.append([word, pos])
+
+        return new_seg
+
+    def _is_reduplication(self, word: str) -> bool:
+        return len(word) == 2 and word[0] == word[1]
+
+    # the last char of first word and the first char of second word is tone_three
+    def _merge_continuous_three_tones_2(
+        self, seg: list[tuple[str, str]]
+    ) -> list[tuple[str, str]]:
+        new_seg = []
+        sub_finals_list = [
+            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
+            for (word, pos) in seg
+        ]
+        assert len(sub_finals_list) == len(seg)
+        merge_last = [False] * len(seg)
+        for i, (word, pos) in enumerate(seg):
+            if (
+                i - 1 >= 0
+                and sub_finals_list[i - 1][-1][-1] == "3"
+                and sub_finals_list[i][0][-1] == "3"
+                and not merge_last[i - 1]
+            ):
+                # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
+                if (
+                    not self._is_reduplication(seg[i - 1][0])
+                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
+                ):
+                    new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
+                    merge_last[i] = True
+                else:
+                    new_seg.append([word, pos])
+            else:
+                new_seg.append([word, pos])
+        return new_seg
+
+    def _merge_er(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        new_seg = []
+        for i, (word, pos) in enumerate(seg):
+            if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
+                new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
+            else:
+                new_seg.append([word, pos])
+        return new_seg
+
+    def _merge_reduplication(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        new_seg = []
+        for i, (word, pos) in enumerate(seg):
+            if new_seg and word == new_seg[-1][0]:
+                new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
+            else:
+                new_seg.append([word, pos])
+        return new_seg
+
+    def pre_merge_for_modify(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        seg = self._merge_bu(seg)
+        try:
+            seg = self._merge_yi(seg)
+        except:
+            print("_merge_yi failed")
+        seg = self._merge_reduplication(seg)
+        seg = self._merge_continuous_three_tones(seg)
+        seg = self._merge_continuous_three_tones_2(seg)
+        seg = self._merge_er(seg)
+        return seg
+
+    def modified_tone(self, word: str, pos: str, finals: list[str]) -> list[str]:
+        finals = self._bu_sandhi(word, finals)
+        finals = self._yi_sandhi(word, finals)
+        finals = self._neural_sandhi(word, pos, finals)
+        finals = self._three_sandhi(word, finals)
+        return finals

style_bert_vits2/nlp/english/__init__.py

@@ -0,0 +1,488 @@
+import pickle
+import os
+import re
+from g2p_en import G2p
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS, SYMBOLS
+
+
+current_file_path = os.path.dirname(__file__)
+CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
+CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
+_g2p = G2p()
+
+arpa = {
+    "AH0",
+    "S",
+    "AH1",
+    "EY2",
+    "AE2",
+    "EH0",
+    "OW2",
+    "UH0",
+    "NG",
+    "B",
+    "G",
+    "AY0",
+    "M",
+    "AA0",
+    "F",
+    "AO0",
+    "ER2",
+    "UH1",
+    "IY1",
+    "AH2",
+    "DH",
+    "IY0",
+    "EY1",
+    "IH0",
+    "K",
+    "N",
+    "W",
+    "IY2",
+    "T",
+    "AA1",
+    "ER1",
+    "EH2",
+    "OY0",
+    "UH2",
+    "UW1",
+    "Z",
+    "AW2",
+    "AW1",
+    "V",
+    "UW2",
+    "AA2",
+    "ER",
+    "AW0",
+    "UW0",
+    "R",
+    "OW1",
+    "EH1",
+    "ZH",
+    "AE0",
+    "IH2",
+    "IH",
+    "Y",
+    "JH",
+    "P",
+    "AY1",
+    "EY0",
+    "OY2",
+    "TH",
+    "HH",
+    "D",
+    "ER0",
+    "CH",
+    "AO1",
+    "AE1",
+    "AO2",
+    "OY1",
+    "AY2",
+    "IH1",
+    "OW0",
+    "L",
+    "SH",
+}
+
+
+def post_replace_ph(ph):
+    rep_map = {
+        "：": ",",
+        "；": ",",
+        "，": ",",
+        "。": ".",
+        "！": "!",
+        "？": "?",
+        "\n": ".",
+        "·": ",",
+        "、": ",",
+        "…": "...",
+        "···": "...",
+        "・・・": "...",
+        "v": "V",
+    }
+    if ph in rep_map.keys():
+        ph = rep_map[ph]
+    if ph in SYMBOLS:
+        return ph
+    if ph not in SYMBOLS:
+        ph = "UNK"
+    return ph
+
+
+rep_map = {
+    "：": ",",
+    "；": ",",
+    "，": ",",
+    "。": ".",
+    "！": "!",
+    "？": "?",
+    "\n": ".",
+    "．": ".",
+    "…": "...",
+    "···": "...",
+    "・・・": "...",
+    "·": ",",
+    "・": ",",
+    "、": ",",
+    "$": ".",
+    "“": "'",
+    "”": "'",
+    '"': "'",
+    "‘": "'",
+    "’": "'",
+    "（": "'",
+    "）": "'",
+    "(": "'",
+    ")": "'",
+    "《": "'",
+    "》": "'",
+    "【": "'",
+    "】": "'",
+    "[": "'",
+    "]": "'",
+    "—": "-",
+    "−": "-",
+    "～": "-",
+    "~": "-",
+    "「": "'",
+    "」": "'",
+}
+
+
+def replace_punctuation(text):
+    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
+
+    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
+
+    # replaced_text = re.sub(
+    #     r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
+    #     + "".join(punctuation)
+    #     + r"]+",
+    #     "",
+    #     replaced_text,
+    # )
+
+    return replaced_text
+
+
+def read_dict():
+    g2p_dict = {}
+    start_line = 49
+    with open(CMU_DICT_PATH) as f:
+        line = f.readline()
+        line_index = 1
+        while line:
+            if line_index >= start_line:
+                line = line.strip()
+                word_split = line.split("  ")
+                word = word_split[0]
+
+                syllable_split = word_split[1].split(" - ")
+                g2p_dict[word] = []
+                for syllable in syllable_split:
+                    phone_split = syllable.split(" ")
+                    g2p_dict[word].append(phone_split)
+
+            line_index = line_index + 1
+            line = f.readline()
+
+    return g2p_dict
+
+
+def cache_dict(g2p_dict, file_path):
+    with open(file_path, "wb") as pickle_file:
+        pickle.dump(g2p_dict, pickle_file)
+
+
+def get_dict():
+    if os.path.exists(CACHE_PATH):
+        with open(CACHE_PATH, "rb") as pickle_file:
+            g2p_dict = pickle.load(pickle_file)
+    else:
+        g2p_dict = read_dict()
+        cache_dict(g2p_dict, CACHE_PATH)
+
+    return g2p_dict
+
+
+eng_dict = get_dict()
+
+
+def refine_ph(phn):
+    tone = 0
+    if re.search(r"\d$", phn):
+        tone = int(phn[-1]) + 1
+        phn = phn[:-1]
+    else:
+        tone = 3
+    return phn.lower(), tone
+
+
+def refine_syllables(syllables):
+    tones = []
+    phonemes = []
+    for phn_list in syllables:
+        for i in range(len(phn_list)):
+            phn = phn_list[i]
+            phn, tone = refine_ph(phn)
+            phonemes.append(phn)
+            tones.append(tone)
+    return phonemes, tones
+
+
+import inflect
+
+_inflect = inflect.engine()
+_comma_number_re = re.compile(r"([0-9][0-9\,]+[0-9])")
+_decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)")
+_pounds_re = re.compile(r"£([0-9\,]*[0-9]+)")
+_dollars_re = re.compile(r"\$([0-9\.\,]*[0-9]+)")
+_ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)")
+_number_re = re.compile(r"[0-9]+")
+
+# List of (regular expression, replacement) pairs for abbreviations:
+_abbreviations = [
+    (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+    for x in [
+        ("mrs", "misess"),
+        ("mr", "mister"),
+        ("dr", "doctor"),
+        ("st", "saint"),
+        ("co", "company"),
+        ("jr", "junior"),
+        ("maj", "major"),
+        ("gen", "general"),
+        ("drs", "doctors"),
+        ("rev", "reverend"),
+        ("lt", "lieutenant"),
+        ("hon", "honorable"),
+        ("sgt", "sergeant"),
+        ("capt", "captain"),
+        ("esq", "esquire"),
+        ("ltd", "limited"),
+        ("col", "colonel"),
+        ("ft", "fort"),
+    ]
+]
+
+
+# List of (ipa, lazy ipa) pairs:
+_lazy_ipa = [
+    (re.compile("%s" % x[0]), x[1])
+    for x in [
+        ("r", "ɹ"),
+        ("æ", "e"),
+        ("ɑ", "a"),
+        ("ɔ", "o"),
+        ("ð", "z"),
+        ("θ", "s"),
+        ("ɛ", "e"),
+        ("ɪ", "i"),
+        ("ʊ", "u"),
+        ("ʒ", "ʥ"),
+        ("ʤ", "ʥ"),
+        ("ˈ", "↓"),
+    ]
+]
+
+# List of (ipa, lazy ipa2) pairs:
+_lazy_ipa2 = [
+    (re.compile("%s" % x[0]), x[1])
+    for x in [
+        ("r", "ɹ"),
+        ("ð", "z"),
+        ("θ", "s"),
+        ("ʒ", "ʑ"),
+        ("ʤ", "dʑ"),
+        ("ˈ", "↓"),
+    ]
+]
+
+# List of (ipa, ipa2) pairs
+_ipa_to_ipa2 = [
+    (re.compile("%s" % x[0]), x[1]) for x in [("r", "ɹ"), ("ʤ", "dʒ"), ("ʧ", "tʃ")]
+]
+
+
+def _expand_dollars(m):
+    match = m.group(1)
+    parts = match.split(".")
+    if len(parts) > 2:
+        return match + " dollars"  # Unexpected format
+    dollars = int(parts[0]) if parts[0] else 0
+    cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+    if dollars and cents:
+        dollar_unit = "dollar" if dollars == 1 else "dollars"
+        cent_unit = "cent" if cents == 1 else "cents"
+        return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit)
+    elif dollars:
+        dollar_unit = "dollar" if dollars == 1 else "dollars"
+        return "%s %s" % (dollars, dollar_unit)
+    elif cents:
+        cent_unit = "cent" if cents == 1 else "cents"
+        return "%s %s" % (cents, cent_unit)
+    else:
+        return "zero dollars"
+
+
+def _remove_commas(m):
+    return m.group(1).replace(",", "")
+
+
+def _expand_ordinal(m):
+    return _inflect.number_to_words(m.group(0))
+
+
+def _expand_number(m):
+    num = int(m.group(0))
+    if num > 1000 and num < 3000:
+        if num == 2000:
+            return "two thousand"
+        elif num > 2000 and num < 2010:
+            return "two thousand " + _inflect.number_to_words(num % 100)
+        elif num % 100 == 0:
+            return _inflect.number_to_words(num // 100) + " hundred"
+        else:
+            return _inflect.number_to_words(
+                num, andword="", zero="oh", group=2
+            ).replace(", ", " ")
+    else:
+        return _inflect.number_to_words(num, andword="")
+
+
+def _expand_decimal_point(m):
+    return m.group(1).replace(".", " point ")
+
+
+def normalize_numbers(text):
+    text = re.sub(_comma_number_re, _remove_commas, text)
+    text = re.sub(_pounds_re, r"\1 pounds", text)
+    text = re.sub(_dollars_re, _expand_dollars, text)
+    text = re.sub(_decimal_number_re, _expand_decimal_point, text)
+    text = re.sub(_ordinal_re, _expand_ordinal, text)
+    text = re.sub(_number_re, _expand_number, text)
+    return text
+
+
+def normalize_text(text: str) -> str:
+    text = normalize_numbers(text)
+    text = replace_punctuation(text)
+    text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
+    return text
+
+
+def distribute_phone(n_phone, n_word):
+    phones_per_word = [0] * n_word
+    for task in range(n_phone):
+        min_tasks = min(phones_per_word)
+        min_index = phones_per_word.index(min_tasks)
+        phones_per_word[min_index] += 1
+    return phones_per_word
+
+
+def sep_text(text):
+    words = re.split(r"([,;.\?\!\s+])", text)
+    words = [word for word in words if word.strip() != ""]
+    return words
+
+
+def text_to_words(text):
+    tokenizer = bert_models.load_tokenizer(Languages.EN)
+    tokens = tokenizer.tokenize(text)
+    words = []
+    for idx, t in enumerate(tokens):
+        if t.startswith("▁"):
+            words.append([t[1:]])
+        else:
+            if t in PUNCTUATIONS:
+                if idx == len(tokens) - 1:
+                    words.append([f"{t}"])
+                else:
+                    if (
+                        not tokens[idx + 1].startswith("▁")
+                        and tokens[idx + 1] not in PUNCTUATIONS
+                    ):
+                        if idx == 0:
+                            words.append([])
+                        words[-1].append(f"{t}")
+                    else:
+                        words.append([f"{t}"])
+            else:
+                if idx == 0:
+                    words.append([])
+                words[-1].append(f"{t}")
+    return words
+
+
+def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
+    phones = []
+    tones = []
+    phone_len = []
+    # words = sep_text(text)
+    # tokens = [tokenizer.tokenize(i) for i in words]
+    words = text_to_words(text)
+
+    for word in words:
+        temp_phones, temp_tones = [], []
+        if len(word) > 1:
+            if "'" in word:
+                word = ["".join(word)]
+        for w in word:
+            if w in PUNCTUATIONS:
+                temp_phones.append(w)
+                temp_tones.append(0)
+                continue
+            if w.upper() in eng_dict:
+                phns, tns = refine_syllables(eng_dict[w.upper()])
+                temp_phones += [post_replace_ph(i) for i in phns]
+                temp_tones += tns
+                # w2ph.append(len(phns))
+            else:
+                phone_list = list(filter(lambda p: p != " ", _g2p(w)))
+                phns = []
+                tns = []
+                for ph in phone_list:
+                    if ph in arpa:
+                        ph, tn = refine_ph(ph)
+                        phns.append(ph)
+                        tns.append(tn)
+                    else:
+                        phns.append(ph)
+                        tns.append(0)
+                temp_phones += [post_replace_ph(i) for i in phns]
+                temp_tones += tns
+        phones += temp_phones
+        tones += temp_tones
+        phone_len.append(len(temp_phones))
+        # phones = [post_replace_ph(i) for i in phones]
+
+    word2ph = []
+    for token, pl in zip(words, phone_len):
+        word_len = len(token)
+
+        aaa = distribute_phone(pl, word_len)
+        word2ph += aaa
+
+    phones = ["_"] + phones + ["_"]
+    tones = [0] + tones + [0]
+    word2ph = [1] + word2ph + [1]
+    assert len(phones) == len(tones), text
+    assert len(phones) == sum(word2ph), text
+
+    return phones, tones, word2ph
+
+
+if __name__ == "__main__":
+    # print(get_dict())
+    # print(eng_word_to_phoneme("hello"))
+    print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
+    # all_phones = set()
+    # for k, syllables in eng_dict.items():
+    #     for group in syllables:
+    #         for ph in group:
+    #             all_phones.add(ph)
+    # print(all_phones)

style_bert_vits2/nlp/english/bert_feature.py

@@ -0,0 +1,80 @@
+import sys
+from typing import Optional
+
+import torch
+from transformers import PreTrainedModel
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+
+
+__models: dict[torch.device | str, PreTrainedModel] = {}
+
+
+def extract_bert_feature(
+    text: str,
+    word2ph: list[int],
+    device: torch.device | str,
+    assist_text: Optional[str] = None,
+    assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+    """
+    英語のテキストから BERT の特徴量を抽出する
+
+    Args:
+        text (str): 英語のテキスト
+        word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+        device (torch.device | str): 推論に利用するデバイス
+        assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+        assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+    Returns:
+        torch.Tensor: BERT の特徴量
+    """
+
+    if (
+        sys.platform == "darwin"
+        and torch.backends.mps.is_available()
+        and device == "cpu"
+    ):
+        device = "mps"
+    if not device:
+        device = "cuda"
+    if device == "cuda" and not torch.cuda.is_available():
+        device = "cpu"
+    if device not in __models.keys():
+        __models[device] = bert_models.load_model(Languages.EN).to(device)  # type: ignore
+
+    style_res_mean = None
+    with torch.no_grad():
+        tokenizer = bert_models.load_tokenizer(Languages.EN)
+        inputs = tokenizer(text, return_tensors="pt")
+        for i in inputs:
+            inputs[i] = inputs[i].to(device)  # type: ignore
+        res = __models[device](**inputs, output_hidden_states=True)
+        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if assist_text:
+            style_inputs = tokenizer(assist_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)  # type: ignore
+            style_res = __models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
+
+    assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
+    word2phone = word2ph
+    phone_level_feature = []
+    for i in range(len(word2phone)):
+        if assist_text:
+            assert style_res_mean is not None
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
+        phone_level_feature.append(repeat_feature)
+
+    phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+    return phone_level_feature.T

style_bert_vits2/nlp/english/opencpop-strict.txt

@@ -0,0 +1,429 @@
+a	AA a
+ai	AA ai
+an	AA an
+ang	AA ang
+ao	AA ao
+ba	b a
+bai	b ai
+ban	b an
+bang	b ang
+bao	b ao
+bei	b ei
+ben	b en
+beng	b eng
+bi	b i
+bian	b ian
+biao	b iao
+bie	b ie
+bin	b in
+bing	b ing
+bo	b o
+bu	b u
+ca	c a
+cai	c ai
+can	c an
+cang	c ang
+cao	c ao
+ce	c e
+cei	c ei
+cen	c en
+ceng	c eng
+cha	ch a
+chai	ch ai
+chan	ch an
+chang	ch ang
+chao	ch ao
+che	ch e
+chen	ch en
+cheng	ch eng
+chi	ch ir
+chong	ch ong
+chou	ch ou
+chu	ch u
+chua	ch ua
+chuai	ch uai
+chuan	ch uan
+chuang	ch uang
+chui	ch ui
+chun	ch un
+chuo	ch uo
+ci	c i0
+cong	c ong
+cou	c ou
+cu	c u
+cuan	c uan
+cui	c ui
+cun	c un
+cuo	c uo
+da	d a
+dai	d ai
+dan	d an
+dang	d ang
+dao	d ao
+de	d e
+dei	d ei
+den	d en
+deng	d eng
+di	d i
+dia	d ia
+dian	d ian
+diao	d iao
+die	d ie
+ding	d ing
+diu	d iu
+dong	d ong
+dou	d ou
+du	d u
+duan	d uan
+dui	d ui
+dun	d un
+duo	d uo
+e	EE e
+ei	EE ei
+en	EE en
+eng	EE eng
+er	EE er
+fa	f a
+fan	f an
+fang	f ang
+fei	f ei
+fen	f en
+feng	f eng
+fo	f o
+fou	f ou
+fu	f u
+ga	g a
+gai	g ai
+gan	g an
+gang	g ang
+gao	g ao
+ge	g e
+gei	g ei
+gen	g en
+geng	g eng
+gong	g ong
+gou	g ou
+gu	g u
+gua	g ua
+guai	g uai
+guan	g uan
+guang	g uang
+gui	g ui
+gun	g un
+guo	g uo
+ha	h a
+hai	h ai
+han	h an
+hang	h ang
+hao	h ao
+he	h e
+hei	h ei
+hen	h en
+heng	h eng
+hong	h ong
+hou	h ou
+hu	h u
+hua	h ua
+huai	h uai
+huan	h uan
+huang	h uang
+hui	h ui
+hun	h un
+huo	h uo
+ji	j i
+jia	j ia
+jian	j ian
+jiang	j iang
+jiao	j iao
+jie	j ie
+jin	j in
+jing	j ing
+jiong	j iong
+jiu	j iu
+ju	j v
+jv	j v
+juan	j van
+jvan	j van
+jue	j ve
+jve	j ve
+jun	j vn
+jvn	j vn
+ka	k a
+kai	k ai
+kan	k an
+kang	k ang
+kao	k ao
+ke	k e
+kei	k ei
+ken	k en
+keng	k eng
+kong	k ong
+kou	k ou
+ku	k u
+kua	k ua
+kuai	k uai
+kuan	k uan
+kuang	k uang
+kui	k ui
+kun	k un
+kuo	k uo
+la	l a
+lai	l ai
+lan	l an
+lang	l ang
+lao	l ao
+le	l e
+lei	l ei
+leng	l eng
+li	l i
+lia	l ia
+lian	l ian
+liang	l iang
+liao	l iao
+lie	l ie
+lin	l in
+ling	l ing
+liu	l iu
+lo	l o
+long	l ong
+lou	l ou
+lu	l u
+luan	l uan
+lun	l un
+luo	l uo
+lv	l v
+lve	l ve
+ma	m a
+mai	m ai
+man	m an
+mang	m ang
+mao	m ao
+me	m e
+mei	m ei
+men	m en
+meng	m eng
+mi	m i
+mian	m ian
+miao	m iao
+mie	m ie
+min	m in
+ming	m ing
+miu	m iu
+mo	m o
+mou	m ou
+mu	m u
+na	n a
+nai	n ai
+nan	n an
+nang	n ang
+nao	n ao
+ne	n e
+nei	n ei
+nen	n en
+neng	n eng
+ni	n i
+nian	n ian
+niang	n iang
+niao	n iao
+nie	n ie
+nin	n in
+ning	n ing
+niu	n iu
+nong	n ong
+nou	n ou
+nu	n u
+nuan	n uan
+nun	n un
+nuo	n uo
+nv	n v
+nve	n ve
+o	OO o
+ou	OO ou
+pa	p a
+pai	p ai
+pan	p an
+pang	p ang
+pao	p ao
+pei	p ei
+pen	p en
+peng	p eng
+pi	p i
+pian	p ian
+piao	p iao
+pie	p ie
+pin	p in
+ping	p ing
+po	p o
+pou	p ou
+pu	p u
+qi	q i
+qia	q ia
+qian	q ian
+qiang	q iang
+qiao	q iao
+qie	q ie
+qin	q in
+qing	q ing
+qiong	q iong
+qiu	q iu
+qu	q v
+qv	q v
+quan	q van
+qvan	q van
+que	q ve
+qve	q ve
+qun	q vn
+qvn	q vn
+ran	r an
+rang	r ang
+rao	r ao
+re	r e
+ren	r en
+reng	r eng
+ri	r ir
+rong	r ong
+rou	r ou
+ru	r u
+rua	r ua
+ruan	r uan
+rui	r ui
+run	r un
+ruo	r uo
+sa	s a
+sai	s ai
+san	s an
+sang	s ang
+sao	s ao
+se	s e
+sen	s en
+seng	s eng
+sha	sh a
+shai	sh ai
+shan	sh an
+shang	sh ang
+shao	sh ao
+she	sh e
+shei	sh ei
+shen	sh en
+sheng	sh eng
+shi	sh ir
+shou	sh ou
+shu	sh u
+shua	sh ua
+shuai	sh uai
+shuan	sh uan
+shuang	sh uang
+shui	sh ui
+shun	sh un
+shuo	sh uo
+si	s i0
+song	s ong
+sou	s ou
+su	s u
+suan	s uan
+sui	s ui
+sun	s un
+suo	s uo
+ta	t a
+tai	t ai
+tan	t an
+tang	t ang
+tao	t ao
+te	t e
+tei	t ei
+teng	t eng
+ti	t i
+tian	t ian
+tiao	t iao
+tie	t ie
+ting	t ing
+tong	t ong
+tou	t ou
+tu	t u
+tuan	t uan
+tui	t ui
+tun	t un
+tuo	t uo
+wa	w a
+wai	w ai
+wan	w an
+wang	w ang
+wei	w ei
+wen	w en
+weng	w eng
+wo	w o
+wu	w u
+xi	x i
+xia	x ia
+xian	x ian
+xiang	x iang
+xiao	x iao
+xie	x ie
+xin	x in
+xing	x ing
+xiong	x iong
+xiu	x iu
+xu	x v
+xv	x v
+xuan	x van
+xvan	x van
+xue	x ve
+xve	x ve
+xun	x vn
+xvn	x vn
+ya	y a
+yan	y En
+yang	y ang
+yao	y ao
+ye	y E
+yi	y i
+yin	y in
+ying	y ing
+yo	y o
+yong	y ong
+you	y ou
+yu	y v
+yv	y v
+yuan	y van
+yvan	y van
+yue	y ve
+yve	y ve
+yun	y vn
+yvn	y vn
+za	z a
+zai	z ai
+zan	z an
+zang	z ang
+zao	z ao
+ze	z e
+zei	z ei
+zen	z en
+zeng	z eng
+zha	zh a
+zhai	zh ai
+zhan	zh an
+zhang	zh ang
+zhao	zh ao
+zhe	zh e
+zhei	zh ei
+zhen	zh en
+zheng	zh eng
+zhi	zh ir
+zhong	zh ong
+zhou	zh ou
+zhu	zh u
+zhua	zh ua
+zhuai	zh uai
+zhuan	zh uan
+zhuang	zh uang
+zhui	zh ui
+zhun	zh un
+zhuo	zh uo
+zi	z i0
+zong	z ong
+zou	z ou
+zu	z u
+zuan	z uan
+zui	z ui
+zun	z un
+zuo	z uo

style_bert_vits2/nlp/japanese/__init__.py

@@ -0,0 +1,2 @@
+from style_bert_vits2.nlp.japanese.g2p import g2p  # noqa: F401
+from style_bert_vits2.nlp.japanese.normalizer import normalize_text  # noqa: F401

style_bert_vits2/nlp/japanese/bert_feature.py

@@ -0,0 +1,87 @@
+import sys
+from typing import Optional
+
+import torch
+from transformers import PreTrainedModel
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.japanese.g2p import text_to_sep_kata
+
+
+__models: dict[torch.device | str, PreTrainedModel] = {}
+
+
+def extract_bert_feature(
+    text: str,
+    word2ph: list[int],
+    device: torch.device | str,
+    assist_text: Optional[str] = None,
+    assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+    """
+    日本語のテキストから BERT の特徴量を抽出する
+
+    Args:
+        text (str): 日本語のテキスト
+        word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+        device (torch.device | str): 推論に利用するデバイス
+        assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+        assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+    Returns:
+        torch.Tensor: BERT の特徴量
+    """
+
+    # 各単語が何文字かを作る `word2ph` を使う必要があるので、読めない文字は必ず無視する
+    # でないと `word2ph` の結果とテキストの文字数結果が整合性が取れない
+    text = "".join(text_to_sep_kata(text, raise_yomi_error=False)[0])
+
+    if assist_text:
+        assist_text = "".join(text_to_sep_kata(assist_text, raise_yomi_error=False)[0])
+    if (
+        sys.platform == "darwin"
+        and torch.backends.mps.is_available()
+        and device == "cpu"
+    ):
+        device = "mps"
+    if not device:
+        device = "cuda"
+    if device == "cuda" and not torch.cuda.is_available():
+        device = "cpu"
+    if device not in __models.keys():
+        __models[device] = bert_models.load_model(Languages.JP).to(device)  # type: ignore
+
+    style_res_mean = None
+    with torch.no_grad():
+        tokenizer = bert_models.load_tokenizer(Languages.JP)
+        inputs = tokenizer(text, return_tensors="pt")
+        for i in inputs:
+            inputs[i] = inputs[i].to(device)  # type: ignore
+        res = __models[device](**inputs, output_hidden_states=True)
+        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if assist_text:
+            style_inputs = tokenizer(assist_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)  # type: ignore
+            style_res = __models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
+
+    assert len(word2ph) == len(text) + 2, text
+    word2phone = word2ph
+    phone_level_feature = []
+    for i in range(len(word2phone)):
+        if assist_text:
+            assert style_res_mean is not None
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
+        phone_level_feature.append(repeat_feature)
+
+    phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+    return phone_level_feature.T

style_bert_vits2/nlp/japanese/g2p.py

@@ -0,0 +1,493 @@
+import re
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.mora_list import MORA_KATA_TO_MORA_PHONEMES
+from style_bert_vits2.nlp.japanese.normalizer import replace_punctuation
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+pyopenjtalk.initialize()
+
+
+def g2p(
+    norm_text: str,
+    use_jp_extra: bool = True,
+    raise_yomi_error: bool = False
+) -> tuple[list[str], list[int], list[int]]:
+    """
+    他で使われるメインの関数。`normalize_text()` で正規化された `norm_text` を受け取り、
+    - phones: 音素のリスト（ただし `!` や `,` や `.` など punctuation が含まれうる）
+    - tones: アクセントのリスト、0（低）と1（高）からなり、phones と同じ長さ
+    - word2ph: 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+    のタプルを返す。
+    ただし `phones` と `tones` の最初と終わりに `_` が入り、応じて `word2ph` の最初と最後に 1 が追加される。
+
+    Args:
+        norm_text (str): 正規化されたテキスト
+        use_jp_extra (bool, optional): False の場合、「ん」の音素を「N」ではなく「n」とする。Defaults to True.
+        raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
+
+    Returns:
+        tuple[list[str], list[int], list[int]]: 音素のリスト、アクセントのリスト、word2ph のリスト
+    """
+
+    # pyopenjtalk のフルコンテキストラベルを使ってアクセントを取り出すと、punctuation の位置が消えてしまい情報が失われてしまう：
+    # 「こんにちは、世界。」と「こんにちは！世界。」と「こんにちは！！！？？？世界……。」は全て同じになる。
+    # よって、まず punctuation 無しの音素とアクセントのリストを作り、
+    # それとは別に pyopenjtalk.run_frontend() で得られる音素リスト（こちらは punctuation が保持される）を使い、
+    # アクセント割当をしなおすことによって punctuation を含めた音素とアクセントのリストを作る。
+
+    # punctuation がすべて消えた、音素とアクセントのタプルのリスト（「ん」は「N」）
+    phone_tone_list_wo_punct = __g2phone_tone_wo_punct(norm_text)
+
+    # sep_text: 単語単位の単語のリスト、読めない文字があったら raise_yomi_error なら例外、そうでないなら読めない文字が消えて返ってくる
+    # sep_kata: 単語単位の単語のカタカナ読みのリスト
+    sep_text, sep_kata = text_to_sep_kata(norm_text, raise_yomi_error=raise_yomi_error)
+
+    # sep_phonemes: 各単語ごとの音素のリストのリスト
+    sep_phonemes = __handle_long([__kata_to_phoneme_list(i) for i in sep_kata])
+
+    # phone_w_punct: sep_phonemes を結合した、punctuation を元のまま保持した音素列
+    phone_w_punct: list[str] = []
+    for i in sep_phonemes:
+        phone_w_punct += i
+
+    # punctuation 無しのアクセント情報を使って、punctuation を含めたアクセント情報を作る
+    phone_tone_list = __align_tones(phone_w_punct, phone_tone_list_wo_punct)
+    # logger.debug(f"phone_tone_list:\n{phone_tone_list}")
+
+    # word2ph は厳密な解答は不可能なので（「今日」「眼鏡」等の熟字訓が存在）、
+    # Bert-VITS2 では、単語単位の分割を使って、単語の文字ごとにだいたい均等に音素を分配する
+
+    # sep_text から、各単語を1文字1文字分割して、文字のリスト（のリスト）を作る
+    sep_tokenized: list[list[str]] = []
+    for i in sep_text:
+        if i not in PUNCTUATIONS:
+            sep_tokenized.append(
+                bert_models.load_tokenizer(Languages.JP).tokenize(i)
+            )  # ここでおそらく`i`が文字単位に分割される
+        else:
+            sep_tokenized.append([i])
+
+    # 各単語について、音素の数と文字の数を比較して、均等っぽく分配する
+    word2ph = []
+    for token, phoneme in zip(sep_tokenized, sep_phonemes):
+        phone_len = len(phoneme)
+        word_len = len(token)
+        word2ph += __distribute_phone(phone_len, word_len)
+
+    # 最初と最後に `_` 記号を追加、アクセントは 0（低）、word2ph もそれに合わせて追加
+    phone_tone_list = [("_", 0)] + phone_tone_list + [("_", 0)]
+    word2ph = [1] + word2ph + [1]
+
+    phones = [phone for phone, _ in phone_tone_list]
+    tones = [tone for _, tone in phone_tone_list]
+
+    assert len(phones) == sum(word2ph), f"{len(phones)} != {sum(word2ph)}"
+
+    # use_jp_extra でない場合は「N」を「n」に変換
+    if not use_jp_extra:
+        phones = [phone if phone != "N" else "n" for phone in phones]
+
+    return phones, tones, word2ph
+
+
+def text_to_sep_kata(
+    norm_text: str,
+    raise_yomi_error: bool = False
+) -> tuple[list[str], list[str]]:
+    """
+    `normalize_text` で正規化済みの `norm_text` を受け取り、それを単語分割し、
+    分割された単語リストとその読み（カタカナ or 記号1文字）のリストのタプルを返す。
+    単語分割結果は、`g2p()` の `word2ph` で1文字あたりに割り振る音素記号の数を決めるために使う。
+    例:
+    `私はそう思う!って感じ?` →
+    ["私", "は", "そう", "思う", "!", "って", "感じ", "?"], ["ワタシ", "ワ", "ソー", "オモウ", "!", "ッテ", "カンジ", "?"]
+
+    Args:
+        norm_text (str): 正規化されたテキスト
+        raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
+
+    Returns:
+        tuple[list[str], list[str]]: 分割された単語リストと、その読み（カタカナ or 記号1文字）のリスト
+    """
+
+    # parsed: OpenJTalkの解析結果
+    parsed = pyopenjtalk.run_frontend(norm_text)
+    sep_text: list[str] = []
+    sep_kata: list[str] = []
+
+    for parts in parsed:
+        # word: 実際の単語の文字列
+        # yomi: その読み、但し無声化サインの`’`は除去
+        word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
+            "’", ""
+        )
+        """
+        ここで `yomi` の取りうる値は以下の通りのはず。
+        - `word` が通常単語 → 通常の読み（カタカナ）
+            （カタカナからなり、長音記号も含みうる、`アー` 等）
+        - `word` が `ー` から始まる → `ーラー` や `ーーー` など
+        - `word` が句読点や空白等 → `、`
+        - `word` が punctuation の繰り返し → 全角にしたもの
+        基本的に punctuation は1文字ずつ分かれるが、何故かある程度連続すると1つにまとまる。
+        他にも `word` が読めないキリル文字アラビア文字等が来ると `、` になるが、正規化でこの場合は起きないはず。
+        また元のコードでは `yomi` が空白の場合の処理があったが、これは起きないはず。
+        処理すべきは `yomi` が `、` の場合のみのはず。
+        """
+        assert yomi != "", f"Empty yomi: {word}"
+        if yomi == "、":
+            # word は正規化されているので、`.`, `,`, `!`, `'`, `-`, `--` のいずれか
+            if not set(word).issubset(set(PUNCTUATIONS)):  # 記号繰り返しか判定
+                # ここは pyopenjtalk が読めない文字等のときに起こる
+                if raise_yomi_error:
+                    raise YomiError(f"Cannot read: {word} in:\n{norm_text}")
+                logger.warning(f"Ignoring unknown: {word} in:\n{norm_text}")
+                continue
+            # yomi は元の記号のままに変更
+            yomi = word
+        elif yomi == "？":
+            assert word == "?", f"yomi `？` comes from: {word}"
+            yomi = "?"
+        sep_text.append(word)
+        sep_kata.append(yomi)
+
+    return sep_text, sep_kata
+
+
+def __g2phone_tone_wo_punct(text: str) -> list[tuple[str, int]]:
+    """
+    テキストに対して、音素とアクセント（0か1）のペアのリストを返す。
+    ただし「!」「.」「?」等の非音素記号 (punctuation) は全て消える（ポーズ記号も残さない）。
+    非音素記号を含める処理は `align_tones()` で行われる。
+    また「っ」は「q」に、「ん」は「N」に変換される。
+    例: "こんにちは、世界ー。。元気？！" →
+    [('k', 0), ('o', 0), ('N', 1), ('n', 1), ('i', 1), ('ch', 1), ('i', 1), ('w', 1), ('a', 1), ('s', 1), ('e', 1), ('k', 0), ('a', 0), ('i', 0), ('i', 0), ('g', 1), ('e', 1), ('N', 0), ('k', 0), ('i', 0)]
+
+    Args:
+        text (str): テキスト
+
+    Returns:
+        list[tuple[str, int]]: 音素とアクセントのペアのリスト
+    """
+
+    prosodies = __pyopenjtalk_g2p_prosody(text, drop_unvoiced_vowels=True)
+    # logger.debug(f"prosodies: {prosodies}")
+    result: list[tuple[str, int]] = []
+    current_phrase: list[tuple[str, int]] = []
+    current_tone = 0
+
+    for i, letter in enumerate(prosodies):
+        # 特殊記号の処理
+
+        # 文頭記号、無視する
+        if letter == "^":
+            assert i == 0, "Unexpected ^"
+        # アクセント句の終わりに来る記号
+        elif letter in ("$", "?", "_", "#"):
+            # 保持しているフレーズを、アクセント数値を 0-1 に修正し結果に追加
+            result.extend(__fix_phone_tone(current_phrase))
+            # 末尾に来る終了記号、無視（文中の疑問文は `_` になる）
+            if letter in ("$", "?"):
+                assert i == len(prosodies) - 1, f"Unexpected {letter}"
+            # あとは "_"（ポーズ）と "#"（アクセント句の境界）のみ
+            # これらは残さず、次のアクセント句に備える。
+            current_phrase = []
+            # 0 を基準点にしてそこから上昇・下降する（負の場合は上の `fix_phone_tone` で直る）
+            current_tone = 0
+        # アクセント上昇記号
+        elif letter == "[":
+            current_tone = current_tone + 1
+        # アクセント下降記号
+        elif letter == "]":
+            current_tone = current_tone - 1
+        # それ以外は通常の音素
+        else:
+            if letter == "cl":  # 「っ」の処理
+                letter = "q"
+            # elif letter == "N":  # 「ん」の処理
+            #     letter = "n"
+            current_phrase.append((letter, current_tone))
+
+    return result
+
+
+def __pyopenjtalk_g2p_prosody(text: str, drop_unvoiced_vowels: bool = True) -> list[str]:
+    """
+    ESPnet の実装から引用、変更点無し。「ん」は「N」なことに注意。
+    ref: https://github.com/espnet/espnet/blob/master/espnet2/text/phoneme_tokenizer.py
+    ------------------------------------------------------------------------------------------
+
+    Extract phoneme + prosoody symbol sequence from input full-context labels.
+
+    The algorithm is based on `Prosodic features control by symbols as input of
+    sequence-to-sequence acoustic modeling for neural TTS`_ with some r9y9's tweaks.
+
+    Args:
+        text (str): Input text.
+        drop_unvoiced_vowels (bool): whether to drop unvoiced vowels.
+
+    Returns:
+        List[str]: List of phoneme + prosody symbols.
+
+    Examples:
+        >>> from espnet2.text.phoneme_tokenizer import pyopenjtalk_g2p_prosody
+        >>> pyopenjtalk_g2p_prosody("こんにちは。")
+        ['^', 'k', 'o', '[', 'N', 'n', 'i', 'ch', 'i', 'w', 'a', '$']
+
+    .. _`Prosodic features control by symbols as input of sequence-to-sequence acoustic
+        modeling for neural TTS`: https://doi.org/10.1587/transinf.2020EDP7104
+    """
+
+    def _numeric_feature_by_regex(regex: str, s: str) -> int:
+        match = re.search(regex, s)
+        if match is None:
+            return -50
+        return int(match.group(1))
+
+    labels = pyopenjtalk.make_label(pyopenjtalk.run_frontend(text))
+    N = len(labels)
+
+    phones = []
+    for n in range(N):
+        lab_curr = labels[n]
+
+        # current phoneme
+        p3 = re.search(r"\-(.*?)\+", lab_curr).group(1)  # type: ignore
+        # deal unvoiced vowels as normal vowels
+        if drop_unvoiced_vowels and p3 in "AEIOU":
+            p3 = p3.lower()
+
+        # deal with sil at the beginning and the end of text
+        if p3 == "sil":
+            assert n == 0 or n == N - 1
+            if n == 0:
+                phones.append("^")
+            elif n == N - 1:
+                # check question form or not
+                e3 = _numeric_feature_by_regex(r"!(\d+)_", lab_curr)
+                if e3 == 0:
+                    phones.append("$")
+                elif e3 == 1:
+                    phones.append("?")
+            continue
+        elif p3 == "pau":
+            phones.append("_")
+            continue
+        else:
+            phones.append(p3)
+
+        # accent type and position info (forward or backward)
+        a1 = _numeric_feature_by_regex(r"/A:([0-9\-]+)\+", lab_curr)
+        a2 = _numeric_feature_by_regex(r"\+(\d+)\+", lab_curr)
+        a3 = _numeric_feature_by_regex(r"\+(\d+)/", lab_curr)
+
+        # number of mora in accent phrase
+        f1 = _numeric_feature_by_regex(r"/F:(\d+)_", lab_curr)
+
+        a2_next = _numeric_feature_by_regex(r"\+(\d+)\+", labels[n + 1])
+        # accent phrase border
+        if a3 == 1 and a2_next == 1 and p3 in "aeiouAEIOUNcl":
+            phones.append("#")
+        # pitch falling
+        elif a1 == 0 and a2_next == a2 + 1 and a2 != f1:
+            phones.append("]")
+        # pitch rising
+        elif a2 == 1 and a2_next == 2:
+            phones.append("[")
+
+    return phones
+
+
+def __fix_phone_tone(phone_tone_list: list[tuple[str, int]]) -> list[tuple[str, int]]:
+    """
+    `phone_tone_list` の tone（アクセントの値）を 0 か 1 の範囲に修正する。
+    例: [(a, 0), (i, -1), (u, -1)] → [(a, 1), (i, 0), (u, 0)]
+
+    Args:
+        phone_tone_list (list[tuple[str, int]]): 音素とアクセントのペアのリスト
+
+    Returns:
+        list[tuple[str, int]]: 修正された音素とアクセントのペアのリスト
+    """
+
+    tone_values = set(tone for _, tone in phone_tone_list)
+    if len(tone_values) == 1:
+        assert tone_values == {0}, tone_values
+        return phone_tone_list
+    elif len(tone_values) == 2:
+        if tone_values == {0, 1}:
+            return phone_tone_list
+        elif tone_values == {-1, 0}:
+            return [
+                (letter, 0 if tone == -1 else 1) for letter, tone in phone_tone_list
+            ]
+        else:
+            raise ValueError(f"Unexpected tone values: {tone_values}")
+    else:
+        raise ValueError(f"Unexpected tone values: {tone_values}")
+
+
+def __handle_long(sep_phonemes: list[list[str]]) -> list[list[str]]:
+    """
+    フレーズごとに分かれた音素（長音記号がそのまま）のリストのリスト `sep_phonemes` を受け取り、
+    その長音記号を処理して、音素のリストのリストを返す。
+    基本的には直前の音素を伸ばすが、直前の音素が母音でない場合もしくは冒頭の場合は、
+    おそらく長音記号とダッシュを勘違いしていると思われるので、ダッシュに対応する音素 `-` に変換する。
+
+    Args:
+        sep_phonemes (list[list[str]]): フレーズごとに分かれた音素のリストのリスト
+
+    Returns:
+        list[list[str]]: 長音記号を処理した音素のリストのリスト
+    """
+
+    # 母音の集合 (便宜上「ん」を含める)
+    VOWELS = {"a", "i", "u", "e", "o", "N"}
+
+    for i in range(len(sep_phonemes)):
+        if len(sep_phonemes[i]) == 0:
+            # 空白文字等でリストが空の場合
+            continue
+        if sep_phonemes[i][0] == "ー":
+            if i != 0:
+                prev_phoneme = sep_phonemes[i - 1][-1]
+                if prev_phoneme in VOWELS:
+                    # 母音と「ん」のあとの伸ばし棒なので、その母音に変換
+                    sep_phonemes[i][0] = sep_phonemes[i - 1][-1]
+                else:
+                    # 「。ーー」等おそらく予期しない長音記号
+                    # ダッシュの勘違いだと思われる
+                    sep_phonemes[i][0] = "-"
+            else:
+                # 冒頭に長音記号が来ていおり、これはダッシュの勘違いと思われる
+                sep_phonemes[i][0] = "-"
+        if "ー" in sep_phonemes[i]:
+            for j in range(len(sep_phonemes[i])):
+                if sep_phonemes[i][j] == "ー":
+                    sep_phonemes[i][j] = sep_phonemes[i][j - 1][-1]
+
+    return sep_phonemes
+
+
+def __kata_to_phoneme_list(text: str) -> list[str]:
+    """
+    原則カタカナの `text` を受け取り、それをそのままいじらずに音素記号のリストに変換。
+    注意点：
+    - punctuation かその繰り返しが来た場合、punctuation たちをそのままリストにして返す。
+    - 冒頭に続く「ー」はそのまま「ー」のままにする（`handle_long()` で処理される）
+    - 文中の「ー」は前の音素記号の最後の音素記号に変換される。
+    例：
+    `ーーソーナノカーー` → ["ー", "ー", "s", "o", "o", "n", "a", "n", "o", "k", "a", "a", "a"]
+    `?` → ["?"]
+    `!?!?!?!?!` → ["!", "?", "!", "?", "!", "?", "!", "?", "!"]
+
+    Args:
+        text (str): カタカナのテキスト
+
+    Returns:
+        list[str]: 音素記号のリスト
+    """
+
+    if set(text).issubset(set(PUNCTUATIONS)):
+        return list(text)
+    # `text` がカタカナ（`ー`含む）のみからなるかどうかをチェック
+    if re.fullmatch(r"[\u30A0-\u30FF]+", text) is None:
+        raise ValueError(f"Input must be katakana only: {text}")
+    sorted_keys = sorted(MORA_KATA_TO_MORA_PHONEMES.keys(), key=len, reverse=True)
+    pattern = "|".join(map(re.escape, sorted_keys))
+
+    def mora2phonemes(mora: str) -> str:
+        cosonant, vowel = MORA_KATA_TO_MORA_PHONEMES[mora]
+        if cosonant is None:
+            return f" {vowel}"
+        return f" {cosonant} {vowel}"
+
+    spaced_phonemes = re.sub(pattern, lambda m: mora2phonemes(m.group()), text)
+
+    # 長音記号「ー」の処理
+    long_pattern = r"(\w)(ー*)"
+    long_replacement = lambda m: m.group(1) + (" " + m.group(1)) * len(m.group(2))  # type: ignore
+    spaced_phonemes = re.sub(long_pattern, long_replacement, spaced_phonemes)
+
+    return spaced_phonemes.strip().split(" ")
+
+
+def __align_tones(
+    phones_with_punct: list[str],
+    phone_tone_list: list[tuple[str, int]]
+) -> list[tuple[str, int]]:
+    """
+    例: …私は、、そう思う。
+    phones_with_punct:
+        [".", ".", ".", "w", "a", "t", "a", "sh", "i", "w", "a", ",", ",", "s", "o", "o", "o", "m", "o", "u", "."]
+    phone_tone_list:
+        [("w", 0), ("a", 0), ("t", 1), ("a", 1), ("sh", 1), ("i", 1), ("w", 1), ("a", 1), ("_", 0), ("s", 0), ("o", 0), ("o", 1), ("o", 1), ("m", 1), ("o", 1), ("u", 0))]
+    Return:
+        [(".", 0), (".", 0), (".", 0), ("w", 0), ("a", 0), ("t", 1), ("a", 1), ("sh", 1), ("i", 1), ("w", 1), ("a", 1), (",", 0), (",", 0), ("s", 0), ("o", 0), ("o", 1), ("o", 1), ("m", 1), ("o", 1), ("u", 0), (".", 0)]
+
+    Args:
+        phones_with_punct (list[str]): punctuation を含む音素のリスト
+        phone_tone_list (list[tuple[str, int]]): punctuation を含まない音素とアクセントのペアのリスト
+
+    Returns:
+        list[tuple[str, int]]: punctuation を含む音素とアクセントのペアのリスト
+    """
+
+    result: list[tuple[str, int]] = []
+    tone_index = 0
+    for phone in phones_with_punct:
+        if tone_index >= len(phone_tone_list):
+            # 余った punctuation がある場合 → (punctuation, 0) を追加
+            result.append((phone, 0))
+        elif phone == phone_tone_list[tone_index][0]:
+            # phone_tone_list の現在の音素と一致する場合 → tone をそこから取得、(phone, tone) を追加
+            result.append((phone, phone_tone_list[tone_index][1]))
+            # 探す index を1つ進める
+            tone_index += 1
+        elif phone in PUNCTUATIONS:
+            # phone が punctuation の場合 → (phone, 0) を追加
+            result.append((phone, 0))
+        else:
+            logger.debug(f"phones: {phones_with_punct}")
+            logger.debug(f"phone_tone_list: {phone_tone_list}")
+            logger.debug(f"result: {result}")
+            logger.debug(f"tone_index: {tone_index}")
+            logger.debug(f"phone: {phone}")
+            raise ValueError(f"Unexpected phone: {phone}")
+
+    return result
+
+
+def __distribute_phone(n_phone: int, n_word: int) -> list[int]:
+    """
+    左から右に 1 ずつ振り分け、次にまた左から右に1ずつ増やし、というふうに、
+    音素の数 `n_phone` を単語の数 `n_word` に分配する。
+
+    Args:
+        n_phone (int): 音素の数
+        n_word (int): 単語の数
+
+    Returns:
+        list[int]: 単語ごとの音素の数のリスト
+    """
+
+    phones_per_word = [0] * n_word
+    for _ in range(n_phone):
+        min_tasks = min(phones_per_word)
+        min_index = phones_per_word.index(min_tasks)
+        phones_per_word[min_index] += 1
+
+    return phones_per_word
+
+
+class YomiError(Exception):
+    """
+    OpenJTalk で、読みが正しく取得できない箇所があるときに発生する例外。
+    基本的に「学習の前処理のテキスト処理時」には発生させ、そうでない場合は、
+    ignore_yomi_error=True にしておいて、この例外を発生させないようにする。
+    """
+
+    pass

style_bert_vits2/nlp/japanese/g2p_utils.py

@@ -0,0 +1,90 @@
+from style_bert_vits2.nlp.japanese.g2p import g2p
+from style_bert_vits2.nlp.japanese.mora_list import (
+    MORA_KATA_TO_MORA_PHONEMES,
+    MORA_PHONEMES_TO_MORA_KATA,
+)
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+def g2kata_tone(norm_text: str) -> list[tuple[str, int]]:
+    """
+    テキストからカタカナとアクセントのペアのリストを返す。
+    推論時のみに使われる関数のため、常に `raise_yomi_error=False` を指定して g2p() を呼ぶ仕様になっている。
+
+    Args:
+        norm_text: 正規化されたテキスト。
+
+    Returns:
+        カタカナと音高のリスト。
+    """
+
+    phones, tones, _ = g2p(norm_text, use_jp_extra=True, raise_yomi_error=False)
+    return phone_tone2kata_tone(list(zip(phones, tones)))
+
+
+def phone_tone2kata_tone(phone_tone: list[tuple[str, int]]) -> list[tuple[str, int]]:
+    """
+    phone_tone の phone 部分をカタカナに変換する。ただし最初と最後の ("_", 0) は無視する。
+
+    Args:
+        phone_tone: 音素と音高のリスト。
+
+    Returns:
+        カタカナと音高のリスト。
+    """
+
+    # 子音の集合
+    CONSONANTS = set([
+        consonant
+        for consonant, _ in MORA_KATA_TO_MORA_PHONEMES.values()
+        if consonant is not None
+    ])
+
+    phone_tone = phone_tone[1:]  # 最初の("_", 0)を無視
+    phones = [phone for phone, _ in phone_tone]
+    tones = [tone for _, tone in phone_tone]
+    result: list[tuple[str, int]] = []
+    current_mora = ""
+    for phone, next_phone, tone, next_tone in zip(phones, phones[1:], tones, tones[1:]):
+        # zip の関係で最後の ("_", 0) は無視されている
+        if phone in PUNCTUATIONS:
+            result.append((phone, tone))
+            continue
+        if phone in CONSONANTS:  # n以外の子音の場合
+            assert current_mora == "", f"Unexpected {phone} after {current_mora}"
+            assert tone == next_tone, f"Unexpected {phone} tone {tone} != {next_tone}"
+            current_mora = phone
+        else:
+            # phoneが母音もしくは「N」
+            current_mora += phone
+            result.append((MORA_PHONEMES_TO_MORA_KATA[current_mora], tone))
+            current_mora = ""
+
+    return result
+
+
+def kata_tone2phone_tone(kata_tone: list[tuple[str, int]]) -> list[tuple[str, int]]:
+    """
+    `phone_tone2kata_tone()` の逆の変換を行う。
+
+    Args:
+        kata_tone: カタカナと音高のリスト。
+
+    Returns:
+        音素と音高のリスト。
+    """
+
+    result: list[tuple[str, int]] = [("_", 0)]
+    for mora, tone in kata_tone:
+        if mora in PUNCTUATIONS:
+            result.append((mora, tone))
+        else:
+            consonant, vowel = MORA_KATA_TO_MORA_PHONEMES[mora]
+            if consonant is None:
+                result.append((vowel, tone))
+            else:
+                result.append((consonant, tone))
+                result.append((vowel, tone))
+    result.append(("_", 0))
+
+    return result

style_bert_vits2/nlp/japanese/mora_list.py

@@ -0,0 +1,236 @@
+"""
+以下のコードは VOICEVOX のソースコードからお借りし最低限の改造を行ったもの。
+https://github.com/VOICEVOX/voicevox_engine/blob/master/voicevox_engine/tts_pipeline/mora_list.py
+"""
+
+"""
+以下のモーラ対応表は OpenJTalk のソースコードから取得し、
+カタカナ表記とモーラが一対一対応するように改造した。
+ライセンス表記：
+-----------------------------------------------------------------
+          The Japanese TTS System "Open JTalk"
+          developed by HTS Working Group
+          http://open-jtalk.sourceforge.net/
+-----------------------------------------------------------------
+
+ Copyright (c) 2008-2014  Nagoya Institute of Technology
+                          Department of Computer Science
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the following
+conditions are met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following
+  disclaimer in the documentation and/or other materials provided
+  with the distribution.
+- Neither the name of the HTS working group nor the names of its
+  contributors may be used to endorse or promote products derived
+  from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+"""
+
+from typing import Optional
+
+
+# (カタカナ, 子音, 母音)の順。子音がない場合は None を入れる。
+# 但し「ン」と「ッ」は母音のみという扱いで、「ン」は「N」、「ッ」は「q」とする。
+# （元々「ッ」は「cl」）
+# また「デェ = dy e」は pyopenjtalk の出力（de e）と合わないため削除
+__MORA_LIST_MINIMUM: list[tuple[str, Optional[str], str]] = [
+    ("ヴォ", "v", "o"),
+    ("ヴェ", "v", "e"),
+    ("ヴィ", "v", "i"),
+    ("ヴァ", "v", "a"),
+    ("ヴ", "v", "u"),
+    ("ン", None, "N"),
+    ("ワ", "w", "a"),
+    ("ロ", "r", "o"),
+    ("レ", "r", "e"),
+    ("ル", "r", "u"),
+    ("リョ", "ry", "o"),
+    ("リュ", "ry", "u"),
+    ("リャ", "ry", "a"),
+    ("リェ", "ry", "e"),
+    ("リ", "r", "i"),
+    ("ラ", "r", "a"),
+    ("ヨ", "y", "o"),
+    ("ユ", "y", "u"),
+    ("ヤ", "y", "a"),
+    ("モ", "m", "o"),
+    ("メ", "m", "e"),
+    ("ム", "m", "u"),
+    ("ミョ", "my", "o"),
+    ("ミュ", "my", "u"),
+    ("ミャ", "my", "a"),
+    ("ミェ", "my", "e"),
+    ("ミ", "m", "i"),
+    ("マ", "m", "a"),
+    ("ポ", "p", "o"),
+    ("ボ", "b", "o"),
+    ("ホ", "h", "o"),
+    ("ペ", "p", "e"),
+    ("ベ", "b", "e"),
+    ("ヘ", "h", "e"),
+    ("プ", "p", "u"),
+    ("ブ", "b", "u"),
+    ("フォ", "f", "o"),
+    ("フェ", "f", "e"),
+    ("フィ", "f", "i"),
+    ("ファ", "f", "a"),
+    ("フ", "f", "u"),
+    ("ピョ", "py", "o"),
+    ("ピュ", "py", "u"),
+    ("ピャ", "py", "a"),
+    ("ピェ", "py", "e"),
+    ("ピ", "p", "i"),
+    ("ビョ", "by", "o"),
+    ("ビュ", "by", "u"),
+    ("ビャ", "by", "a"),
+    ("ビェ", "by", "e"),
+    ("ビ", "b", "i"),
+    ("ヒョ", "hy", "o"),
+    ("ヒュ", "hy", "u"),
+    ("ヒャ", "hy", "a"),
+    ("ヒェ", "hy", "e"),
+    ("ヒ", "h", "i"),
+    ("パ", "p", "a"),
+    ("バ", "b", "a"),
+    ("ハ", "h", "a"),
+    ("ノ", "n", "o"),
+    ("ネ", "n", "e"),
+    ("ヌ", "n", "u"),
+    ("ニョ", "ny", "o"),
+    ("ニュ", "ny", "u"),
+    ("ニャ", "ny", "a"),
+    ("ニェ", "ny", "e"),
+    ("ニ", "n", "i"),
+    ("ナ", "n", "a"),
+    ("ドゥ", "d", "u"),
+    ("ド", "d", "o"),
+    ("トゥ", "t", "u"),
+    ("ト", "t", "o"),
+    ("デョ", "dy", "o"),
+    ("デュ", "dy", "u"),
+    ("デャ", "dy", "a"),
+    # ("デェ", "dy", "e"),
+    ("ディ", "d", "i"),
+    ("デ", "d", "e"),
+    ("テョ", "ty", "o"),
+    ("テュ", "ty", "u"),
+    ("テャ", "ty", "a"),
+    ("ティ", "t", "i"),
+    ("テ", "t", "e"),
+    ("ツォ", "ts", "o"),
+    ("ツェ", "ts", "e"),
+    ("ツィ", "ts", "i"),
+    ("ツァ", "ts", "a"),
+    ("ツ", "ts", "u"),
+    ("ッ", None, "q"),  # 「cl」から「q」に変更
+    ("チョ", "ch", "o"),
+    ("チュ", "ch", "u"),
+    ("チャ", "ch", "a"),
+    ("チェ", "ch", "e"),
+    ("チ", "ch", "i"),
+    ("ダ", "d", "a"),
+    ("タ", "t", "a"),
+    ("ゾ", "z", "o"),
+    ("ソ", "s", "o"),
+    ("ゼ", "z", "e"),
+    ("セ", "s", "e"),
+    ("ズィ", "z", "i"),
+    ("ズ", "z", "u"),
+    ("スィ", "s", "i"),
+    ("ス", "s", "u"),
+    ("ジョ", "j", "o"),
+    ("ジュ", "j", "u"),
+    ("ジャ", "j", "a"),
+    ("ジェ", "j", "e"),
+    ("ジ", "j", "i"),
+    ("ショ", "sh", "o"),
+    ("シュ", "sh", "u"),
+    ("シャ", "sh", "a"),
+    ("シェ", "sh", "e"),
+    ("シ", "sh", "i"),
+    ("ザ", "z", "a"),
+    ("サ", "s", "a"),
+    ("ゴ", "g", "o"),
+    ("コ", "k", "o"),
+    ("ゲ", "g", "e"),
+    ("ケ", "k", "e"),
+    ("グヮ", "gw", "a"),
+    ("グ", "g", "u"),
+    ("クヮ", "kw", "a"),
+    ("ク", "k", "u"),
+    ("ギョ", "gy", "o"),
+    ("ギュ", "gy", "u"),
+    ("ギャ", "gy", "a"),
+    ("ギェ", "gy", "e"),
+    ("ギ", "g", "i"),
+    ("キョ", "ky", "o"),
+    ("キュ", "ky", "u"),
+    ("キャ", "ky", "a"),
+    ("キェ", "ky", "e"),
+    ("キ", "k", "i"),
+    ("ガ", "g", "a"),
+    ("カ", "k", "a"),
+    ("オ", None, "o"),
+    ("エ", None, "e"),
+    ("ウォ", "w", "o"),
+    ("ウェ", "w", "e"),
+    ("ウィ", "w", "i"),
+    ("ウ", None, "u"),
+    ("イェ", "y", "e"),
+    ("イ", None, "i"),
+    ("ア", None, "a"),
+]
+__MORA_LIST_ADDITIONAL: list[tuple[str, Optional[str], str]] = [
+    ("ヴョ", "by", "o"),
+    ("ヴュ", "by", "u"),
+    ("ヴャ", "by", "a"),
+    ("ヲ", None, "o"),
+    ("ヱ", None, "e"),
+    ("ヰ", None, "i"),
+    ("ヮ", "w", "a"),
+    ("ョ", "y", "o"),
+    ("ュ", "y", "u"),
+    ("ヅ", "z", "u"),
+    ("ヂ", "j", "i"),
+    ("ヶ", "k", "e"),
+    ("ャ", "y", "a"),
+    ("ォ", None, "o"),
+    ("ェ", None, "e"),
+    ("ゥ", None, "u"),
+    ("ィ", None, "i"),
+    ("ァ", None, "a"),
+]
+
+# モーラの音素表記とカタカナの対応表
+# 例: "vo" -> "ヴォ", "a" -> "ア"
+MORA_PHONEMES_TO_MORA_KATA: dict[str, str] = {
+    (consonant or "") + vowel: kana for [kana, consonant, vowel] in __MORA_LIST_MINIMUM
+}
+
+# モーラのカタカナ表記と音素の対応表
+# 例: "ヴォ" -> ("v", "o"), "ア" -> (None, "a")
+MORA_KATA_TO_MORA_PHONEMES: dict[str, tuple[Optional[str], str]] = {
+    kana: (consonant, vowel)
+    for [kana, consonant, vowel] in __MORA_LIST_MINIMUM + __MORA_LIST_ADDITIONAL
+}

style_bert_vits2/nlp/japanese/normalizer.py

@@ -0,0 +1,161 @@
+import re
+import unicodedata
+from num2words import num2words
+
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+def normalize_text(text: str) -> str:
+    """
+    日本語のテキストを正規化する。
+    結果は、ちょうど次の文字のみからなる：
+    - ひらがな
+    - カタカナ（全角長音記号「ー」が入る！）
+    - 漢字
+    - 半角アルファベット（大文字と小文字）
+    - ギリシャ文字
+    - `.` （句点`。`や`…`の一部や改行等）
+    - `,` （読点`、`や`:`等）
+    - `?` （疑問符`？`）
+    - `!` （感嘆符`！`）
+    - `'` （`「`や`」`等）
+    - `-` （`―`（ダッシュ、長音記号ではない）や`-`等）
+
+    注意点:
+    - 三点リーダー`…`は`...`に変換される（`なるほど…。` → `なるほど....`）
+    - 数字は漢字に変換される（`1,100円` → `千百円`、`52.34` → `五十二点三四`）
+    - 読点や疑問符等の位置・個数等は保持される（`??あ、、！！！` → `??あ,,!!!`）
+
+    Args:
+        text (str): 正規化するテキスト
+
+    Returns:
+        str: 正規化されたテキスト
+    """
+
+    res = unicodedata.normalize("NFKC", text)  # ここでアルファベットは半角になる
+    res = __convert_numbers_to_words(res)  # 「100円」→「百円」等
+    # 「～」と「~」も長音記号として扱う
+    res = res.replace("~", "ー")
+    res = res.replace("～", "ー")
+
+    res = replace_punctuation(res)  # 句読点等正規化、読めない文字を削除
+
+    # 結合文字の濁点・半濁点を削除
+    # 通常の「ば」等はそのままのこされる、「あ゛」は上で「あ゙」になりここで「あ」になる
+    res = res.replace("\u3099", "")  # 結合文字の濁点を削除、る゙ → る
+    res = res.replace("\u309A", "")  # 結合文字の半濁点を削除、な゚ → な
+    return res
+
+
+def replace_punctuation(text: str) -> str:
+    """
+    句読点等を「.」「,」「!」「?」「'」「-」に正規化し、OpenJTalk で読みが取得できるもののみ残す：
+    漢字・平仮名・カタカナ、アルファベット、ギリシャ文字
+
+    Args:
+        text (str): 正規化するテキスト
+
+    Returns:
+        str: 正規化されたテキスト
+    """
+
+    # 記号類の正規化変換マップ
+    REPLACE_MAP = {
+        "：": ",",
+        "；": ",",
+        "，": ",",
+        "。": ".",
+        "！": "!",
+        "？": "?",
+        "\n": ".",
+        "．": ".",
+        "…": "...",
+        "···": "...",
+        "・・・": "...",
+        "·": ",",
+        "・": ",",
+        "、": ",",
+        "$": ".",
+        "“": "'",
+        "”": "'",
+        '"': "'",
+        "‘": "'",
+        "’": "'",
+        "（": "'",
+        "）": "'",
+        "(": "'",
+        ")": "'",
+        "《": "'",
+        "》": "'",
+        "【": "'",
+        "】": "'",
+        "[": "'",
+        "]": "'",
+        # NFKC 正規化後のハイフン・ダッシュの変種を全て通常半角ハイフン - \u002d に変換
+        "\u02d7": "\u002d",  # ˗, Modifier Letter Minus Sign
+        "\u2010": "\u002d",  # ‐, Hyphen,
+        # "\u2011": "\u002d",  # ‑, Non-Breaking Hyphen, NFKC により \u2010 に変換される
+        "\u2012": "\u002d",  # ‒, Figure Dash
+        "\u2013": "\u002d",  # –, En Dash
+        "\u2014": "\u002d",  # —, Em Dash
+        "\u2015": "\u002d",  # ―, Horizontal Bar
+        "\u2043": "\u002d",  # ⁃, Hyphen Bullet
+        "\u2212": "\u002d",  # −, Minus Sign
+        "\u23af": "\u002d",  # ⎯, Horizontal Line Extension
+        "\u23e4": "\u002d",  # ⏤, Straightness
+        "\u2500": "\u002d",  # ─, Box Drawings Light Horizontal
+        "\u2501": "\u002d",  # ━, Box Drawings Heavy Horizontal
+        "\u2e3a": "\u002d",  # ⸺, Two-Em Dash
+        "\u2e3b": "\u002d",  # ⸻, Three-Em Dash
+        # "～": "-",  # これは長音記号「ー」として扱うよう変更
+        # "~": "-",  # これも長音記号「ー」として扱うよう変更
+        "「": "'",
+        "」": "'",
+    }
+
+    pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
+
+    # 句読点を辞書で置換
+    replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
+
+    replaced_text = re.sub(
+        # ↓ ひらがな、カタカナ、漢字
+        r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
+        # ↓ 半角アルファベット（大文字と小文字）
+        + r"\u0041-\u005A\u0061-\u007A"
+        # ↓ 全角アルファベット（大文字と小文字）
+        + r"\uFF21-\uFF3A\uFF41-\uFF5A"
+        # ↓ ギリシャ文字
+        + r"\u0370-\u03FF\u1F00-\u1FFF"
+        # ↓ "!", "?", "…", ",", ".", "'", "-", 但し`…`はすでに`...`に変換されている
+        + "".join(PUNCTUATIONS) + r"]+",
+        # 上述以外の文字を削除
+        "",
+        replaced_text,
+    )
+
+    return replaced_text
+
+
+def __convert_numbers_to_words(text: str) -> str:
+    """
+    記号や数字を日本語の文字表現に変換する。
+
+    Args:
+        text (str): 変換するテキスト
+
+    Returns:
+        str: 変換されたテキスト
+    """
+
+    NUMBER_WITH_SEPARATOR_PATTERN = re.compile("[0-9]{1,3}(,[0-9]{3})+")
+    CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
+    CURRENCY_PATTERN = re.compile(r"([$¥£€])([0-9.]*[0-9])")
+    NUMBER_PATTERN = re.compile(r"[0-9]+(\.[0-9]+)?")
+
+    res = NUMBER_WITH_SEPARATOR_PATTERN.sub(lambda m: m[0].replace(",", ""), text)
+    res = CURRENCY_PATTERN.sub(lambda m: m[2] + CURRENCY_MAP.get(m[1], m[1]), res)
+    res = NUMBER_PATTERN.sub(lambda m: num2words(m[0], lang="ja"), res)
+
+    return res

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__init__.py

@@ -0,0 +1,126 @@
+"""
+Run the pyopenjtalk worker in a separate process
+to avoid user dictionary access error
+"""
+
+from typing import Any, Optional
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_client import WorkerClient
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
+
+
+WORKER_CLIENT: Optional[WorkerClient] = None
+
+
+# pyopenjtalk interface
+# g2p(): not used
+
+
+def run_frontend(text: str) -> list[dict[str, Any]]:
+    assert WORKER_CLIENT
+    ret = WORKER_CLIENT.dispatch_pyopenjtalk("run_frontend", text)
+    assert isinstance(ret, list)
+    return ret
+
+
+def make_label(njd_features: Any) -> list[str]:
+    assert WORKER_CLIENT
+    ret = WORKER_CLIENT.dispatch_pyopenjtalk("make_label", njd_features)
+    assert isinstance(ret, list)
+    return ret
+
+
+def mecab_dict_index(path: str, out_path: str, dn_mecab: Optional[str] = None):
+    assert WORKER_CLIENT
+    WORKER_CLIENT.dispatch_pyopenjtalk("mecab_dict_index", path, out_path, dn_mecab)
+
+
+def update_global_jtalk_with_user_dict(path: str):
+    assert WORKER_CLIENT
+    WORKER_CLIENT.dispatch_pyopenjtalk("update_global_jtalk_with_user_dict", path)
+
+
+def unset_user_dict():
+    assert WORKER_CLIENT
+    WORKER_CLIENT.dispatch_pyopenjtalk("unset_user_dict")
+
+
+# initialize module when imported
+
+
+def initialize(port: int = WORKER_PORT) -> None:
+    import time
+    import socket
+    import sys
+    import atexit
+    import signal
+
+    logger.debug("initialize")
+    global WORKER_CLIENT
+    if WORKER_CLIENT:
+        return
+
+    client = None
+    try:
+        client = WorkerClient(port)
+    except (socket.timeout, socket.error):
+        logger.debug("try starting pyopenjtalk worker server")
+        import os
+        import subprocess
+
+        worker_pkg_path = os.path.relpath(
+            os.path.dirname(__file__), os.getcwd()
+        ).replace(os.sep, ".")
+        args = [sys.executable, "-m", worker_pkg_path, "--port", str(port)]
+        # new session, new process group
+        if sys.platform.startswith("win"):
+            cf = subprocess.CREATE_NEW_CONSOLE | subprocess.CREATE_NEW_PROCESS_GROUP  # type: ignore
+            si = subprocess.STARTUPINFO()  # type: ignore
+            si.dwFlags |= subprocess.STARTF_USESHOWWINDOW  # type: ignore
+            si.wShowWindow = subprocess.SW_HIDE  # type: ignore
+            subprocess.Popen(args, creationflags=cf, startupinfo=si)
+        else:
+            # align with Windows behavior
+            # start_new_session is same as specifying setsid in preexec_fn
+            subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)  # type: ignore
+
+        # wait until server listening
+        count = 0
+        while True:
+            try:
+                client = WorkerClient(port)
+                break
+            except socket.error:
+                time.sleep(1)
+                count += 1
+                # 10: max number of retries
+                if count == 10:
+                    raise TimeoutError("サーバーに接続できませんでした")
+
+    WORKER_CLIENT = client
+    atexit.register(terminate)
+
+    # when the process is killed
+    def signal_handler(signum: int, frame: Any):
+        terminate()
+
+    signal.signal(signal.SIGTERM, signal_handler)
+
+
+# top-level declaration
+def terminate() -> None:
+    logger.debug("terminate")
+    global WORKER_CLIENT
+    if not WORKER_CLIENT:
+        return
+
+    # repare for unexpected errors
+    try:
+        if WORKER_CLIENT.status() == 1:
+            WORKER_CLIENT.quit_server()
+    except Exception as e:
+        logger.error(e)
+
+    WORKER_CLIENT.close()
+    WORKER_CLIENT = None

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__main__.py

@@ -0,0 +1,16 @@
+import argparse
+
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_server import WorkerServer
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=WORKER_PORT)
+    args = parser.parse_args()
+    server = WorkerServer()
+    server.start_server(port=args.port)
+
+
+if __name__ == "__main__":
+    main()

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_client.py

@@ -0,0 +1,63 @@
+import socket
+from typing import Any, cast
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import RequestType, receive_data, send_data
+
+
+class WorkerClient:
+    """ pyopenjtalk worker client """
+
+
+    def __init__(self, port: int) -> None:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        # 5: timeout
+        sock.settimeout(5)
+        sock.connect((socket.gethostname(), port))
+        self.sock = sock
+
+
+    def __enter__(self) -> "WorkerClient":
+        return self
+
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.close()
+
+
+    def close(self) -> None:
+        self.sock.close()
+
+
+    def dispatch_pyopenjtalk(self, func: str, *args: Any, **kwargs: Any) -> Any:
+        data = {
+            "request-type": RequestType.PYOPENJTALK,
+            "func": func,
+            "args": args,
+            "kwargs": kwargs,
+        }
+        logger.trace(f"client sends request: {data}")
+        send_data(self.sock, data)
+        logger.trace("client sent request successfully")
+        response = receive_data(self.sock)
+        logger.trace(f"client received response: {response}")
+        return response.get("return")
+
+
+    def status(self) -> int:
+        data = {"request-type": RequestType.STATUS}
+        logger.trace(f"client sends request: {data}")
+        send_data(self.sock, data)
+        logger.trace("client sent request successfully")
+        response = receive_data(self.sock)
+        logger.trace(f"client received response: {response}")
+        return cast(int, response.get("client-count"))
+
+
+    def quit_server(self) -> None:
+        data = {"request-type": RequestType.QUIT_SERVER}
+        logger.trace(f"client sends request: {data}")
+        send_data(self.sock, data)
+        logger.trace("client sent request successfully")
+        response = receive_data(self.sock)
+        logger.trace(f"client received response: {response}")

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_common.py

@@ -0,0 +1,45 @@
+import json
+import socket
+from enum import IntEnum, auto
+from typing import Any, Final
+
+
+WORKER_PORT: Final[int] = 7861
+HEADER_SIZE: Final[int] = 4
+
+
+class RequestType(IntEnum):
+    STATUS = auto()
+    QUIT_SERVER = auto()
+    PYOPENJTALK = auto()
+
+
+class ConnectionClosedException(Exception):
+    pass
+
+
+# socket communication
+
+
+def send_data(sock: socket.socket, data: dict[str, Any]):
+    json_data = json.dumps(data).encode()
+    header = len(json_data).to_bytes(HEADER_SIZE, byteorder="big")
+    sock.sendall(header + json_data)
+
+
+def __receive_until(sock: socket.socket, size: int):
+    data = b""
+    while len(data) < size:
+        part = sock.recv(size - len(data))
+        if part == b"":
+            raise ConnectionClosedException("接続が閉じられました")
+        data += part
+
+    return data
+
+
+def receive_data(sock: socket.socket) -> dict[str, Any]:
+    header = __receive_until(sock, HEADER_SIZE)
+    data_length = int.from_bytes(header, byteorder="big")
+    body = __receive_until(sock, data_length)
+    return json.loads(body.decode())

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py

@@ -0,0 +1,126 @@
+import select
+import socket
+import time
+from typing import Any, cast
+
+import pyopenjtalk
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import (
+    ConnectionClosedException,
+    RequestType,
+    receive_data,
+    send_data,
+)
+
+
+# To make it as fast as possible
+# Probably faster than calling getattr every time
+__PYOPENJTALK_FUNC_DICT = {
+    "run_frontend": pyopenjtalk.run_frontend,
+    "make_label": pyopenjtalk.make_label,
+    "mecab_dict_index": pyopenjtalk.mecab_dict_index,
+    "update_global_jtalk_with_user_dict": pyopenjtalk.update_global_jtalk_with_user_dict,
+    "unset_user_dict": pyopenjtalk.unset_user_dict,
+}
+
+
+class WorkerServer:
+    """ pyopenjtalk worker server """
+
+
+    def __init__(self) -> None:
+        self.client_count: int = 0
+        self.quit: bool = False
+
+
+    def handle_request(self, request: dict[str, Any]) -> dict[str, Any]:
+        request_type = None
+        try:
+            request_type = RequestType(cast(int, request.get("request-type")))
+        except Exception:
+            return {
+                "success": False,
+                "reason": "request-type is invalid",
+            }
+
+        response: dict[str, Any] = {}
+        if request_type:
+            if request_type == RequestType.STATUS:
+                response = {
+                    "success": True,
+                    "client-count": self.client_count,
+                }
+            elif request_type == RequestType.QUIT_SERVER:
+                self.quit = True
+                response = {"success": True}
+            elif request_type == RequestType.PYOPENJTALK:
+                func_name = request.get("func")
+                assert isinstance(func_name, str)
+                func = __PYOPENJTALK_FUNC_DICT[func_name]
+                args = request.get("args")
+                kwargs = request.get("kwargs")
+                assert isinstance(args, list)
+                assert isinstance(kwargs, dict)
+                ret = func(*args, **kwargs)
+                response = {"success": True, "return": ret}
+            else:
+                # NOT REACHED
+                response = request
+
+        return response
+
+
+    def start_server(self, port: int, no_client_timeout: int = 30) -> None:
+        logger.info("start pyopenjtalk worker server")
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket:
+            server_socket.bind((socket.gethostname(), port))
+            server_socket.listen()
+            sockets = [server_socket]
+            no_client_since = time.time()
+            while True:
+                if self.client_count == 0:
+                    if no_client_since is None:
+                        no_client_since = time.time()
+                    elif (time.time() - no_client_since) > no_client_timeout:
+                        logger.info("quit because there is no client")
+                        return
+                else:
+                    no_client_since = None
+
+                ready_sockets, _, _ = select.select(sockets, [], [], 0.1)
+                for sock in ready_sockets:
+                    if sock is server_socket:
+                        logger.info("new client connected")
+                        client_socket, _ = server_socket.accept()
+                        sockets.append(client_socket)
+                        self.client_count += 1
+                    else:
+                        # client
+                        try:
+                            request = receive_data(sock)
+                        except Exception as e:
+                            sock.close()
+                            sockets.remove(sock)
+                            self.client_count -= 1
+                            # unexpected disconnections
+                            if not isinstance(e, ConnectionClosedException):
+                                logger.error(e)
+
+                            logger.info("close connection")
+                            continue
+
+                        logger.trace(f"server received request: {request}")
+
+                        response = self.handle_request(request)
+                        logger.trace(f"server sends response: {response}")
+                        try:
+                            send_data(sock, response)
+                            logger.trace("server sent response successfully")
+                        except Exception:
+                            logger.warning(
+                                "an exception occurred during sending responce"
+                            )
+                        if self.quit:
+                            logger.info("quit pyopenjtalk worker server")
+                            return

style_bert_vits2/nlp/japanese/user_dict/README.md

@@ -0,0 +1,27 @@
+
+## ユーザー辞書関連のコードについて
+
+このフォルダに含まれるユーザー辞書関連のコードは、[VOICEVOX ENGINE](https://github.com/VOICEVOX/voicevox_engine) プロジェクトのコードを改変したものを使用しています。  
+VOICEVOX プロジェクトのチームに深く感謝し、その貢献を尊重します。
+
+### 元のコード
+
+- [voicevox_engine/user_dict/](https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict)
+- [voicevox_engine/model.py](https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207)
+
+### 改変の詳細
+
+- ファイル名の書き換えおよびそれに伴う import 文の書き換え。
+- VOICEVOX 固有の部分をコメントアウト。
+- mutex を使用している部分をコメントアウト。
+- 参照している pyopenjtalk の違いによるメソッド名の書き換え。
+- UserDictWord の mora_count のデフォルト値を None に指定。
+- `model.py` のうち、必要な Pydantic モデルのみを抽出。
+
+### ライセンス
+
+元の VOICEVOX ENGINE のリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。  
+当プロジェクトにおけるこのモジュールも LGPL ライセンスの下にあります。
+
+詳細については、プロジェクトのルートディレクトリにある [LGPL_LICENSE](/LGPL_LICENSE) ファイルをご参照ください。  
+また、元の VOICEVOX ENGINE プロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE) をご覧ください。

style_bert_vits2/nlp/japanese/user_dict/__init__.py

@@ -0,0 +1,464 @@
+"""
+このファイルは、VOICEVOX プロジェクトの VOICEVOX ENGINE からお借りしています。
+引用元: https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/user_dict.py
+ライセンス: LGPL-3.0
+詳しくは、このファイルと同じフォルダにある README.md を参照してください。
+"""
+
+import json
+import sys
+import traceback
+from pathlib import Path
+from typing import Dict, List, Optional
+from uuid import UUID, uuid4
+
+import numpy as np
+from fastapi import HTTPException
+
+from style_bert_vits2.constants import DEFAULT_USER_DICT_DIR
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.user_dict.word_model import UserDictWord, WordTypes
+from style_bert_vits2.nlp.japanese.user_dict.part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data
+
+pyopenjtalk.initialize()
+
+# root_dir = engine_root()
+# save_dir = get_save_dir()
+
+# if not save_dir.is_dir():
+#     save_dir.mkdir(parents=True)
+
+default_dict_path = DEFAULT_USER_DICT_DIR / "default.csv"  # VOICEVOXデフォルト辞書ファイルのパス
+user_dict_path = DEFAULT_USER_DICT_DIR / "user_dict.json"  # ユーザー辞書ファイルのパス
+compiled_dict_path = DEFAULT_USER_DICT_DIR / "user.dic"  # コンパイル済み辞書ファイルのパス
+
+
+# # 同時書き込みの制御
+# mutex_user_dict = threading.Lock()
+# mutex_openjtalk_dict = threading.Lock()
+
+
+# @mutex_wrapper(mutex_user_dict)
+def _write_to_json(user_dict: Dict[str, UserDictWord], user_dict_path: Path) -> None:
+    """
+    ユーザー辞書ファイルへのユーザー辞書データ書き込み
+    Parameters
+    ----------
+    user_dict : Dict[str, UserDictWord]
+        ユーザー辞書データ
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    """
+    converted_user_dict = {}
+    for word_uuid, word in user_dict.items():
+        word_dict = word.model_dump()
+        word_dict["cost"] = _priority2cost(
+            word_dict["context_id"], word_dict["priority"]
+        )
+        del word_dict["priority"]
+        converted_user_dict[word_uuid] = word_dict
+    # 予めjsonに変換できることを確かめる
+    user_dict_json = json.dumps(converted_user_dict, ensure_ascii=False)
+
+    # ユーザー辞書ファイルへの書き込み
+    user_dict_path.write_text(user_dict_json, encoding="utf-8")
+
+
+# @mutex_wrapper(mutex_openjtalk_dict)
+def update_dict(
+    default_dict_path: Path = default_dict_path,
+    user_dict_path: Path = user_dict_path,
+    compiled_dict_path: Path = compiled_dict_path,
+) -> None:
+    """
+    辞書の更新
+    Parameters
+    ----------
+    default_dict_path : Path
+        デフォルト辞書ファイルのパス
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    compiled_dict_path : Path
+        コンパイル済み辞書ファイルのパス
+    """
+    random_string = uuid4()
+    tmp_csv_path = compiled_dict_path.with_suffix(
+        f".dict_csv-{random_string}.tmp"
+    )  # csv形式辞書データの一時保存ファイル
+    tmp_compiled_path = compiled_dict_path.with_suffix(
+        f".dict_compiled-{random_string}.tmp"
+    )  # コンパイル済み辞書データの一時保存ファイル
+
+    try:
+        # 辞書.csvを作成
+        csv_text = ""
+
+        # デフォルト辞書データの追加
+        if not default_dict_path.is_file():
+            print("Warning: Cannot find default dictionary.", file=sys.stderr)
+            return
+        default_dict = default_dict_path.read_text(encoding="utf-8")
+        if default_dict == default_dict.rstrip():
+            default_dict += "\n"
+        csv_text += default_dict
+
+        # ユーザー辞書データの追加
+        user_dict = read_dict(user_dict_path=user_dict_path)
+        for word_uuid in user_dict:
+            word = user_dict[word_uuid]
+            csv_text += (
+                "{surface},{context_id},{context_id},{cost},{part_of_speech},"
+                + "{part_of_speech_detail_1},{part_of_speech_detail_2},"
+                + "{part_of_speech_detail_3},{inflectional_type},"
+                + "{inflectional_form},{stem},{yomi},{pronunciation},"
+                + "{accent_type}/{mora_count},{accent_associative_rule}\n"
+            ).format(
+                surface=word.surface,
+                context_id=word.context_id,
+                cost=_priority2cost(word.context_id, word.priority),
+                part_of_speech=word.part_of_speech,
+                part_of_speech_detail_1=word.part_of_speech_detail_1,
+                part_of_speech_detail_2=word.part_of_speech_detail_2,
+                part_of_speech_detail_3=word.part_of_speech_detail_3,
+                inflectional_type=word.inflectional_type,
+                inflectional_form=word.inflectional_form,
+                stem=word.stem,
+                yomi=word.yomi,
+                pronunciation=word.pronunciation,
+                accent_type=word.accent_type,
+                mora_count=word.mora_count,
+                accent_associative_rule=word.accent_associative_rule,
+            )
+        # 辞書データを辞書.csv へ一時保存
+        tmp_csv_path.write_text(csv_text, encoding="utf-8")
+
+        # 辞書.csvをOpenJTalk用にコンパイル
+        # pyopenjtalk.create_user_dict(str(tmp_csv_path), str(tmp_compiled_path))
+        pyopenjtalk.mecab_dict_index(str(tmp_csv_path), str(tmp_compiled_path))
+        if not tmp_compiled_path.is_file():
+            raise RuntimeError("辞書のコンパイル時にエラーが発生しました。")
+
+        # コンパイル済み辞書の置き換え・読み込み
+        pyopenjtalk.unset_user_dict()
+        tmp_compiled_path.replace(compiled_dict_path)
+        if compiled_dict_path.is_file():
+            # pyopenjtalk.set_user_dict(str(compiled_dict_path.resolve(strict=True)))
+            pyopenjtalk.update_global_jtalk_with_user_dict(str(compiled_dict_path))
+
+    except Exception as e:
+        print("Error: Failed to update dictionary.", file=sys.stderr)
+        traceback.print_exc(file=sys.stderr)
+        raise e
+
+    finally:
+        # 後処理
+        if tmp_csv_path.exists():
+            tmp_csv_path.unlink()
+        if tmp_compiled_path.exists():
+            tmp_compiled_path.unlink()
+
+
+# @mutex_wrapper(mutex_user_dict)
+def read_dict(user_dict_path: Path = user_dict_path) -> Dict[str, UserDictWord]:
+    """
+    ユーザー辞書の読み出し
+    Parameters
+    ----------
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    Returns
+    -------
+    result : Dict[str, UserDictWord]
+        ユーザー辞書
+    """
+    # 指定ユーザー辞書が存在しない場合、空辞書を返す
+    if not user_dict_path.is_file():
+        return {}
+
+    with user_dict_path.open(encoding="utf-8") as f:
+        result: Dict[str, UserDictWord] = {}
+        for word_uuid, word in json.load(f).items():
+            # cost2priorityで変換を行う際にcontext_idが必要となるが、
+            # 0.12以前の辞書は、context_idがハードコーディングされていたためにユーザー辞書内に保管されていない
+            # ハードコーディングされていたcontext_idは固有名詞を意味するものなので、固有名詞のcontext_idを補完する
+            if word.get("context_id") is None:
+                word["context_id"] = part_of_speech_data[
+                    WordTypes.PROPER_NOUN
+                ].context_id
+            word["priority"] = _cost2priority(word["context_id"], word["cost"])
+            del word["cost"]
+            result[str(UUID(word_uuid))] = UserDictWord(**word)
+
+    return result
+
+
+def _create_word(
+    surface: str,
+    pronunciation: str,
+    accent_type: int,
+    word_type: Optional[WordTypes] = None,
+    priority: Optional[int] = None,
+) -> UserDictWord:
+    """
+    単語オブジェクトの生成
+    Parameters
+    ----------
+    surface : str
+        単語情報
+    pronunciation : str
+        単語情報
+    accent_type : int
+        単語情報
+    word_type : Optional[WordTypes]
+        品詞
+    priority : Optional[int]
+        優先度
+    Returns
+    -------
+    : UserDictWord
+        単語オブジェクト
+    """
+    if word_type is None:
+        word_type = WordTypes.PROPER_NOUN
+    if word_type not in part_of_speech_data.keys():
+        raise HTTPException(status_code=422, detail="不明な品詞です")
+    if priority is None:
+        priority = 5
+    if not MIN_PRIORITY <= priority <= MAX_PRIORITY:
+        raise HTTPException(status_code=422, detail="優先度の値が無効です")
+    pos_detail = part_of_speech_data[word_type]
+    return UserDictWord(
+        surface=surface,
+        context_id=pos_detail.context_id,
+        priority=priority,
+        part_of_speech=pos_detail.part_of_speech,
+        part_of_speech_detail_1=pos_detail.part_of_speech_detail_1,
+        part_of_speech_detail_2=pos_detail.part_of_speech_detail_2,
+        part_of_speech_detail_3=pos_detail.part_of_speech_detail_3,
+        inflectional_type="*",
+        inflectional_form="*",
+        stem="*",
+        yomi=pronunciation,
+        pronunciation=pronunciation,
+        accent_type=accent_type,
+        accent_associative_rule="*",
+    )
+
+
+def apply_word(
+    surface: str,
+    pronunciation: str,
+    accent_type: int,
+    word_type: Optional[WordTypes] = None,
+    priority: Optional[int] = None,
+    user_dict_path: Path = user_dict_path,
+    compiled_dict_path: Path = compiled_dict_path,
+) -> str:
+    """
+    新規単語の追加
+    Parameters
+    ----------
+    surface : str
+        単語情報
+    pronunciation : str
+        単語情報
+    accent_type : int
+        単語情報
+    word_type : Optional[WordTypes]
+        品詞
+    priority : Optional[int]
+        優先度
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    compiled_dict_path : Path
+        コンパイル済み辞書ファイルのパス
+    Returns
+    -------
+    word_uuid : UserDictWord
+        追加された単語に発行されたUUID
+    """
+    # 新規単語の追加による辞書データの更新
+    word = _create_word(
+        surface=surface,
+        pronunciation=pronunciation,
+        accent_type=accent_type,
+        word_type=word_type,
+        priority=priority,
+    )
+    user_dict = read_dict(user_dict_path=user_dict_path)
+    word_uuid = str(uuid4())
+    user_dict[word_uuid] = word
+
+    # 更新された辞書データの保存と適用
+    _write_to_json(user_dict, user_dict_path)
+    update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path)
+
+    return word_uuid
+
+
+def rewrite_word(
+    word_uuid: str,
+    surface: str,
+    pronunciation: str,
+    accent_type: int,
+    word_type: Optional[WordTypes] = None,
+    priority: Optional[int] = None,
+    user_dict_path: Path = user_dict_path,
+    compiled_dict_path: Path = compiled_dict_path,
+) -> None:
+    """
+    既存単語の上書き更新
+    Parameters
+    ----------
+    word_uuid : str
+        単語UUID
+    surface : str
+        単語情報
+    pronunciation : str
+        単語情報
+    accent_type : int
+        単語情報
+    word_type : Optional[WordTypes]
+        品詞
+    priority : Optional[int]
+        優先度
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    compiled_dict_path : Path
+        コンパイル済み辞書ファイルのパス
+    """
+    word = _create_word(
+        surface=surface,
+        pronunciation=pronunciation,
+        accent_type=accent_type,
+        word_type=word_type,
+        priority=priority,
+    )
+
+    # 既存単語の上書きによる辞書データの更新
+    user_dict = read_dict(user_dict_path=user_dict_path)
+    if word_uuid not in user_dict:
+        raise HTTPException(
+            status_code=422, detail="UUIDに該当するワードが見つかりませんでした"
+        )
+    user_dict[word_uuid] = word
+
+    # 更新された辞書データの保存と適用
+    _write_to_json(user_dict, user_dict_path)
+    update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path)
+
+
+def delete_word(
+    word_uuid: str,
+    user_dict_path: Path = user_dict_path,
+    compiled_dict_path: Path = compiled_dict_path,
+) -> None:
+    """
+    単語の削除
+    Parameters
+    ----------
+    word_uuid : str
+        単語UUID
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    compiled_dict_path : Path
+        コンパイル済み辞書ファイルのパス
+    """
+    # 既存単語の削除による辞書データの更新
+    user_dict = read_dict(user_dict_path=user_dict_path)
+    if word_uuid not in user_dict:
+        raise HTTPException(
+            status_code=422, detail="IDに該当するワードが見つかりませんでした"
+        )
+    del user_dict[word_uuid]
+
+    # 更新された辞書データの保存と適用
+    _write_to_json(user_dict, user_dict_path)
+    update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path)
+
+
+def import_user_dict(
+    dict_data: Dict[str, UserDictWord],
+    override: bool = False,
+    user_dict_path: Path = user_dict_path,
+    default_dict_path: Path = default_dict_path,
+    compiled_dict_path: Path = compiled_dict_path,
+) -> None:
+    """
+    ユーザー辞書のインポート
+    Parameters
+    ----------
+    dict_data : Dict[str, UserDictWord]
+        インポートするユーザー辞書のデータ
+    override : bool
+        重複したエントリがあった場合、上書きするかどうか
+    user_dict_path : Path
+        ユーザー辞書ファイルのパス
+    default_dict_path : Path
+        デフォルト辞書ファイルのパス
+    compiled_dict_path : Path
+        コンパイル済み辞書ファイルのパス
+    """
+    # インポートする辞書データのバリデーション
+    for word_uuid, word in dict_data.items():
+        UUID(word_uuid)
+        assert isinstance(word, UserDictWord)
+        for pos_detail in part_of_speech_data.values():
+            if word.context_id == pos_detail.context_id:
+                assert word.part_of_speech == pos_detail.part_of_speech
+                assert (
+                    word.part_of_speech_detail_1 == pos_detail.part_of_speech_detail_1
+                )
+                assert (
+                    word.part_of_speech_detail_2 == pos_detail.part_of_speech_detail_2
+                )
+                assert (
+                    word.part_of_speech_detail_3 == pos_detail.part_of_speech_detail_3
+                )
+                assert (
+                    word.accent_associative_rule in pos_detail.accent_associative_rules
+                )
+                break
+        else:
+            raise ValueError("対応していない品詞です")
+
+    # 既存辞書の読み出し
+    old_dict = read_dict(user_dict_path=user_dict_path)
+
+    # 辞書データの更新
+    # 重複エントリの上書き
+    if override:
+        new_dict = {**old_dict, **dict_data}
+    # 重複エントリの保持
+    else:
+        new_dict = {**dict_data, **old_dict}
+
+    # 更新された辞書データの保存と適用
+    _write_to_json(user_dict=new_dict, user_dict_path=user_dict_path)
+    update_dict(
+        default_dict_path=default_dict_path,
+        user_dict_path=user_dict_path,
+        compiled_dict_path=compiled_dict_path,
+    )
+
+
+def _search_cost_candidates(context_id: int) -> List[int]:
+    for value in part_of_speech_data.values():
+        if value.context_id == context_id:
+            return value.cost_candidates
+    raise HTTPException(status_code=422, detail="品詞IDが不正です")
+
+
+def _cost2priority(context_id: int, cost: int) -> int:
+    assert -32768 <= cost <= 32767
+    cost_candidates = _search_cost_candidates(context_id)
+    # cost_candidatesの中にある値で最も近い値を元にpriorityを返す
+    # 参考: https://qiita.com/Krypf/items/2eada91c37161d17621d
+    # この関数とpriority2cost関数によって、辞書ファイルのcostを操作しても最も近いpriorityのcostに上書きされる
+    return MAX_PRIORITY - np.argmin(np.abs(np.array(cost_candidates) - cost)).item()
+
+
+def _priority2cost(context_id: int, priority: int) -> int:
+    assert MIN_PRIORITY <= priority <= MAX_PRIORITY
+    cost_candidates = _search_cost_candidates(context_id)
+    return cost_candidates[MAX_PRIORITY - priority]

style_bert_vits2/nlp/japanese/user_dict/part_of_speech_data.py

@@ -0,0 +1,151 @@
+"""
+このファイルは、VOICEVOX プロジェクトの VOICEVOX ENGINE からお借りしています。
+引用元: https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/part_of_speech_data.py
+ライセンス: LGPL-3.0
+詳しくは、このファイルと同じフォルダにある README.md を参照してください。
+"""
+
+from typing import Dict
+
+from style_bert_vits2.nlp.japanese.user_dict.word_model import (
+    USER_DICT_MAX_PRIORITY,
+    USER_DICT_MIN_PRIORITY,
+    PartOfSpeechDetail,
+    WordTypes,
+)
+
+MIN_PRIORITY = USER_DICT_MIN_PRIORITY
+MAX_PRIORITY = USER_DICT_MAX_PRIORITY
+
+part_of_speech_data: Dict[WordTypes, PartOfSpeechDetail] = {
+    WordTypes.PROPER_NOUN: PartOfSpeechDetail(
+        part_of_speech="名詞",
+        part_of_speech_detail_1="固有名詞",
+        part_of_speech_detail_2="一般",
+        part_of_speech_detail_3="*",
+        context_id=1348,
+        cost_candidates=[
+            -988,
+            3488,
+            4768,
+            6048,
+            7328,
+            8609,
+            8734,
+            8859,
+            8984,
+            9110,
+            14176,
+        ],
+        accent_associative_rules=[
+            "*",
+            "C1",
+            "C2",
+            "C3",
+            "C4",
+            "C5",
+        ],
+    ),
+    WordTypes.COMMON_NOUN: PartOfSpeechDetail(
+        part_of_speech="名詞",
+        part_of_speech_detail_1="一般",
+        part_of_speech_detail_2="*",
+        part_of_speech_detail_3="*",
+        context_id=1345,
+        cost_candidates=[
+            -4445,
+            49,
+            1473,
+            2897,
+            4321,
+            5746,
+            6554,
+            7362,
+            8170,
+            8979,
+            15001,
+        ],
+        accent_associative_rules=[
+            "*",
+            "C1",
+            "C2",
+            "C3",
+            "C4",
+            "C5",
+        ],
+    ),
+    WordTypes.VERB: PartOfSpeechDetail(
+        part_of_speech="動詞",
+        part_of_speech_detail_1="自立",
+        part_of_speech_detail_2="*",
+        part_of_speech_detail_3="*",
+        context_id=642,
+        cost_candidates=[
+            3100,
+            6160,
+            6360,
+            6561,
+            6761,
+            6962,
+            7414,
+            7866,
+            8318,
+            8771,
+            13433,
+        ],
+        accent_associative_rules=[
+            "*",
+        ],
+    ),
+    WordTypes.ADJECTIVE: PartOfSpeechDetail(
+        part_of_speech="形容詞",
+        part_of_speech_detail_1="自立",
+        part_of_speech_detail_2="*",
+        part_of_speech_detail_3="*",
+        context_id=20,
+        cost_candidates=[
+            1527,
+            3266,
+            3561,
+            3857,
+            4153,
+            4449,
+            5149,
+            5849,
+            6549,
+            7250,
+            10001,
+        ],
+        accent_associative_rules=[
+            "*",
+        ],
+    ),
+    WordTypes.SUFFIX: PartOfSpeechDetail(
+        part_of_speech="名詞",
+        part_of_speech_detail_1="接尾",
+        part_of_speech_detail_2="一般",
+        part_of_speech_detail_3="*",
+        context_id=1358,
+        cost_candidates=[
+            4399,
+            5373,
+            6041,
+            6710,
+            7378,
+            8047,
+            9440,
+            10834,
+            12228,
+            13622,
+            15847,
+        ],
+        accent_associative_rules=[
+            "*",
+            "C1",
+            "C2",
+            "C3",
+            "C4",
+            "C5",
+        ],
+    ),
+}

style_bert_vits2/nlp/japanese/user_dict/word_model.py

@@ -0,0 +1,131 @@
+"""
+このファイルは、VOICEVOX プロジェクトの VOICEVOX ENGINE からお借りしています。
+引用元: https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207
+ライセンス: LGPL-3.0
+詳しくは、このファイルと同じフォルダにある README.md を参照してください。
+"""
+
+from enum import Enum
+from re import findall, fullmatch
+from typing import List, Optional
+
+from pydantic import BaseModel, Field, validator
+
+USER_DICT_MIN_PRIORITY = 0
+USER_DICT_MAX_PRIORITY = 10
+
+
+class UserDictWord(BaseModel):
+    """
+    辞書のコンパイルに使われる情報
+    """
+
+    surface: str = Field(title="表層形")
+    priority: int = Field(
+        title="優先度", ge=USER_DICT_MIN_PRIORITY, le=USER_DICT_MAX_PRIORITY
+    )
+    context_id: int = Field(title="文脈ID", default=1348)
+    part_of_speech: str = Field(title="品詞")
+    part_of_speech_detail_1: str = Field(title="品詞細分類1")
+    part_of_speech_detail_2: str = Field(title="品詞細分類2")
+    part_of_speech_detail_3: str = Field(title="品詞細分類3")
+    inflectional_type: str = Field(title="活用型")
+    inflectional_form: str = Field(title="活用形")
+    stem: str = Field(title="原形")
+    yomi: str = Field(title="読み")
+    pronunciation: str = Field(title="発音")
+    accent_type: int = Field(title="アクセント型")
+    mora_count: Optional[int] = Field(title="モーラ数", default=None)
+    accent_associative_rule: str = Field(title="アクセント結合規則")
+
+    class Config:
+        validate_assignment = True
+
+    @validator("surface")
+    def convert_to_zenkaku(cls, surface):
+        return surface.translate(
+            str.maketrans(
+                "".join(chr(0x21 + i) for i in range(94)),
+                "".join(chr(0xFF01 + i) for i in range(94)),
+            )
+        )
+
+    @validator("pronunciation", pre=True)
+    def check_is_katakana(cls, pronunciation):
+        if not fullmatch(r"[ァ-ヴー]+", pronunciation):
+            raise ValueError("発音は有効なカタカナでなくてはいけません。")
+        sutegana = ["ァ", "ィ", "ゥ", "ェ", "ォ", "ャ", "ュ", "ョ", "ヮ", "ッ"]
+        for i in range(len(pronunciation)):
+            if pronunciation[i] in sutegana:
+                # 「キャット」のように、捨て仮名が連続する可能性が考えられるので、
+                # 「ッ」に関しては「ッ」そのものが連続している場合と、「ッ」の後にほかの捨て仮名が連続する場合のみ無効とする
+                if i < len(pronunciation) - 1 and (
+                    pronunciation[i + 1] in sutegana[:-1]
+                    or (
+                        pronunciation[i] == sutegana[-1]
+                        and pronunciation[i + 1] == sutegana[-1]
+                    )
+                ):
+                    raise ValueError("無効な発音です。(捨て仮名の連続)")
+            if pronunciation[i] == "ヮ":
+                if i != 0 and pronunciation[i - 1] not in ["ク", "グ"]:
+                    raise ValueError(
+                        "無効な発音です。(「くゎ」「ぐゎ」以外の「ゎ」の使用)"
+                    )
+        return pronunciation
+
+    @validator("mora_count", pre=True, always=True)
+    def check_mora_count_and_accent_type(cls, mora_count, values):
+        if "pronunciation" not in values or "accent_type" not in values:
+            # 適切な場所でエラーを出すようにする
+            return mora_count
+
+        if mora_count is None:
+            rule_others = (
+                "[イ][ェ]|[ヴ][ャュョ]|[トド][ゥ]|[テデ][ィャュョ]|[デ][ェ]|[クグ][ヮ]"
+            )
+            rule_line_i = "[キシチニヒミリギジビピ][ェャュョ]"
+            rule_line_u = "[ツフヴ][ァ]|[ウスツフヴズ][ィ]|[ウツフヴ][ェォ]"
+            rule_one_mora = "[ァ-ヴー]"
+            mora_count = len(
+                findall(
+                    f"(?:{rule_others}|{rule_line_i}|{rule_line_u}|{rule_one_mora})",
+                    values["pronunciation"],
+                )
+            )
+
+        if not 0 <= values["accent_type"] <= mora_count:
+            raise ValueError(
+                "誤ったアクセント型です({})。 expect: 0 <= accent_type <= {}".format(
+                    values["accent_type"], mora_count
+                )
+            )
+        return mora_count
+
+
+class PartOfSpeechDetail(BaseModel):
+    """
+    品詞ごとの情報
+    """
+
+    part_of_speech: str = Field(title="品詞")
+    part_of_speech_detail_1: str = Field(title="品詞細分類1")
+    part_of_speech_detail_2: str = Field(title="品詞細分類2")
+    part_of_speech_detail_3: str = Field(title="品詞細分類3")
+    # context_idは辞書の左・右文脈IDのこと
+    # https://github.com/VOICEVOX/open_jtalk/blob/427cfd761b78efb6094bea3c5bb8c968f0d711ab/src/mecab-naist-jdic/_left-id.def # noqa
+    context_id: int = Field(title="文脈ID")
+    cost_candidates: List[int] = Field(title="コストのパーセンタイル")
+    accent_associative_rules: List[str] = Field(title="アクセント結合規則の一覧")
+
+
+class WordTypes(str, Enum):
+    """
+    fastapiでword_type引数を検証する時に使用するクラス
+    """
+
+    PROPER_NOUN = "PROPER_NOUN"
+    COMMON_NOUN = "COMMON_NOUN"
+    VERB = "VERB"
+    ADJECTIVE = "ADJECTIVE"
+    SUFFIX = "SUFFIX"

style_bert_vits2/nlp/symbols.py

@@ -0,0 +1,194 @@
+# Punctuations
+PUNCTUATIONS = ["!", "?", "…", ",", ".", "'", "-"]
+
+# Punctuations and special tokens
+PUNCTUATION_SYMBOLS = PUNCTUATIONS + ["SP", "UNK"]
+
+# Padding
+PAD = "_"
+
+# Chinese symbols
+ZH_SYMBOLS = [
+    "E",
+    "En",
+    "a",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "b",
+    "c",
+    "ch",
+    "d",
+    "e",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "f",
+    "g",
+    "h",
+    "i",
+    "i0",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "j",
+    "k",
+    "l",
+    "m",
+    "n",
+    "o",
+    "ong",
+    "ou",
+    "p",
+    "q",
+    "r",
+    "s",
+    "sh",
+    "t",
+    "u",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "ui",
+    "un",
+    "uo",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "AA",
+    "EE",
+    "OO",
+]
+NUM_ZH_TONES = 6
+
+# Japanese
+JP_SYMBOLS = [
+    "N",
+    "a",
+    "a:",
+    "b",
+    "by",
+    "ch",
+    "d",
+    "dy",
+    "e",
+    "e:",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hy",
+    "i",
+    "i:",
+    "j",
+    "k",
+    "ky",
+    "m",
+    "my",
+    "n",
+    "ny",
+    "o",
+    "o:",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "w",
+    "y",
+    "z",
+    "zy",
+]
+NUM_JP_TONES = 2
+
+# English
+EN_SYMBOLS = [
+    "aa",
+    "ae",
+    "ah",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "ch",
+    "d",
+    "dh",
+    "eh",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "hh",
+    "ih",
+    "iy",
+    "jh",
+    "k",
+    "l",
+    "m",
+    "n",
+    "ng",
+    "ow",
+    "oy",
+    "p",
+    "r",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "uh",
+    "uw",
+    "V",
+    "w",
+    "y",
+    "z",
+    "zh",
+]
+NUM_EN_TONES = 4
+
+# Combine all symbols
+NORMAL_SYMBOLS = sorted(set(ZH_SYMBOLS + JP_SYMBOLS + EN_SYMBOLS))
+SYMBOLS = [PAD] + NORMAL_SYMBOLS + PUNCTUATION_SYMBOLS
+SIL_PHONEMES_IDS = [SYMBOLS.index(i) for i in PUNCTUATION_SYMBOLS]
+
+# Combine all tones
+NUM_TONES = NUM_ZH_TONES + NUM_JP_TONES + NUM_EN_TONES
+
+# Language maps
+LANGUAGE_ID_MAP = {"ZH": 0, "JP": 1, "EN": 2}
+NUM_LANGUAGES = len(LANGUAGE_ID_MAP.keys())
+
+# Language tone start map
+LANGUAGE_TONE_START_MAP = {
+    "ZH": 0,
+    "JP": NUM_ZH_TONES,
+    "EN": NUM_ZH_TONES + NUM_JP_TONES,
+}
+
+
+if __name__ == "__main__":
+    a = set(ZH_SYMBOLS)
+    b = set(EN_SYMBOLS)
+    print(sorted(a & b))