Refactor: rename text_processing to nlp

"text_processing" is clearer, but the import statement is longer.
"nlp" is shorter and makes it clear that it is natural language processing.

bert_gen.py

@@ -9,8 +9,8 @@ import utils
 from config import config
 from style_bert_vits2.logging import logger
 from style_bert_vits2.models import commons
-from style_bert_vits2.text_processing import cleaned_text_to_sequence, extract_bert_feature
+from style_bert_vits2.nlp import cleaned_text_to_sequence, extract_bert_feature
-from style_bert_vits2.text_processing.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
 from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
 
 pyopenjtalk.initialize()

data_utils.py

@@ -12,7 +12,7 @@ from mel_processing import mel_spectrogram_torch, spectrogram_torch
 from utils import load_filepaths_and_text, load_wav_to_torch
 from style_bert_vits2.logging import logger
 from style_bert_vits2.models import commons
-from style_bert_vits2.text_processing import cleaned_text_to_sequence
+from style_bert_vits2.nlp import cleaned_text_to_sequence
 
 """Multi speaker version"""
 

preprocess_text.py

@@ -9,7 +9,7 @@ from tqdm import tqdm
 
 from config import config
 from style_bert_vits2.logging import logger
-from style_bert_vits2.text_processing import clean_text
+from style_bert_vits2.nlp import clean_text
 from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
 
 preprocess_text_config = config.preprocess_text_config

server_editor.py

@@ -42,10 +42,10 @@ from style_bert_vits2.constants import (
     Languages,
 )
 from style_bert_vits2.logging import logger
-from style_bert_vits2.text_processing import bert_models
+from style_bert_vits2.nlp import bert_models
-from style_bert_vits2.text_processing.japanese import normalize_text
+from style_bert_vits2.nlp.japanese import normalize_text
-from style_bert_vits2.text_processing.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone
+from style_bert_vits2.nlp.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone
-from style_bert_vits2.text_processing.japanese.user_dict import (
+from style_bert_vits2.nlp.japanese.user_dict import (
     apply_word,
     delete_word,
     read_dict,

style_bert_vits2/constants.py

@@ -23,7 +23,7 @@ DEFAULT_BERT_TOKENIZER_PATHS = {
 }
 
 # デフォルトのユーザー辞書ディレクトリ
-## style_bert_vits2.text_processing.japanese.user_dict モジュールのデフォルト値として利用される
+## style_bert_vits2.nlp.japanese.user_dict モジュールのデフォルト値として利用される
 ## ライブラリとしての利用などで外部のユーザー辞書を指定したい場合は、user_dict 以下の各関数の実行時、引数に辞書データファイルのパスを指定する
 DEFAULT_USER_DICT_DIR = BASE_DIR / "dict_data"
 

style_bert_vits2/models/infer.py

@@ -7,8 +7,8 @@ from style_bert_vits2.logging import logger
 from style_bert_vits2.models import commons
 from style_bert_vits2.models.models import SynthesizerTrn
 from style_bert_vits2.models.models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra
-from style_bert_vits2.text_processing import clean_text, cleaned_text_to_sequence, extract_bert_feature
+from style_bert_vits2.nlp import clean_text, cleaned_text_to_sequence, extract_bert_feature
-from style_bert_vits2.text_processing.symbols import SYMBOLS
+from style_bert_vits2.nlp.symbols import SYMBOLS
 
 
 def get_net_g(model_path: str, version: str, device: str, hps):

style_bert_vits2/models/models.py

@@ -11,7 +11,7 @@ from style_bert_vits2.models import commons
 from style_bert_vits2.models import modules
 from style_bert_vits2.models import monotonic_alignment
 from style_bert_vits2.models.commons import get_padding, init_weights
-from style_bert_vits2.text_processing.symbols import NUM_LANGUAGES, NUM_TONES, SYMBOLS
+from style_bert_vits2.nlp.symbols import NUM_LANGUAGES, NUM_TONES, SYMBOLS
 
 
 class DurationDiscriminator(nn.Module):  # vits2

style_bert_vits2/models/models_jp_extra.py

@@ -10,7 +10,7 @@ from style_bert_vits2.models import attentions
 from style_bert_vits2.models import commons
 from style_bert_vits2.models import modules
 from style_bert_vits2.models import monotonic_alignment
-from style_bert_vits2.text_processing.symbols import SYMBOLS, NUM_TONES, NUM_LANGUAGES
+from style_bert_vits2.nlp.symbols import SYMBOLS, NUM_TONES, NUM_LANGUAGES
 
 
 class DurationDiscriminator(nn.Module):  # vits2

style_bert_vits2/nlp/__init__.py

@@ -2,7 +2,7 @@ import torch
 from typing import Optional
 
 from style_bert_vits2.constants import Languages
-from style_bert_vits2.text_processing.symbols import (
+from style_bert_vits2.nlp.symbols import (
     LANGUAGE_ID_MAP,
     LANGUAGE_TONE_START_MAP,
     SYMBOLS,
@@ -36,11 +36,11 @@ def extract_bert_feature(
     """
 
     if language == Languages.JP:
-        from style_bert_vits2.text_processing.japanese.bert_feature import extract_bert_feature
+        from style_bert_vits2.nlp.japanese.bert_feature import extract_bert_feature
     elif language == Languages.EN:
-        from style_bert_vits2.text_processing.english.bert_feature import extract_bert_feature
+        from style_bert_vits2.nlp.english.bert_feature import extract_bert_feature
     elif language == Languages.ZH:
-        from style_bert_vits2.text_processing.chinese.bert_feature import extract_bert_feature
+        from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature
     else:
         raise ValueError(f"Language {language} not supported")
 
@@ -68,15 +68,15 @@ def clean_text(
 
     # Changed to import inside if condition to avoid unnecessary import
     if language == Languages.JP:
-        from style_bert_vits2.text_processing.japanese import g2p, normalize_text
+        from style_bert_vits2.nlp.japanese import g2p, normalize_text
         norm_text = normalize_text(text)
         phones, tones, word2ph = g2p(norm_text, use_jp_extra, raise_yomi_error)
     elif language == Languages.EN:
-        from style_bert_vits2.text_processing.english import g2p, normalize_text
+        from style_bert_vits2.nlp.english import g2p, normalize_text
         norm_text = normalize_text(text)
         phones, tones, word2ph = g2p(norm_text)
     elif language == Languages.ZH:
-        from style_bert_vits2.text_processing.chinese import g2p, normalize_text
+        from style_bert_vits2.nlp.chinese import g2p, normalize_text
         norm_text = normalize_text(text)
         phones, tones, word2ph = g2p(norm_text)
     else:

style_bert_vits2/nlp/chinese/__init__.py

@@ -5,8 +5,8 @@ import cn2an
 import jieba.posseg as psg
 from pypinyin import lazy_pinyin, Style
 
-from style_bert_vits2.text_processing.chinese.tone_sandhi import ToneSandhi
+from style_bert_vits2.nlp.chinese.tone_sandhi import ToneSandhi
-from style_bert_vits2.text_processing.symbols import PUNCTUATIONS
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
 
 
 current_file_path = os.path.dirname(__file__)
@@ -177,7 +177,7 @@ def normalize_text(text: str) -> str:
 
 
 if __name__ == "__main__":
-    from style_bert_vits2.text_processing.chinese.bert_feature import extract_bert_feature
+    from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature
 
     text = "啊！但是《原神》是由,米哈\游自主，  [研发]的一款全.新开放世界.冒险游戏"
     text = normalize_text(text)

style_bert_vits2/nlp/chinese/bert_feature.py

@@ -5,7 +5,7 @@ import torch
 from transformers import PreTrainedModel
 
 from style_bert_vits2.constants import Languages
-from style_bert_vits2.text_processing import bert_models
+from style_bert_vits2.nlp import bert_models
 
 
 __models: dict[torch.device | str, PreTrainedModel] = {}

style_bert_vits2/nlp/english/__init__.py

@@ -4,8 +4,8 @@ import re
 from g2p_en import G2p
 
 from style_bert_vits2.constants import Languages
-from style_bert_vits2.text_processing import bert_models
+from style_bert_vits2.nlp import bert_models
-from style_bert_vits2.text_processing.symbols import PUNCTUATIONS, SYMBOLS
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS, SYMBOLS
 
 
 current_file_path = os.path.dirname(__file__)

style_bert_vits2/nlp/english/bert_feature.py

@@ -5,7 +5,7 @@ import torch
 from transformers import PreTrainedModel
 
 from style_bert_vits2.constants import Languages
-from style_bert_vits2.text_processing import bert_models
+from style_bert_vits2.nlp import bert_models
 
 
 __models: dict[torch.device | str, PreTrainedModel] = {}

style_bert_vits2/nlp/japanese/__init__.py

@@ -0,0 +1,2 @@
+from style_bert_vits2.nlp.japanese.g2p import g2p  # noqa: F401
+from style_bert_vits2.nlp.japanese.normalizer import normalize_text  # noqa: F401

style_bert_vits2/nlp/japanese/bert_feature.py

@@ -5,8 +5,8 @@ import torch
 from transformers import PreTrainedModel
 
 from style_bert_vits2.constants import Languages
-from style_bert_vits2.text_processing import bert_models
+from style_bert_vits2.nlp import bert_models
-from style_bert_vits2.text_processing.japanese.g2p import text_to_sep_kata
+from style_bert_vits2.nlp.japanese.g2p import text_to_sep_kata
 
 
 __models: dict[torch.device | str, PreTrainedModel] = {}

style_bert_vits2/nlp/japanese/g2p.py

@@ -2,11 +2,11 @@ import re
 
 from style_bert_vits2.constants import Languages
 from style_bert_vits2.logging import logger
-from style_bert_vits2.text_processing import bert_models
+from style_bert_vits2.nlp import bert_models
-from style_bert_vits2.text_processing.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
-from style_bert_vits2.text_processing.japanese.mora_list import MORA_KATA_TO_MORA_PHONEMES
+from style_bert_vits2.nlp.japanese.mora_list import MORA_KATA_TO_MORA_PHONEMES
-from style_bert_vits2.text_processing.japanese.normalizer import replace_punctuation
+from style_bert_vits2.nlp.japanese.normalizer import replace_punctuation
-from style_bert_vits2.text_processing.symbols import PUNCTUATIONS
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
 
 pyopenjtalk.initialize()
 

style_bert_vits2/nlp/japanese/g2p_utils.py

@@ -1,9 +1,9 @@
-from style_bert_vits2.text_processing.japanese.g2p import g2p
+from style_bert_vits2.nlp.japanese.g2p import g2p
-from style_bert_vits2.text_processing.japanese.mora_list import (
+from style_bert_vits2.nlp.japanese.mora_list import (
     MORA_KATA_TO_MORA_PHONEMES,
     MORA_PHONEMES_TO_MORA_KATA,
 )
-from style_bert_vits2.text_processing.symbols import PUNCTUATIONS
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
 
 
 def g2kata_tone(norm_text: str) -> list[tuple[str, int]]:

style_bert_vits2/nlp/japanese/normalizer.py

@@ -2,7 +2,7 @@ import re
 import unicodedata
 from num2words import num2words
 
-from style_bert_vits2.text_processing.symbols import PUNCTUATIONS
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
 
 
 def normalize_text(text: str) -> str:

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__init__.py

@@ -6,8 +6,8 @@ to avoid user dictionary access error
 from typing import Any, Optional
 
 from style_bert_vits2.logging import logger
-from style_bert_vits2.text_processing.japanese.pyopenjtalk_worker.worker_client import WorkerClient
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_client import WorkerClient
-from style_bert_vits2.text_processing.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
 
 
 WORKER_CLIENT: Optional[WorkerClient] = None

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__main__.py

@@ -1,7 +1,7 @@
 import argparse
 
-from style_bert_vits2.text_processing.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
-from style_bert_vits2.text_processing.japanese.pyopenjtalk_worker.worker_server import WorkerServer
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_server import WorkerServer
 
 
 def main() -> None:

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_client.py

@@ -2,7 +2,7 @@ import socket
 from typing import Any, cast
 
 from style_bert_vits2.logging import logger
-from style_bert_vits2.text_processing.japanese.pyopenjtalk_worker.worker_common import RequestType, receive_data, send_data
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import RequestType, receive_data, send_data
 
 
 class WorkerClient:

style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py

@@ -6,7 +6,7 @@ from typing import Any, cast
 import pyopenjtalk
 
 from style_bert_vits2.logging import logger
-from style_bert_vits2.text_processing.japanese.pyopenjtalk_worker.worker_common import (
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import (
     ConnectionClosedException,
     RequestType,
     receive_data,

style_bert_vits2/nlp/japanese/user_dict/__init__.py

@@ -16,9 +16,9 @@ import numpy as np
 from fastapi import HTTPException
 
 from style_bert_vits2.constants import DEFAULT_USER_DICT_DIR
-from style_bert_vits2.text_processing.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
-from style_bert_vits2.text_processing.japanese.user_dict.word_model import UserDictWord, WordTypes
+from style_bert_vits2.nlp.japanese.user_dict.word_model import UserDictWord, WordTypes
-from style_bert_vits2.text_processing.japanese.user_dict.part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data
+from style_bert_vits2.nlp.japanese.user_dict.part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data
 
 pyopenjtalk.initialize()
 

style_bert_vits2/nlp/japanese/user_dict/part_of_speech_data.py

@@ -7,7 +7,7 @@
 
 from typing import Dict
 
-from style_bert_vits2.text_processing.japanese.user_dict.word_model import (
+from style_bert_vits2.nlp.japanese.user_dict.word_model import (
     USER_DICT_MAX_PRIORITY,
     USER_DICT_MIN_PRIORITY,
     PartOfSpeechDetail,

style_bert_vits2/text_processing/japanese/__init__.py

@@ -1,2 +0,0 @@
-from style_bert_vits2.text_processing.japanese.g2p import g2p  # noqa: F401
-from style_bert_vits2.text_processing.japanese.normalizer import normalize_text  # noqa: F401

train_ms.py

@@ -31,7 +31,7 @@ from style_bert_vits2.models.models import (
     MultiPeriodDiscriminator,
     SynthesizerTrn,
 )
-from style_bert_vits2.text_processing.symbols import SYMBOLS
+from style_bert_vits2.nlp.symbols import SYMBOLS
 from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
 
 torch.backends.cuda.matmul.allow_tf32 = True

train_ms_jp_extra.py

@@ -32,7 +32,7 @@ from style_bert_vits2.models.models_jp_extra import (
     SynthesizerTrn,
     WavLMDiscriminator,
 )
-from style_bert_vits2.text_processing.symbols import SYMBOLS
+from style_bert_vits2.nlp.symbols import SYMBOLS
 from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
 
 torch.backends.cuda.matmul.allow_tf32 = True

webui/inference.py

@@ -20,8 +20,8 @@ from style_bert_vits2.constants import (
 )
 from style_bert_vits2.logging import logger
 from style_bert_vits2.models.infer import InvalidToneError
-from style_bert_vits2.text_processing.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone
+from style_bert_vits2.nlp.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone
-from style_bert_vits2.text_processing.japanese.normalizer import normalize_text
+from style_bert_vits2.nlp.japanese.normalizer import normalize_text
 
 languages = [l.value for l in Languages]