Revert "change g2p and other fix" (#40)

bert/bert-base-japanese-v3/README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+datasets:
+- cc100
+- wikipedia
+language:
+- ja
+widget:
+- text: 東北大学で[MASK]の研究をしています。
+---
+
+# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
+
+This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
+
+This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
+Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
+
+The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
+
+## Model architecture
+
+The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
+
+## Training Data
+
+The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
+For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
+The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
+
+For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
+
+## Tokenization
+
+The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
+The vocabulary size is 32768.
+
+We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
+
+## Training
+
+We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
+For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
+
+For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
+
+## Licenses
+
+The pretrained models are distributed under the Apache License 2.0.
+
+## Acknowledgments
+
+This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.

bert/bert-base-japanese-v3/config.json

@@ -5,14 +5,14 @@
     "attention_probs_dropout_prob": 0.1,
     "hidden_act": "gelu",
     "hidden_dropout_prob": 0.1,
-    "hidden_size": 1024,
+    "hidden_size": 768,
     "initializer_range": 0.02,
-    "intermediate_size": 4096,
+    "intermediate_size": 3072,
     "layer_norm_eps": 1e-12,
     "max_position_embeddings": 512,
     "model_type": "bert",
-    "num_attention_heads": 16,
+    "num_attention_heads": 12,
-    "num_hidden_layers": 24,
+    "num_hidden_layers": 12,
     "pad_token_id": 0,
     "type_vocab_size": 2,
     "vocab_size": 32768

bert/bert-large-japanese-v2/tokenizer_config.json

@@ -1,10 +0,0 @@
-{
-    "tokenizer_class": "BertJapaneseTokenizer",
-    "model_max_length": 512,
-    "do_lower_case": false,
-    "word_tokenizer_type": "mecab",
-    "subword_tokenizer_type": "wordpiece",
-    "mecab_kwargs": {
-        "mecab_dic": "unidic_lite"
-    }
-}

data_utils.py

@@ -154,13 +154,13 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
 
         if language_str == "ZH":
             bert = bert
-            ja_bert = torch.zeros(1024, len(phone))
+            ja_bert = torch.zeros(768, len(phone))
         elif language_str == "JA":
             ja_bert = bert
             bert = torch.zeros(1024, len(phone))
         else:
             bert = torch.zeros(1024, len(phone))
-            ja_bert = torch.zeros(1024, len(phone))
+            ja_bert = torch.zeros(768, len(phone))
         assert bert.shape[-1] == len(phone), (
             bert.shape,
             len(phone),
@@ -208,13 +208,7 @@ class TextAudioSpeakerCollate:
             torch.LongTensor([x[1].size(1) for x in batch]), dim=0, descending=True
         )
 
-        max_text_len = max(
+        max_text_len = max([len(x[0]) for x in batch])
-            [
-                batch[ids_sorted_decreasing[i]][7].size(1)
-                for i in range(len(ids_sorted_decreasing))
-            ]
-            + [len(x[0]) for x in batch]
-        )
         max_spec_len = max([x[1].size(1) for x in batch])
         max_wav_len = max([x[2].size(1) for x in batch])
 
@@ -227,7 +221,7 @@ class TextAudioSpeakerCollate:
         tone_padded = torch.LongTensor(len(batch), max_text_len)
         language_padded = torch.LongTensor(len(batch), max_text_len)
         bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
-        ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
+        ja_bert_padded = torch.FloatTensor(len(batch), 768, max_text_len)
 
         spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
         wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)

models.py

@@ -340,7 +340,7 @@ class TextEncoder(nn.Module):
         self.language_emb = nn.Embedding(num_languages, hidden_channels)
         nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
         self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
-        self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
+        self.ja_bert_proj = nn.Conv1d(768, hidden_channels, 1)
 
         self.encoder = attentions.Encoder(
             hidden_channels,

requirements.txt

@@ -15,7 +15,7 @@ pypinyin
 cn2an
 gradio
 av
-pyopenjtalk
+mecab-python3
 loguru
 unidic-lite
 cmudict

text/__init__.py

@@ -11,12 +11,7 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
     Returns:
       List of integers corresponding to the symbols in the text
     """
-    phones = []  # _symbol_to_id[symbol] for symbol in cleaned_text
+    phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
-    for symbol in cleaned_text:
-        try:
-            phones.append(_symbol_to_id[symbol])
-        except KeyError:
-            phones.append(0)  # symbol not found in ID map, use 0('_') by default
     tone_start = language_tone_start_map[language]
     tones = [i + tone_start for i in tones]
     lang_id = language_id_map[language]

text/japanese.py

@@ -2,13 +2,17 @@
 # compatible with Julius https://github.com/julius-speech/segmentation-kit
 import re
 import unicodedata
-import sys
 
 from transformers import AutoTokenizer
 
-import pyopenjtalk
+from text import punctuation, symbols
+
+try:
+    import MeCab
+except ImportError as e:
+    raise ImportError("Japanese requires mecab-python3 and unidic-lite.") from e
+from num2words import num2words
 
-BERT = "./bert/bert-large-japanese-v2"
 _CONVRULES = [
     # Conversion of 2 letters
     "アァ/ a a",
@@ -349,6 +353,99 @@ def hira2kata(text: str) -> str:
     return text.replace("う゛", "ヴ")
 
 
+_SYMBOL_TOKENS = set(list("・、。？！"))
+_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
+_TAGGER = MeCab.Tagger()
+
+
+def text2kata(text: str) -> str:
+    parsed = _TAGGER.parse(text)
+    res = []
+    for line in parsed.split("\n"):
+        if line == "EOS":
+            break
+        parts = line.split("\t")
+
+        word, yomi = parts[0], parts[1]
+        if yomi:
+            res.append(yomi)
+        else:
+            if word in _SYMBOL_TOKENS:
+                res.append(word)
+            elif word in ("っ", "ッ"):
+                res.append("ッ")
+            elif word in _NO_YOMI_TOKENS:
+                pass
+            else:
+                res.append(word)
+    return hira2kata("".join(res))
+
+
+_ALPHASYMBOL_YOMI = {
+    "#": "シャープ",
+    "%": "パーセント",
+    "&": "アンド",
+    "+": "プラス",
+    "-": "マイナス",
+    ":": "コロン",
+    ";": "セミコロン",
+    "<": "小なり",
+    "=": "イコール",
+    ">": "大なり",
+    "@": "アット",
+    "a": "エー",
+    "b": "ビー",
+    "c": "シー",
+    "d": "ディー",
+    "e": "イー",
+    "f": "エフ",
+    "g": "ジー",
+    "h": "エイチ",
+    "i": "アイ",
+    "j": "ジェー",
+    "k": "ケー",
+    "l": "エル",
+    "m": "エム",
+    "n": "エヌ",
+    "o": "オー",
+    "p": "ピー",
+    "q": "キュー",
+    "r": "アール",
+    "s": "エス",
+    "t": "ティー",
+    "u": "ユー",
+    "v": "ブイ",
+    "w": "ダブリュー",
+    "x": "エックス",
+    "y": "ワイ",
+    "z": "ゼット",
+    "α": "アルファ",
+    "β": "ベータ",
+    "γ": "ガンマ",
+    "δ": "デルタ",
+    "ε": "イプシロン",
+    "ζ": "ゼータ",
+    "η": "イータ",
+    "θ": "シータ",
+    "ι": "イオタ",
+    "κ": "カッパ",
+    "λ": "ラムダ",
+    "μ": "ミュー",
+    "ν": "ニュー",
+    "ξ": "クサイ",
+    "ο": "オミクロン",
+    "π": "パイ",
+    "ρ": "ロー",
+    "σ": "シグマ",
+    "τ": "タウ",
+    "υ": "ウプシロン",
+    "φ": "ファイ",
+    "χ": "カイ",
+    "ψ": "プサイ",
+    "ω": "オメガ",
+}
+
+
 _NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
 _CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
 _CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
@@ -356,12 +453,48 @@ _NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
 
 
 def japanese_convert_numbers_to_words(text: str) -> str:
-    res = text
+    res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
-    for x in _CURRENCY_MAP.keys():
+    res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
-        res = res.replace(x, _CURRENCY_MAP[x])
+    res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
     return res
 
 
+def japanese_convert_alpha_symbols_to_words(text: str) -> str:
+    return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()])
+
+
+def japanese_text_to_phonemes(text: str) -> str:
+    """Convert Japanese text to phonemes."""
+    res = unicodedata.normalize("NFKC", text)
+    res = japanese_convert_numbers_to_words(res)
+    # res = japanese_convert_alpha_symbols_to_words(res)
+    res = text2kata(res)
+    res = kata2phoneme(res)
+    return res
+
+
+def is_japanese_character(char):
+    # 定义日语文字系统的 Unicode 范围
+    japanese_ranges = [
+        (0x3040, 0x309F),  # 平假名
+        (0x30A0, 0x30FF),  # 片假名
+        (0x4E00, 0x9FFF),  # 汉字 (CJK Unified Ideographs)
+        (0x3400, 0x4DBF),  # 汉字扩展 A
+        (0x20000, 0x2A6DF),  # 汉字扩展 B
+        # 可以根据需要添加其他汉字扩展范围
+    ]
+
+    # 将字符的 Unicode 编码转换为整数
+    char_code = ord(char)
+
+    # 检查字符是否在任何一个日语范围内
+    for start, end in japanese_ranges:
+        if start <= char_code <= end:
+            return True
+
+    return False
+
+
 rep_map = {
     "：": ",",
     "；": ",",
@@ -377,9 +510,18 @@ rep_map = {
 
 
 def replace_punctuation(text):
-    replaced_text = text
+    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-    for x in rep_map.keys():
+
-        replaced_text = replaced_text.replace(x, rep_map[x])
+    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
+
+    replaced_text = re.sub(
+        r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF"
+        + "".join(punctuation)
+        + r"]+",
+        "",
+        replaced_text,
+    )
+
     return replaced_text
 
 
@@ -391,42 +533,54 @@ def text_normalize(text):
     return res
 
 
-tokenizer = AutoTokenizer.from_pretrained(BERT)
+def distribute_phone(n_phone, n_word):
+    phones_per_word = [0] * n_word
+    for task in range(n_phone):
+        min_tasks = min(phones_per_word)
+        min_index = phones_per_word.index(min_tasks)
+        phones_per_word[min_index] += 1
+    return phones_per_word
+
+
+tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
 
 
 def g2p(norm_text):
     tokenized = tokenizer.tokenize(norm_text)
-    st = [x.replace("#", "") for x in tokenized]
+    phs = []
+    ph_groups = []
+    for t in tokenized:
+        if not t.startswith("#"):
+            ph_groups.append([t])
+        else:
+            ph_groups[-1].append(t.replace("#", ""))
     word2ph = []
-    phs = pyopenjtalk.g2p(norm_text).split(
+    for group in ph_groups:
-        " "
+        phonemes = kata2phoneme(text2kata("".join(group)))
-    )  # Directly use the entire norm_text sequence.
+        # phonemes = [i for i in phonemes if i in symbols]
-    for sub in st:  # the following code is only for calculating word2ph
+        for i in phonemes:
-        wph = 0
+            assert i in symbols, (group, norm_text, tokenized)
-        for x in sub:
+        phone_len = len(phonemes)
-            sys.stdout.flush()
+        word_len = len(group)
-            if x not in ["?", ".", "!", "…", ","]:  # This will throw warnings.
+
-                phonemes = pyopenjtalk.g2p(x)
+        aaa = distribute_phone(phone_len, word_len)
-            else:
+        word2ph += aaa
-                phonemes = "pau"
+
-            # for x in range(repeat):
+        phs += phonemes
-            wph += len(phonemes.split(" "))
+    phones = ["_"] + phs + ["_"]
-            # print(f'{x}-->:{phones}')
+    tones = [0 for i in phones]
-        word2ph.append(wph)
-    phonemes = ["_"] + phs + ["_"]
-    tones = [0 for i in phonemes]
     word2ph = [1] + word2ph + [1]
-    return phonemes, tones, word2ph
+    return phones, tones, word2ph
 
 
 if __name__ == "__main__":
-    tokenizer = AutoTokenizer.from_pretrained(BERT)
+    tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
     text = "hello,こんにちは、世界！……"
     from text.japanese_bert import get_bert_feature
 
     text = text_normalize(text)
     print(text)
-    phones, tones, word2ph = g2p_ojt(text)
+    phones, tones, word2ph = g2p(text)
     bert = get_bert_feature(text, word2ph)
 
     print(phones, tones, word2ph, bert.shape)

text/japanese_bert.py

@@ -2,12 +2,7 @@ import torch
 from transformers import AutoTokenizer, AutoModelForMaskedLM
 import sys
 
-BERT = "./bert/bert-large-japanese-v2"
+tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
-
-tokenizer = AutoTokenizer.from_pretrained(BERT)
-# bert-large model has 25 hidden layers.You can decide which layer to use by setting this variable to a specific value
-# default value is 3(untested)
-BERT_LAYER = 3
 
 models = dict()
 
@@ -29,13 +24,13 @@ def get_bert_feature(text, word2ph, device=None):
         inputs = tokenizer(text, return_tensors="pt")
         for i in inputs:
             inputs[i] = inputs[i].to(device)
-        res = model(**inputs, output_hidden_states=True)
+        res = models[device](**inputs, output_hidden_states=True)
-        res = res["hidden_states"][BERT_LAYER]
+        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
     assert inputs["input_ids"].shape[-1] == len(word2ph)
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[0][i].repeat(word2phone[i], 1)
+        repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

text/symbols.py

@@ -74,7 +74,6 @@ num_zh_tones = 6
 
 # japanese
 ja_symbols = [
-    "pau",
     "N",
     "a",
     "a:",
@@ -118,8 +117,6 @@ ja_symbols = [
     "z",
     "zy",
 ]
-for x in range(ord("a"), ord("z") + 1):
-    ja_symbols.append(chr(x).upper())
 num_ja_tones = 1
 
 # English