diff --git a/bert/bert-base-japanese-v3/README.md b/bert/bert-base-japanese-v3/README.md deleted file mode 100644 index c5b3456..0000000 --- a/bert/bert-base-japanese-v3/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -license: apache-2.0 -datasets: -- cc100 -- wikipedia -language: -- ja -widget: -- text: 東北大学で[MASK]の研究をしています。 ---- - -# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102) - -This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language. - -This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization. -Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective. - -The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/). - -## Model architecture - -The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads. - -## Training Data - -The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia. -For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023. -The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively. - -For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7). - -## Tokenization - -The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm. -The vocabulary size is 32768. - -We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization. - -## Training - -We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps. -For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once. - -For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/). - -## Licenses - -The pretrained models are distributed under the Apache License 2.0. - -## Acknowledgments - -This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program. diff --git a/bert/bert-base-japanese-v3/config.json b/bert/bert-large-japanese-v2/config.json similarity index 75% rename from bert/bert-base-japanese-v3/config.json rename to bert/bert-large-japanese-v2/config.json index 65a2f33..427134b 100644 --- a/bert/bert-base-japanese-v3/config.json +++ b/bert/bert-large-japanese-v2/config.json @@ -5,14 +5,14 @@ "attention_probs_dropout_prob": 0.1, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, - "hidden_size": 768, + "hidden_size": 1024, "initializer_range": 0.02, - "intermediate_size": 3072, + "intermediate_size": 4096, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", - "num_attention_heads": 12, - "num_hidden_layers": 12, + "num_attention_heads": 16, + "num_hidden_layers": 24, "pad_token_id": 0, "type_vocab_size": 2, "vocab_size": 32768 diff --git a/bert/bert-large-japanese-v2/tokenizer_config.json b/bert/bert-large-japanese-v2/tokenizer_config.json new file mode 100644 index 0000000..dfbcc46 --- /dev/null +++ b/bert/bert-large-japanese-v2/tokenizer_config.json @@ -0,0 +1,10 @@ +{ + "tokenizer_class": "BertJapaneseTokenizer", + "model_max_length": 512, + "do_lower_case": false, + "word_tokenizer_type": "mecab", + "subword_tokenizer_type": "wordpiece", + "mecab_kwargs": { + "mecab_dic": "unidic_lite" + } +} diff --git a/bert/bert-base-japanese-v3/vocab.txt b/bert/bert-large-japanese-v2/vocab.txt similarity index 100% rename from bert/bert-base-japanese-v3/vocab.txt rename to bert/bert-large-japanese-v2/vocab.txt diff --git a/data_utils.py b/data_utils.py index 5bf1132..7f490e3 100644 --- a/data_utils.py +++ b/data_utils.py @@ -154,13 +154,13 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset): if language_str == "ZH": bert = bert - ja_bert = torch.zeros(768, len(phone)) + ja_bert = torch.zeros(1024, len(phone)) elif language_str == "JA": ja_bert = bert bert = torch.zeros(1024, len(phone)) else: bert = torch.zeros(1024, len(phone)) - ja_bert = torch.zeros(768, len(phone)) + ja_bert = torch.zeros(1024, len(phone)) assert bert.shape[-1] == len(phone), ( bert.shape, len(phone), @@ -208,7 +208,13 @@ class TextAudioSpeakerCollate: torch.LongTensor([x[1].size(1) for x in batch]), dim=0, descending=True ) - max_text_len = max([len(x[0]) for x in batch]) + max_text_len = max( + [ + batch[ids_sorted_decreasing[i]][7].size(1) + for i in range(len(ids_sorted_decreasing)) + ] + + [len(x[0]) for x in batch] + ) max_spec_len = max([x[1].size(1) for x in batch]) max_wav_len = max([x[2].size(1) for x in batch]) @@ -221,7 +227,7 @@ class TextAudioSpeakerCollate: tone_padded = torch.LongTensor(len(batch), max_text_len) language_padded = torch.LongTensor(len(batch), max_text_len) bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len) - ja_bert_padded = torch.FloatTensor(len(batch), 768, max_text_len) + ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len) spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len) wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len) diff --git a/models.py b/models.py index f392136..44e56a2 100644 --- a/models.py +++ b/models.py @@ -340,7 +340,7 @@ class TextEncoder(nn.Module): self.language_emb = nn.Embedding(num_languages, hidden_channels) nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5) self.bert_proj = nn.Conv1d(1024, hidden_channels, 1) - self.ja_bert_proj = nn.Conv1d(768, hidden_channels, 1) + self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1) self.encoder = attentions.Encoder( hidden_channels, diff --git a/requirements.txt b/requirements.txt index b809dd5..923cbb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ pypinyin cn2an gradio av -mecab-python3 +pyopenjtalk loguru unidic-lite cmudict diff --git a/text/__init__.py b/text/__init__.py index 8dd10db..35b8e21 100644 --- a/text/__init__.py +++ b/text/__init__.py @@ -11,7 +11,12 @@ def cleaned_text_to_sequence(cleaned_text, tones, language): Returns: List of integers corresponding to the symbols in the text """ - phones = [_symbol_to_id[symbol] for symbol in cleaned_text] + phones = [] # _symbol_to_id[symbol] for symbol in cleaned_text + for symbol in cleaned_text: + try: + phones.append(_symbol_to_id[symbol]) + except KeyError: + phones.append(0) # symbol not found in ID map, use 0('_') by default tone_start = language_tone_start_map[language] tones = [i + tone_start for i in tones] lang_id = language_id_map[language] diff --git a/text/japanese.py b/text/japanese.py index 53db38b..0d43c7f 100644 --- a/text/japanese.py +++ b/text/japanese.py @@ -2,17 +2,13 @@ # compatible with Julius https://github.com/julius-speech/segmentation-kit import re import unicodedata +import sys from transformers import AutoTokenizer -from text import punctuation, symbols - -try: - import MeCab -except ImportError as e: - raise ImportError("Japanese requires mecab-python3 and unidic-lite.") from e -from num2words import num2words +import pyopenjtalk +BERT = "./bert/bert-large-japanese-v2" _CONVRULES = [ # Conversion of 2 letters "アァ/ a a", @@ -353,99 +349,6 @@ def hira2kata(text: str) -> str: return text.replace("う゛", "ヴ") -_SYMBOL_TOKENS = set(list("・、。？！")) -_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]")) -_TAGGER = MeCab.Tagger() - - -def text2kata(text: str) -> str: - parsed = _TAGGER.parse(text) - res = [] - for line in parsed.split("\n"): - if line == "EOS": - break - parts = line.split("\t") - - word, yomi = parts[0], parts[1] - if yomi: - res.append(yomi) - else: - if word in _SYMBOL_TOKENS: - res.append(word) - elif word in ("っ", "ッ"): - res.append("ッ") - elif word in _NO_YOMI_TOKENS: - pass - else: - res.append(word) - return hira2kata("".join(res)) - - -_ALPHASYMBOL_YOMI = { - "#": "シャープ", - "%": "パーセント", - "&": "アンド", - "+": "プラス", - "-": "マイナス", - ":": "コロン", - ";": "セミコロン", - "<": "小なり", - "=": "イコール", - ">": "大なり", - "@": "アット", - "a": "エー", - "b": "ビー", - "c": "シー", - "d": "ディー", - "e": "イー", - "f": "エフ", - "g": "ジー", - "h": "エイチ", - "i": "アイ", - "j": "ジェー", - "k": "ケー", - "l": "エル", - "m": "エム", - "n": "エヌ", - "o": "オー", - "p": "ピー", - "q": "キュー", - "r": "アール", - "s": "エス", - "t": "ティー", - "u": "ユー", - "v": "ブイ", - "w": "ダブリュー", - "x": "エックス", - "y": "ワイ", - "z": "ゼット", - "α": "アルファ", - "β": "ベータ", - "γ": "ガンマ", - "δ": "デルタ", - "ε": "イプシロン", - "ζ": "ゼータ", - "η": "イータ", - "θ": "シータ", - "ι": "イオタ", - "κ": "カッパ", - "λ": "ラムダ", - "μ": "ミュー", - "ν": "ニュー", - "ξ": "クサイ", - "ο": "オミクロン", - "π": "パイ", - "ρ": "ロー", - "σ": "シグマ", - "τ": "タウ", - "υ": "ウプシロン", - "φ": "ファイ", - "χ": "カイ", - "ψ": "プサイ", - "ω": "オメガ", -} - - _NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+") _CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"} _CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])") @@ -453,48 +356,12 @@ _NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?") def japanese_convert_numbers_to_words(text: str) -> str: - res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text) - res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res) - res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res) + res = text + for x in _CURRENCY_MAP.keys(): + res = res.replace(x, _CURRENCY_MAP[x]) return res -def japanese_convert_alpha_symbols_to_words(text: str) -> str: - return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()]) - - -def japanese_text_to_phonemes(text: str) -> str: - """Convert Japanese text to phonemes.""" - res = unicodedata.normalize("NFKC", text) - res = japanese_convert_numbers_to_words(res) - # res = japanese_convert_alpha_symbols_to_words(res) - res = text2kata(res) - res = kata2phoneme(res) - return res - - -def is_japanese_character(char): - # 定义日语文字系统的 Unicode 范围 - japanese_ranges = [ - (0x3040, 0x309F), # 平假名 - (0x30A0, 0x30FF), # 片假名 - (0x4E00, 0x9FFF), # 汉字 (CJK Unified Ideographs) - (0x3400, 0x4DBF), # 汉字扩展 A - (0x20000, 0x2A6DF), # 汉字扩展 B - # 可以根据需要添加其他汉字扩展范围 - ] - - # 将字符的 Unicode 编码转换为整数 - char_code = ord(char) - - # 检查字符是否在任何一个日语范围内 - for start, end in japanese_ranges: - if start <= char_code <= end: - return True - - return False - - rep_map = { "：": ",", "；": ",", @@ -510,18 +377,9 @@ rep_map = { def replace_punctuation(text): - pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys())) - - replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) - - replaced_text = re.sub( - r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF" - + "".join(punctuation) - + r"]+", - "", - replaced_text, - ) - + replaced_text = text + for x in rep_map.keys(): + replaced_text = replaced_text.replace(x, rep_map[x]) return replaced_text @@ -533,54 +391,42 @@ def text_normalize(text): return res -def distribute_phone(n_phone, n_word): - phones_per_word = [0] * n_word - for task in range(n_phone): - min_tasks = min(phones_per_word) - min_index = phones_per_word.index(min_tasks) - phones_per_word[min_index] += 1 - return phones_per_word - - -tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") +tokenizer = AutoTokenizer.from_pretrained(BERT) def g2p(norm_text): tokenized = tokenizer.tokenize(norm_text) - phs = [] - ph_groups = [] - for t in tokenized: - if not t.startswith("#"): - ph_groups.append([t]) - else: - ph_groups[-1].append(t.replace("#", "")) + st = [x.replace("#", "") for x in tokenized] word2ph = [] - for group in ph_groups: - phonemes = kata2phoneme(text2kata("".join(group))) - # phonemes = [i for i in phonemes if i in symbols] - for i in phonemes: - assert i in symbols, (group, norm_text, tokenized) - phone_len = len(phonemes) - word_len = len(group) - - aaa = distribute_phone(phone_len, word_len) - word2ph += aaa - - phs += phonemes - phones = ["_"] + phs + ["_"] - tones = [0 for i in phones] + phs = pyopenjtalk.g2p(norm_text).split( + " " + ) # Directly use the entire norm_text sequence. + for sub in st: # the following code is only for calculating word2ph + wph = 0 + for x in sub: + sys.stdout.flush() + if x not in ["?", ".", "!", "…", ","]: # This will throw warnings. + phonemes = pyopenjtalk.g2p(x) + else: + phonemes = "pau" + # for x in range(repeat): + wph += len(phonemes.split(" ")) + # print(f'{x}-->:{phones}') + word2ph.append(wph) + phonemes = ["_"] + phs + ["_"] + tones = [0 for i in phonemes] word2ph = [1] + word2ph + [1] - return phones, tones, word2ph + return phonemes, tones, word2ph if __name__ == "__main__": - tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") + tokenizer = AutoTokenizer.from_pretrained(BERT) text = "hello,こんにちは、世界！……" from text.japanese_bert import get_bert_feature text = text_normalize(text) print(text) - phones, tones, word2ph = g2p(text) + phones, tones, word2ph = g2p_ojt(text) bert = get_bert_feature(text, word2ph) print(phones, tones, word2ph, bert.shape) diff --git a/text/japanese_bert.py b/text/japanese_bert.py index 5dd1964..6900400 100644 --- a/text/japanese_bert.py +++ b/text/japanese_bert.py @@ -2,7 +2,12 @@ import torch from transformers import AutoTokenizer, AutoModelForMaskedLM import sys -tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") +BERT = "./bert/bert-large-japanese-v2" + +tokenizer = AutoTokenizer.from_pretrained(BERT) +# bert-large model has 25 hidden layers.You can decide which layer to use by setting this variable to a specific value +# default value is 3(untested) +BERT_LAYER = 3 models = dict() @@ -24,13 +29,13 @@ def get_bert_feature(text, word2ph, device=None): inputs = tokenizer(text, return_tensors="pt") for i in inputs: inputs[i] = inputs[i].to(device) - res = models[device](**inputs, output_hidden_states=True) - res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu() + res = model(**inputs, output_hidden_states=True) + res = res["hidden_states"][BERT_LAYER] assert inputs["input_ids"].shape[-1] == len(word2ph) word2phone = word2ph phone_level_feature = [] for i in range(len(word2phone)): - repeat_feature = res[i].repeat(word2phone[i], 1) + repeat_feature = res[0][i].repeat(word2phone[i], 1) phone_level_feature.append(repeat_feature) phone_level_feature = torch.cat(phone_level_feature, dim=0) diff --git a/text/symbols.py b/text/symbols.py index 161ae9f..94f0f63 100644 --- a/text/symbols.py +++ b/text/symbols.py @@ -74,6 +74,7 @@ num_zh_tones = 6 # japanese ja_symbols = [ + "pau", "N", "a", "a:", @@ -117,6 +118,8 @@ ja_symbols = [ "z", "zy", ] +for x in range(ord("a"), ord("z") + 1): + ja_symbols.append(chr(x).upper()) num_ja_tones = 1 # English