Merge branch 'dev' into master
This commit is contained in:
Stardust·减
@@ -1,53 +0,0 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
datasets:
|
||||
- cc100
|
||||
- wikipedia
|
||||
language:
|
||||
- ja
|
||||
widget:
|
||||
- text: 東北大学で[MASK]の研究をしています。
|
||||
---
|
||||
|
||||
# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
|
||||
|
||||
This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
|
||||
|
||||
This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
|
||||
Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
|
||||
|
||||
The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
|
||||
|
||||
## Model architecture
|
||||
|
||||
The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
|
||||
|
||||
## Training Data
|
||||
|
||||
The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
|
||||
For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
|
||||
The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
|
||||
|
||||
For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
|
||||
|
||||
## Tokenization
|
||||
|
||||
The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
|
||||
The vocabulary size is 32768.
|
||||
|
||||
We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
|
||||
|
||||
## Training
|
||||
|
||||
We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
|
||||
For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
|
||||
|
||||
For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
|
||||
|
||||
## Licenses
|
||||
|
||||
The pretrained models are distributed under the Apache License 2.0.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.
|
||||
@@ -5,14 +5,14 @@
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 768,
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"intermediate_size": 4096,
|
||||
"layer_norm_eps": 1e-12,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "bert",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 12,
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"pad_token_id": 0,
|
||||
"type_vocab_size": 2,
|
||||
"vocab_size": 32768
|
||||
10
bert/bert-large-japanese-v2/tokenizer_config.json
Normal file
10
bert/bert-large-japanese-v2/tokenizer_config.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"tokenizer_class": "BertJapaneseTokenizer",
|
||||
"model_max_length": 512,
|
||||
"do_lower_case": false,
|
||||
"word_tokenizer_type": "mecab",
|
||||
"subword_tokenizer_type": "wordpiece",
|
||||
"mecab_kwargs": {
|
||||
"mecab_dic": "unidic_lite"
|
||||
}
|
||||
}
|
||||
@@ -154,13 +154,13 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert
|
||||
ja_bert = torch.zeros(768, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "JA":
|
||||
ja_bert = bert
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
else:
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(768, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
assert bert.shape[-1] == len(phone), (
|
||||
bert.shape,
|
||||
len(phone),
|
||||
@@ -208,7 +208,13 @@ class TextAudioSpeakerCollate:
|
||||
torch.LongTensor([x[1].size(1) for x in batch]), dim=0, descending=True
|
||||
)
|
||||
|
||||
max_text_len = max([len(x[0]) for x in batch])
|
||||
max_text_len = max(
|
||||
[
|
||||
batch[ids_sorted_decreasing[i]][7].size(1)
|
||||
for i in range(len(ids_sorted_decreasing))
|
||||
]
|
||||
+ [len(x[0]) for x in batch]
|
||||
)
|
||||
max_spec_len = max([x[1].size(1) for x in batch])
|
||||
max_wav_len = max([x[2].size(1) for x in batch])
|
||||
|
||||
@@ -221,7 +227,7 @@ class TextAudioSpeakerCollate:
|
||||
tone_padded = torch.LongTensor(len(batch), max_text_len)
|
||||
language_padded = torch.LongTensor(len(batch), max_text_len)
|
||||
bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
ja_bert_padded = torch.FloatTensor(len(batch), 768, max_text_len)
|
||||
ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
|
||||
spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
|
||||
wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
|
||||
|
||||
@@ -340,7 +340,7 @@ class TextEncoder(nn.Module):
|
||||
self.language_emb = nn.Embedding(num_languages, hidden_channels)
|
||||
nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.ja_bert_proj = nn.Conv1d(768, hidden_channels, 1)
|
||||
self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
|
||||
self.encoder = attentions.Encoder(
|
||||
hidden_channels,
|
||||
|
||||
@@ -15,7 +15,7 @@ pypinyin
|
||||
cn2an
|
||||
gradio
|
||||
av
|
||||
mecab-python3
|
||||
pyopenjtalk
|
||||
loguru
|
||||
unidic-lite
|
||||
cmudict
|
||||
|
||||
@@ -11,7 +11,12 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
|
||||
Returns:
|
||||
List of integers corresponding to the symbols in the text
|
||||
"""
|
||||
phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
|
||||
phones = [] # _symbol_to_id[symbol] for symbol in cleaned_text
|
||||
for symbol in cleaned_text:
|
||||
try:
|
||||
phones.append(_symbol_to_id[symbol])
|
||||
except KeyError:
|
||||
phones.append(0) # symbol not found in ID map, use 0('_') by default
|
||||
tone_start = language_tone_start_map[language]
|
||||
tones = [i + tone_start for i in tones]
|
||||
lang_id = language_id_map[language]
|
||||
|
||||
216
text/japanese.py
216
text/japanese.py
@@ -2,17 +2,13 @@
|
||||
# compatible with Julius https://github.com/julius-speech/segmentation-kit
|
||||
import re
|
||||
import unicodedata
|
||||
import sys
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from text import punctuation, symbols
|
||||
|
||||
try:
|
||||
import MeCab
|
||||
except ImportError as e:
|
||||
raise ImportError("Japanese requires mecab-python3 and unidic-lite.") from e
|
||||
from num2words import num2words
|
||||
import pyopenjtalk
|
||||
|
||||
BERT = "./bert/bert-large-japanese-v2"
|
||||
_CONVRULES = [
|
||||
# Conversion of 2 letters
|
||||
"アァ/ a a",
|
||||
@@ -353,99 +349,6 @@ def hira2kata(text: str) -> str:
|
||||
return text.replace("う゛", "ヴ")
|
||||
|
||||
|
||||
_SYMBOL_TOKENS = set(list("・、。？！"))
|
||||
_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
|
||||
_TAGGER = MeCab.Tagger()
|
||||
|
||||
|
||||
def text2kata(text: str) -> str:
|
||||
parsed = _TAGGER.parse(text)
|
||||
res = []
|
||||
for line in parsed.split("\n"):
|
||||
if line == "EOS":
|
||||
break
|
||||
parts = line.split("\t")
|
||||
|
||||
word, yomi = parts[0], parts[1]
|
||||
if yomi:
|
||||
res.append(yomi)
|
||||
else:
|
||||
if word in _SYMBOL_TOKENS:
|
||||
res.append(word)
|
||||
elif word in ("っ", "ッ"):
|
||||
res.append("ッ")
|
||||
elif word in _NO_YOMI_TOKENS:
|
||||
pass
|
||||
else:
|
||||
res.append(word)
|
||||
return hira2kata("".join(res))
|
||||
|
||||
|
||||
_ALPHASYMBOL_YOMI = {
|
||||
"#": "シャープ",
|
||||
"%": "パーセント",
|
||||
"&": "アンド",
|
||||
"+": "プラス",
|
||||
"-": "マイナス",
|
||||
":": "コロン",
|
||||
";": "セミコロン",
|
||||
"<": "小なり",
|
||||
"=": "イコール",
|
||||
">": "大なり",
|
||||
"@": "アット",
|
||||
"a": "エー",
|
||||
"b": "ビー",
|
||||
"c": "シー",
|
||||
"d": "ディー",
|
||||
"e": "イー",
|
||||
"f": "エフ",
|
||||
"g": "ジー",
|
||||
"h": "エイチ",
|
||||
"i": "アイ",
|
||||
"j": "ジェー",
|
||||
"k": "ケー",
|
||||
"l": "エル",
|
||||
"m": "エム",
|
||||
"n": "エヌ",
|
||||
"o": "オー",
|
||||
"p": "ピー",
|
||||
"q": "キュー",
|
||||
"r": "アール",
|
||||
"s": "エス",
|
||||
"t": "ティー",
|
||||
"u": "ユー",
|
||||
"v": "ブイ",
|
||||
"w": "ダブリュー",
|
||||
"x": "エックス",
|
||||
"y": "ワイ",
|
||||
"z": "ゼット",
|
||||
"α": "アルファ",
|
||||
"β": "ベータ",
|
||||
"γ": "ガンマ",
|
||||
"δ": "デルタ",
|
||||
"ε": "イプシロン",
|
||||
"ζ": "ゼータ",
|
||||
"η": "イータ",
|
||||
"θ": "シータ",
|
||||
"ι": "イオタ",
|
||||
"κ": "カッパ",
|
||||
"λ": "ラムダ",
|
||||
"μ": "ミュー",
|
||||
"ν": "ニュー",
|
||||
"ξ": "クサイ",
|
||||
"ο": "オミクロン",
|
||||
"π": "パイ",
|
||||
"ρ": "ロー",
|
||||
"σ": "シグマ",
|
||||
"τ": "タウ",
|
||||
"υ": "ウプシロン",
|
||||
"φ": "ファイ",
|
||||
"χ": "カイ",
|
||||
"ψ": "プサイ",
|
||||
"ω": "オメガ",
|
||||
}
|
||||
|
||||
|
||||
_NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
|
||||
_CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
|
||||
_CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
|
||||
@@ -453,48 +356,12 @@ _NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
|
||||
|
||||
|
||||
def japanese_convert_numbers_to_words(text: str) -> str:
|
||||
res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
|
||||
res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
|
||||
res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
|
||||
res = text
|
||||
for x in _CURRENCY_MAP.keys():
|
||||
res = res.replace(x, _CURRENCY_MAP[x])
|
||||
return res
|
||||
|
||||
|
||||
def japanese_convert_alpha_symbols_to_words(text: str) -> str:
|
||||
return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()])
|
||||
|
||||
|
||||
def japanese_text_to_phonemes(text: str) -> str:
|
||||
"""Convert Japanese text to phonemes."""
|
||||
res = unicodedata.normalize("NFKC", text)
|
||||
res = japanese_convert_numbers_to_words(res)
|
||||
# res = japanese_convert_alpha_symbols_to_words(res)
|
||||
res = text2kata(res)
|
||||
res = kata2phoneme(res)
|
||||
return res
|
||||
|
||||
|
||||
def is_japanese_character(char):
|
||||
# 定义日语文字系统的 Unicode 范围
|
||||
japanese_ranges = [
|
||||
(0x3040, 0x309F), # 平假名
|
||||
(0x30A0, 0x30FF), # 片假名
|
||||
(0x4E00, 0x9FFF), # 汉字 (CJK Unified Ideographs)
|
||||
(0x3400, 0x4DBF), # 汉字扩展 A
|
||||
(0x20000, 0x2A6DF), # 汉字扩展 B
|
||||
# 可以根据需要添加其他汉字扩展范围
|
||||
]
|
||||
|
||||
# 将字符的 Unicode 编码转换为整数
|
||||
char_code = ord(char)
|
||||
|
||||
# 检查字符是否在任何一个日语范围内
|
||||
for start, end in japanese_ranges:
|
||||
if start <= char_code <= end:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
@@ -510,18 +377,9 @@ rep_map = {
|
||||
|
||||
|
||||
def replace_punctuation(text):
|
||||
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
||||
|
||||
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
||||
|
||||
replaced_text = re.sub(
|
||||
r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF"
|
||||
+ "".join(punctuation)
|
||||
+ r"]+",
|
||||
"",
|
||||
replaced_text,
|
||||
)
|
||||
|
||||
replaced_text = text
|
||||
for x in rep_map.keys():
|
||||
replaced_text = replaced_text.replace(x, rep_map[x])
|
||||
return replaced_text
|
||||
|
||||
|
||||
@@ -533,54 +391,42 @@ def text_normalize(text):
|
||||
return res
|
||||
|
||||
|
||||
def distribute_phone(n_phone, n_word):
|
||||
phones_per_word = [0] * n_word
|
||||
for task in range(n_phone):
|
||||
min_tasks = min(phones_per_word)
|
||||
min_index = phones_per_word.index(min_tasks)
|
||||
phones_per_word[min_index] += 1
|
||||
return phones_per_word
|
||||
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
|
||||
tokenizer = AutoTokenizer.from_pretrained(BERT)
|
||||
|
||||
|
||||
def g2p(norm_text):
|
||||
tokenized = tokenizer.tokenize(norm_text)
|
||||
phs = []
|
||||
ph_groups = []
|
||||
for t in tokenized:
|
||||
if not t.startswith("#"):
|
||||
ph_groups.append([t])
|
||||
else:
|
||||
ph_groups[-1].append(t.replace("#", ""))
|
||||
st = [x.replace("#", "") for x in tokenized]
|
||||
word2ph = []
|
||||
for group in ph_groups:
|
||||
phonemes = kata2phoneme(text2kata("".join(group)))
|
||||
# phonemes = [i for i in phonemes if i in symbols]
|
||||
for i in phonemes:
|
||||
assert i in symbols, (group, norm_text, tokenized)
|
||||
phone_len = len(phonemes)
|
||||
word_len = len(group)
|
||||
|
||||
aaa = distribute_phone(phone_len, word_len)
|
||||
word2ph += aaa
|
||||
|
||||
phs += phonemes
|
||||
phones = ["_"] + phs + ["_"]
|
||||
tones = [0 for i in phones]
|
||||
phs = pyopenjtalk.g2p(norm_text).split(
|
||||
" "
|
||||
) # Directly use the entire norm_text sequence.
|
||||
for sub in st: # the following code is only for calculating word2ph
|
||||
wph = 0
|
||||
for x in sub:
|
||||
sys.stdout.flush()
|
||||
if x not in ["?", ".", "!", "…", ","]: # This will throw warnings.
|
||||
phonemes = pyopenjtalk.g2p(x)
|
||||
else:
|
||||
phonemes = "pau"
|
||||
# for x in range(repeat):
|
||||
wph += len(phonemes.split(" "))
|
||||
# print(f'{x}-->:{phones}')
|
||||
word2ph.append(wph)
|
||||
phonemes = ["_"] + phs + ["_"]
|
||||
tones = [0 for i in phonemes]
|
||||
word2ph = [1] + word2ph + [1]
|
||||
return phones, tones, word2ph
|
||||
return phonemes, tones, word2ph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
|
||||
tokenizer = AutoTokenizer.from_pretrained(BERT)
|
||||
text = "hello,こんにちは、世界！……"
|
||||
from text.japanese_bert import get_bert_feature
|
||||
|
||||
text = text_normalize(text)
|
||||
print(text)
|
||||
phones, tones, word2ph = g2p(text)
|
||||
phones, tones, word2ph = g2p_ojt(text)
|
||||
bert = get_bert_feature(text, word2ph)
|
||||
|
||||
print(phones, tones, word2ph, bert.shape)
|
||||
|
||||
@@ -2,7 +2,12 @@ import torch
|
||||
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
||||
import sys
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
|
||||
BERT = "./bert/bert-large-japanese-v2"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(BERT)
|
||||
# bert-large model has 25 hidden layers.You can decide which layer to use by setting this variable to a specific value
|
||||
# default value is 3(untested)
|
||||
BERT_LAYER = 3
|
||||
|
||||
models = dict()
|
||||
|
||||
@@ -24,13 +29,13 @@ def get_bert_feature(text, word2ph, device=None):
|
||||
inputs = tokenizer(text, return_tensors="pt")
|
||||
for i in inputs:
|
||||
inputs[i] = inputs[i].to(device)
|
||||
res = models[device](**inputs, output_hidden_states=True)
|
||||
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
||||
res = model(**inputs, output_hidden_states=True)
|
||||
res = res["hidden_states"][BERT_LAYER]
|
||||
assert inputs["input_ids"].shape[-1] == len(word2ph)
|
||||
word2phone = word2ph
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
repeat_feature = res[i].repeat(word2phone[i], 1)
|
||||
repeat_feature = res[0][i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
|
||||
@@ -74,6 +74,7 @@ num_zh_tones = 6
|
||||
|
||||
# japanese
|
||||
ja_symbols = [
|
||||
"pau",
|
||||
"N",
|
||||
"a",
|
||||
"a:",
|
||||
@@ -117,6 +118,8 @@ ja_symbols = [
|
||||
"z",
|
||||
"zy",
|
||||
]
|
||||
for x in range(ord("a"), ord("z") + 1):
|
||||
ja_symbols.append(chr(x).upper())
|
||||
num_ja_tones = 1
|
||||
|
||||
# English
|
||||
|
||||
Reference in New Issue
Block a user