Fix: The conversion script for English BERT to Fast Tokenizer was incorrect, so tokenization was not performed correctly

convert_bert_onnx.py

@@ -15,8 +15,8 @@ from rich import print
 from rich.rule import Rule
 from rich.style import Style
 from torch import nn
-from transformers import PreTrainedTokenizerBase
-from transformers.convert_slow_tokenizer import BertConverter
+from transformers import AutoTokenizer, DebertaV2Tokenizer, PreTrainedTokenizerBase
+from transformers.convert_slow_tokenizer import BertConverter, convert_slow_tokenizer
 
 from style_bert_vits2.constants import DEFAULT_BERT_MODEL_PATHS, Languages
 from style_bert_vits2.nlp import bert_models
@@ -227,9 +227,23 @@ if __name__ == "__main__":
     print(Rule(characters="=", style=Style(color="blue")))
 
     # トークナイザーを Fast Tokenizer 用形式に変換して保存
-    tokenizer = bert_models.load_tokenizer(language)
-    converter = BertConverter(tokenizer)
-    converter.converted().save(str(tokenizer_json_path))
+    if language == Languages.EN:
+        slow_tokenizer = DebertaV2Tokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+        )
+        convert_slow_tokenizer(slow_tokenizer).save(str(tokenizer_json_path))
+    elif language == Languages.JP:
+        slow_tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            use_fast=False,  # 明示的に Slow Tokenizer を使う
+        )
+        BertConverter(slow_tokenizer).converted().save(str(tokenizer_json_path))
+    elif language == Languages.ZH:
+        slow_tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            use_fast=False,  # 明示的に Slow Tokenizer を使う
+        )
+        convert_slow_tokenizer(slow_tokenizer).save(str(tokenizer_json_path))
     print(Rule(characters="=", style=Style(color="blue")))
     print(f"[bold green]Tokenizer JSON saved to {tokenizer_json_path}[/bold green]")
     print(Rule(characters="=", style=Style(color="blue")))