Fix: The conversion script for English BERT to Fast Tokenizer was incorrect, so tokenization was not performed correctly

bert/chinese-roberta-wwm-ext-large-onnx/tokenizer.json

@@ -6,9 +6,9 @@
   "normalizer": {
     "type": "BertNormalizer",
     "clean_text": true,
-    "handle_chinese_chars": false,
-    "strip_accents": false,
-    "lowercase": false
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
   },
   "pre_tokenizer": {
     "type": "BertPreTokenizer"

bert/chinese-roberta-wwm-ext-large/tokenizer.json

@@ -6,9 +6,9 @@
   "normalizer": {
     "type": "BertNormalizer",
     "clean_text": true,
-    "handle_chinese_chars": false,
-    "strip_accents": false,
-    "lowercase": false
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
   },
   "pre_tokenizer": {
     "type": "BertPreTokenizer"

convert_bert_onnx.py

@@ -15,8 +15,8 @@ from rich import print
 from rich.rule import Rule
 from rich.style import Style
 from torch import nn
-from transformers import PreTrainedTokenizerBase
-from transformers.convert_slow_tokenizer import BertConverter
+from transformers import AutoTokenizer, DebertaV2Tokenizer, PreTrainedTokenizerBase
+from transformers.convert_slow_tokenizer import BertConverter, convert_slow_tokenizer
 
 from style_bert_vits2.constants import DEFAULT_BERT_MODEL_PATHS, Languages
 from style_bert_vits2.nlp import bert_models
@@ -227,9 +227,23 @@ if __name__ == "__main__":
     print(Rule(characters="=", style=Style(color="blue")))
 
     # トークナイザーを Fast Tokenizer 用形式に変換して保存
-    tokenizer = bert_models.load_tokenizer(language)
-    converter = BertConverter(tokenizer)
-    converter.converted().save(str(tokenizer_json_path))
+    if language == Languages.EN:
+        slow_tokenizer = DebertaV2Tokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+        )
+        convert_slow_tokenizer(slow_tokenizer).save(str(tokenizer_json_path))
+    elif language == Languages.JP:
+        slow_tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            use_fast=False,  # 明示的に Slow Tokenizer を使う
+        )
+        BertConverter(slow_tokenizer).converted().save(str(tokenizer_json_path))
+    elif language == Languages.ZH:
+        slow_tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            use_fast=False,  # 明示的に Slow Tokenizer を使う
+        )
+        convert_slow_tokenizer(slow_tokenizer).save(str(tokenizer_json_path))
     print(Rule(characters="=", style=Style(color="blue")))
     print(f"[bold green]Tokenizer JSON saved to {tokenizer_json_path}[/bold green]")
     print(Rule(characters="=", style=Style(color="blue")))

style_bert_vits2/nlp/bert_models.py

@@ -142,7 +142,7 @@ def load_tokenizer(
         revision (str): モデルの Hugging Face 上の Git リビジョン。指定しない場合は最新の main ブランチの内容が利用される (デフォルト: None)
 
     Returns:
-        Union[PreTrainedTokenizer, PreTrainedTokenizerFast, DebertaV2Tokenizer]: ロード済みの BERT トークナイザー
+        Union[PreTrainedTokenizer, PreTrainedTokenizerFast, DebertaV2TokenizerFast]: ロード済みの BERT トークナイザー
     """
 
     # すでにロード済みの場合はそのまま返す
@@ -251,8 +251,6 @@ def unload_tokenizer(language: Languages) -> None:
         language (Languages): アンロードする BERT トークナイザーの言語
     """
 
-    import torch
-
     if language in __loaded_tokenizers:
         del __loaded_tokenizers[language]
         gc.collect()