From e1ce12a336371d26f68b2825c28b3a9381eb6301 Mon Sep 17 00:00:00 2001
From: tsukumi <tsukumijima@users.noreply.github.com>
Date: Thu, 19 Dec 2024 13:41:09 +0900
Subject: [PATCH] Fix: Use Fast Tokenizer instead of Slow Tokenizer, which
 makes validation very slow

---
 convert_bert_onnx.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/convert_bert_onnx.py b/convert_bert_onnx.py
index e4f43a5..055e7b8 100644
--- a/convert_bert_onnx.py
+++ b/convert_bert_onnx.py
@@ -265,6 +265,9 @@ if __name__ == "__main__":
             res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
             return res
 
+    # 再度 Fast Tokenizer でロード
+    tokenizer = bert_models.load_tokenizer(language)
+
     # ONNX 変換用の BERT モデルをロード
     model = ONNXBert()
     inputs = tokenizer("今日はいい天気ですね", return_tensors="pt")