Improve: Support ONNX inference, add ONNX conversion script

2024-09-17 16:55:03 +09:00
parent f42b9f0f89
commit 5e2c83c6c7
21 changed files with 44483 additions and 376 deletions
--- a/bert/deberta-v2-large-japanese-char-wwm-onnx/tokenizer_config.json
+++ b/bert/deberta-v2-large-japanese-char-wwm-onnx/tokenizer_config.json
@@ -0,0 +1,19 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "do_subword_tokenize": true,
+  "do_word_tokenize": true,
+  "jumanpp_kwargs": null,
+  "mask_token": "[MASK]",
+  "mecab_kwargs": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "subword_tokenizer_type": "character",
+  "sudachi_kwargs": null,
+  "tokenizer_class": "BertJapaneseTokenizer",
+  "unk_token": "[UNK]",
+  "word_tokenizer_type": "basic"
+}