Fix: The conversion script for English BERT to Fast Tokenizer was incorrect, so tokenization was not performed correctly

2024-12-19 11:10:33 +09:00
parent c1fce3fec7
commit 2833fb5eeb
6 changed files with 1024100 additions and 256046 deletions
--- a/bert/chinese-roberta-wwm-ext-large-onnx/tokenizer.json
+++ b/bert/chinese-roberta-wwm-ext-large-onnx/tokenizer.json
@@ -6,9 +6,9 @@
  "normalizer": {
    "type": "BertNormalizer",
    "clean_text": true,
-    "handle_chinese_chars": false,
-    "strip_accents": false,
-    "lowercase": false
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
  },
  "pre_tokenizer": {
    "type": "BertPreTokenizer"
--- a/bert/chinese-roberta-wwm-ext-large/tokenizer.json
+++ b/bert/chinese-roberta-wwm-ext-large/tokenizer.json
@@ -6,9 +6,9 @@
  "normalizer": {
    "type": "BertNormalizer",
    "clean_text": true,
-    "handle_chinese_chars": false,
-    "strip_accents": false,
-    "lowercase": false
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
  },
  "pre_tokenizer": {
    "type": "BertPreTokenizer"
--- a/bert/deberta-v3-large-onnx/tokenizer.json
+++ b/bert/deberta-v3-large-onnx/tokenizer.json
--- a/bert/deberta-v3-large/tokenizer.json
+++ b/bert/deberta-v3-large/tokenizer.json