init (not checked bat script yet)
This commit is contained in:
litagin02
34
bert/bert-base-japanese-v3/.gitattributes
vendored
34
bert/bert-base-japanese-v3/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,53 +0,0 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
datasets:
|
||||
- cc100
|
||||
- wikipedia
|
||||
language:
|
||||
- ja
|
||||
widget:
|
||||
- text: 東北大学で[MASK]の研究をしています。
|
||||
---
|
||||
|
||||
# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
|
||||
|
||||
This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
|
||||
|
||||
This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
|
||||
Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
|
||||
|
||||
The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
|
||||
|
||||
## Model architecture
|
||||
|
||||
The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
|
||||
|
||||
## Training Data
|
||||
|
||||
The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
|
||||
For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
|
||||
The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
|
||||
|
||||
For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
|
||||
|
||||
## Tokenization
|
||||
|
||||
The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
|
||||
The vocabulary size is 32768.
|
||||
|
||||
We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
|
||||
|
||||
## Training
|
||||
|
||||
We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
|
||||
For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
|
||||
|
||||
For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
|
||||
|
||||
## Licenses
|
||||
|
||||
The pretrained models are distributed under the Apache License 2.0.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"architectures": [
|
||||
"BertForPreTraining"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 768,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"layer_norm_eps": 1e-12,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "bert",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 12,
|
||||
"pad_token_id": 0,
|
||||
"type_vocab_size": 2,
|
||||
"vocab_size": 32768
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"tokenizer_class": "BertJapaneseTokenizer",
|
||||
"model_max_length": 512,
|
||||
"do_lower_case": false,
|
||||
"word_tokenizer_type": "mecab",
|
||||
"subword_tokenizer_type": "wordpiece",
|
||||
"mecab_kwargs": {
|
||||
"mecab_dic": "unidic_lite"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
34
bert/bert-large-japanese-v2/.gitattributes
vendored
34
bert/bert-large-japanese-v2/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,53 +0,0 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
datasets:
|
||||
- cc100
|
||||
- wikipedia
|
||||
language:
|
||||
- ja
|
||||
widget:
|
||||
- text: 東北大学で[MASK]の研究をしています。
|
||||
---
|
||||
|
||||
# BERT large Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
|
||||
|
||||
This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
|
||||
|
||||
This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
|
||||
Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
|
||||
|
||||
The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
|
||||
|
||||
## Model architecture
|
||||
|
||||
The model architecture is the same as the original BERT large model; 24 layers, 1024 dimensions of hidden states, and 16 attention heads.
|
||||
|
||||
## Training Data
|
||||
|
||||
The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
|
||||
For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
|
||||
The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
|
||||
|
||||
For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
|
||||
|
||||
## Tokenization
|
||||
|
||||
The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
|
||||
The vocabulary size is 32768.
|
||||
|
||||
We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
|
||||
|
||||
## Training
|
||||
|
||||
We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
|
||||
For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
|
||||
|
||||
For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
|
||||
|
||||
## Licenses
|
||||
|
||||
The pretrained models are distributed under the Apache License 2.0.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"architectures": [
|
||||
"BertForPreTraining"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"layer_norm_eps": 1e-12,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "bert",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"pad_token_id": 0,
|
||||
"type_vocab_size": 2,
|
||||
"vocab_size": 32768
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"tokenizer_class": "BertJapaneseTokenizer",
|
||||
"model_max_length": 512,
|
||||
"do_lower_case": false,
|
||||
"word_tokenizer_type": "mecab",
|
||||
"subword_tokenizer_type": "wordpiece",
|
||||
"mecab_kwargs": {
|
||||
"mecab_dic": "unidic_lite"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
34
bert/deberta-v2-large-japanese/.gitattributes
vendored
34
bert/deberta-v2-large-japanese/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,111 +0,0 @@
|
||||
---
|
||||
language: ja
|
||||
license: cc-by-sa-4.0
|
||||
library_name: transformers
|
||||
tags:
|
||||
- deberta
|
||||
- deberta-v2
|
||||
- fill-mask
|
||||
datasets:
|
||||
- wikipedia
|
||||
- cc100
|
||||
- oscar
|
||||
metrics:
|
||||
- accuracy
|
||||
mask_token: "[MASK]"
|
||||
widget:
|
||||
- text: "京都 大学 で 自然 言語 処理 を [MASK] する 。"
|
||||
---
|
||||
|
||||
# Model Card for Japanese DeBERTa V2 large
|
||||
|
||||
## Model description
|
||||
|
||||
This is a Japanese DeBERTa V2 large model pre-trained on Japanese Wikipedia, the Japanese portion of CC-100, and the
|
||||
Japanese portion of OSCAR.
|
||||
|
||||
## How to use
|
||||
|
||||
You can use this model for masked language modeling as follows:
|
||||
|
||||
```python
|
||||
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-large-japanese')
|
||||
model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-large-japanese')
|
||||
|
||||
sentence = '京都 大学 で 自然 言語 処理 を [MASK] する 。' # input should be segmented into words by Juman++ in advance
|
||||
encoding = tokenizer(sentence, return_tensors='pt')
|
||||
...
|
||||
```
|
||||
|
||||
You can also fine-tune this model on downstream tasks.
|
||||
|
||||
## Tokenization
|
||||
|
||||
The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in
|
||||
advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each
|
||||
word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).
|
||||
|
||||
## Training data
|
||||
|
||||
We used the following corpora for pre-training:
|
||||
|
||||
- Japanese Wikipedia (as of 20221020, 3.2GB, 27M sentences, 1.3M documents)
|
||||
- Japanese portion of CC-100 (85GB, 619M sentences, 66M documents)
|
||||
- Japanese portion of OSCAR (54GB, 326M sentences, 25M documents)
|
||||
|
||||
Note that we filtered out documents annotated with "header", "footer", or "noisy" tags in OSCAR.
|
||||
Also note that Japanese Wikipedia was duplicated 10 times to make the total size of the corpus comparable to that of
|
||||
CC-100 and OSCAR. As a result, the total size of the training data is 171GB.
|
||||
|
||||
## Training procedure
|
||||
|
||||
We first segmented texts in the corpora into words using [Juman++](https://github.com/ku-nlp/jumanpp).
|
||||
Then, we built a sentencepiece model with 32000 tokens including words ([JumanDIC](https://github.com/ku-nlp/JumanDIC))
|
||||
and subwords induced by the unigram language model of [sentencepiece](https://github.com/google/sentencepiece).
|
||||
|
||||
We tokenized the segmented corpora into subwords using the sentencepiece model and trained the Japanese DeBERTa model
|
||||
using [transformers](https://github.com/huggingface/transformers) library.
|
||||
The training took 36 days using 8 NVIDIA A100-SXM4-40GB GPUs.
|
||||
|
||||
The following hyperparameters were used during pre-training:
|
||||
|
||||
- learning_rate: 1e-4
|
||||
- per_device_train_batch_size: 18
|
||||
- distributed_type: multi-GPU
|
||||
- num_devices: 8
|
||||
- gradient_accumulation_steps: 16
|
||||
- total_train_batch_size: 2,304
|
||||
- max_seq_length: 512
|
||||
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-06
|
||||
- lr_scheduler_type: linear schedule with warmup
|
||||
- training_steps: 300,000
|
||||
- warmup_steps: 10,000
|
||||
|
||||
The accuracy of the trained model on the masked language modeling task was 0.799.
|
||||
The evaluation set consists of 5,000 randomly sampled documents from each of the training corpora.
|
||||
|
||||
## Fine-tuning on NLU tasks
|
||||
|
||||
We fine-tuned the following models and evaluated them on the dev set of JGLUE.
|
||||
We tuned learning rate and training epochs for each model and task
|
||||
following [the JGLUE paper](https://www.jstage.jst.go.jp/article/jnlp/30/1/30_63/_pdf/-char/ja).
|
||||
|
||||
| Model | MARC-ja/acc | JSTS/pearson | JSTS/spearman | JNLI/acc | JSQuAD/EM | JSQuAD/F1 | JComQA/acc |
|
||||
|-------------------------------|-------------|--------------|---------------|----------|-----------|-----------|------------|
|
||||
| Waseda RoBERTa base | 0.965 | 0.913 | 0.876 | 0.905 | 0.853 | 0.916 | 0.853 |
|
||||
| Waseda RoBERTa large (seq512) | 0.969 | 0.925 | 0.890 | 0.928 | 0.910 | 0.955 | 0.900 |
|
||||
| LUKE Japanese base* | 0.965 | 0.916 | 0.877 | 0.912 | - | - | 0.842 |
|
||||
| LUKE Japanese large* | 0.965 | 0.932 | 0.902 | 0.927 | - | - | 0.893 |
|
||||
| DeBERTaV2 base | 0.970 | 0.922 | 0.886 | 0.922 | 0.899 | 0.951 | 0.873 |
|
||||
| DeBERTaV2 large | 0.968 | 0.925 | 0.892 | 0.924 | 0.912 | 0.959 | 0.890 |
|
||||
|
||||
*The scores of LUKE are from [the official repository](https://github.com/studio-ousia/luke).
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This work was supported by Joint Usage/Research Center for Interdisciplinary Large-scale Information Infrastructures (
|
||||
JHPCN) through General Collaboration Project no. jh221004, "Developing a Platform for Constructing and Sharing of
|
||||
Large-Scale Japanese Language Models".
|
||||
For training models, we used the mdx: a platform for the data-driven future.
|
||||
@@ -1,38 +0,0 @@
|
||||
{
|
||||
"_name_or_path": "configs/deberta_v2_large.json",
|
||||
"architectures": [
|
||||
"DebertaV2ForMaskedLM"
|
||||
],
|
||||
"attention_head_size": 64,
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"conv_act": "gelu",
|
||||
"conv_kernel_size": 3,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"layer_norm_eps": 1e-07,
|
||||
"max_position_embeddings": 512,
|
||||
"max_relative_positions": -1,
|
||||
"model_type": "deberta-v2",
|
||||
"norm_rel_ebd": "layer_norm",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"pad_token_id": 0,
|
||||
"pooler_dropout": 0,
|
||||
"pooler_hidden_act": "gelu",
|
||||
"pooler_hidden_size": 1024,
|
||||
"pos_att_type": [
|
||||
"p2c",
|
||||
"c2p"
|
||||
],
|
||||
"position_biased_input": false,
|
||||
"position_buckets": 256,
|
||||
"relative_attention": true,
|
||||
"share_att_key": true,
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.23.1",
|
||||
"type_vocab_size": 0,
|
||||
"vocab_size": 32000
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"bos_token": "[CLS]",
|
||||
"cls_token": "[CLS]",
|
||||
"eos_token": "[SEP]",
|
||||
"mask_token": "[MASK]",
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"bos_token": "[CLS]",
|
||||
"cls_token": "[CLS]",
|
||||
"do_lower_case": false,
|
||||
"eos_token": "[SEP]",
|
||||
"keep_accents": true,
|
||||
"mask_token": "[MASK]",
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"sp_model_kwargs": {},
|
||||
"special_tokens_map_file": null,
|
||||
"split_by_punct": false,
|
||||
"tokenizer_class": "DebertaV2Tokenizer",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
Reference in New Issue
Block a user