Auto download missing model for bert_gen.py (#146)

* auto download missing model

* support openi

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix wrong delete

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* pass pre-commit

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix repeat login

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Isotr0py
2023-11-04 04:06:10 +08:00
committed by GitHub
parent 4d6de240a0
commit 8609449b63
9 changed files with 92 additions and 25 deletions

View File

@@ -1,9 +1,13 @@
import torch
import sys
from transformers import AutoTokenizer, AutoModelForMaskedLM
import torch
from transformers import AutoModelForMaskedLM, AutoTokenizer
from config import config
tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large")
LOCAL_PATH = "./bert/chinese-roberta-wwm-ext-large"
tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
models = dict()
@@ -18,9 +22,7 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
if not device:
device = "cuda"
if device not in models.keys():
models[device] = AutoModelForMaskedLM.from_pretrained(
"./bert/chinese-roberta-wwm-ext-large"
).to(device)
models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
with torch.no_grad():
inputs = tokenizer(text, return_tensors="pt")
for i in inputs:
@@ -41,8 +43,6 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
if __name__ == "__main__":
import torch
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
word2phone = [
1,