Auto download missing model for bert_gen.py (#146)

* auto download missing model

* support openi

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix wrong delete

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* pass pre-commit

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix repeat login

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

text/__init__.py

@@ -26,3 +26,23 @@ def get_bert(norm_text, word2ph, language, device):
     lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
     bert = lang_bert_func_map[language](norm_text, word2ph, device)
     return bert
+
+
+def check_bert_models():
+    import json
+    from pathlib import Path
+
+    from config import config
+    from .bert_utils import _check_bert
+
+    if config.mirror.lower() == "openi":
+        import openi
+
+        kwargs = {"token": config.openi_token} if config.openi_token else {}
+        openi.login(**kwargs)
+
+    with open("./bert/bert_models.json", "r") as fp:
+        models = json.load(fp)
+        for k, v in models.items():
+            local_path = Path("./bert").joinpath(k)
+            _check_bert(v["repo_id"], v["files"], local_path)

text/bert_utils.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from huggingface_hub import hf_hub_download
+
+from config import config
+
+
+MIRROR: str = config.mirror
+
+
+def _check_bert(repo_id, files, local_path):
+    for file in files:
+        if not Path(local_path).joinpath(file).exists():
+            if MIRROR.lower() == "openi":
+                import openi
+
+                openi.model.download_model(
+                    "Stardust_minus/Bert-VITS2", repo_id.split("/")[-1], "./bert"
+                )
+            else:
+                hf_hub_download(
+                    repo_id, file, local_dir=local_path, local_dir_use_symlinks=False
+                )

text/chinese_bert.py

@@ -1,9 +1,13 @@
-import torch
 import sys
-from transformers import AutoTokenizer, AutoModelForMaskedLM
+
+import torch
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
 from config import config
 
-tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large")
+LOCAL_PATH = "./bert/chinese-roberta-wwm-ext-large"
+
+tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
 
 models = dict()
 
@@ -18,9 +22,7 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
     if not device:
         device = "cuda"
     if device not in models.keys():
-        models[device] = AutoModelForMaskedLM.from_pretrained(
-            "./bert/chinese-roberta-wwm-ext-large"
-        ).to(device)
+        models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
     with torch.no_grad():
         inputs = tokenizer(text, return_tensors="pt")
         for i in inputs:
@@ -41,8 +43,6 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
 
 
 if __name__ == "__main__":
-    import torch
-
     word_level_feature = torch.rand(38, 1024)  # 12个词,每个词1024维特征
     word2phone = [
         1,

text/english_bert_mock.py

@@ -1,9 +1,14 @@
-import torch
-from transformers import DebertaV2Model, DebertaV2Tokenizer
-from config import config
 import sys
 
-tokenizer = DebertaV2Tokenizer.from_pretrained("./bert/deberta-v3-large")
+import torch
+from transformers import DebertaV2Model, DebertaV2Tokenizer
+
+from config import config
+
+
+LOCAL_PATH = "./bert/deberta-v3-large"
+
+tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
 
 models = dict()
 
@@ -18,9 +23,7 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
     if not device:
         device = "cuda"
     if device not in models.keys():
-        models[device] = DebertaV2Model.from_pretrained("./bert/deberta-v3-large").to(
-            device
-        )
+        models[device] = DebertaV2Model.from_pretrained(LOCAL_PATH).to(device)
     with torch.no_grad():
         inputs = tokenizer(text, return_tensors="pt")
         for i in inputs:

text/japanese_bert.py

@@ -1,10 +1,14 @@
-import torch
-from transformers import AutoTokenizer, AutoModelForMaskedLM
 import sys
-from text.japanese import text2sep_kata
-from config import config
 
-tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese")
+import torch
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+from config import config
+from text.japanese import text2sep_kata
+
+LOCAL_PATH = "./bert/deberta-v2-large-japanese"
+
+tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
 
 models = dict()
 
@@ -27,9 +31,7 @@ def get_bert_feature_with_token(tokens, word2ph, device=config.bert_gen_config.d
     if not device:
         device = "cuda"
     if device not in models.keys():
-        models[device] = AutoModelForMaskedLM.from_pretrained(
-            "./bert/deberta-v2-large-japanese"
-        ).to(device)
+        models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
     with torch.no_grad():
         inputs = torch.tensor(tokens).to(device).unsqueeze(0)
         token_type_ids = torch.zeros_like(inputs).to(device)