tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Refactor: Use I/O Binding during BERT inference and always release memory after inference

Browse Source
This commit is contained in:
tsukumi
2024-12-22 06:49:43 +09:00
parent 15af441cba
commit 08c439e88e
5 changed files with 223 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
style_bert_vits2/nlp/chinese/bert_feature.py
View File

@@ -3,10 +3,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Sequence, Union
import numpy as np
import onnxruntime
from numpy.typing import NDArray
from style_bert_vits2.constants import Languages
from style_bert_vits2.nlp import bert_models, onnx_bert_models
from style_bert_vits2.utils import get_onnx_device_options
if TYPE_CHECKING:
@@ -97,34 +99,59 @@ def extract_bert_feature_onnx(
NDArray[Any]: BERT の特徴量
"""
# トークナイザーとモデルの読み込み
tokenizer = onnx_bert_models.load_tokenizer(Languages.ZH)
inputs = tokenizer(text, return_tensors="np")
session = onnx_bert_models.load_model(
language=Languages.ZH,
onnx_providers=onnx_providers,
)
input_names = [input.name for input in session.get_inputs()]
output_name = session.get_outputs()[0].name
res = session.run(
[output_name],
{
"input_ids": inputs["input_ids"].astype(np.int64), # type: ignore
"token_type_ids": inputs["token_type_ids"].astype(np.int64), # type: ignore
"attention_mask": inputs["attention_mask"].astype(np.int64), # type: ignore
},
)[0]
# 入力テンソルの転送に使用するデバイス種別, デバイス ID, 実行オプションを取得
device_type, device_id, run_options = get_onnx_device_options(session, onnx_providers) # fmt: skip
# 入力をテンソルに変換
inputs = tokenizer(text, return_tensors="np")
input_tensor = [
inputs["input_ids"].astype(np.int64), # type: ignore
inputs["token_type_ids"].astype(np.int64), # type: ignore
inputs["attention_mask"].astype(np.int64), # type: ignore
]
# 推論デバイスに入力テンソルを割り当て
## GPU 推論の場合、device_type + device_id に対応する GPU デバイスに入力テンソルが割り当てられる
io_binding = session.io_binding()
for name, value in zip(input_names, input_tensor):
gpu_tensor = onnxruntime.OrtValue.ortvalue_from_numpy(
value, device_type, device_id
)
io_binding.bind_ortvalue_input(name, gpu_tensor)
# text から BERT 特徴量を抽出
io_binding.bind_output(output_name, device_type)
session.run_with_iobinding(io_binding, run_options=run_options)
res = io_binding.get_outputs()[0].numpy()
style_res_mean = None
if assist_text:
# 入力をテンソルに変換
style_inputs = tokenizer(assist_text, return_tensors="np")
style_res = session.run(
[output_name],
{
"input_ids": style_inputs["input_ids"].astype(np.int64), # type: ignore
"token_type_ids": style_inputs["token_type_ids"].astype(np.int64), # type: ignore
"attention_mask": style_inputs["attention_mask"].astype(np.int64), # type: ignore
},
)[0]
style_input_tensor = [
style_inputs["input_ids"].astype(np.int64), # type: ignore
style_inputs["token_type_ids"].astype(np.int64), # type: ignore
style_inputs["attention_mask"].astype(np.int64), # type: ignore
]
# 推論デバイスに入力テンソルを割り当て
## GPU 推論の場合、device_type + device_id に対応する GPU デバイスに入力テンソルが割り当てられる
io_binding = session.io_binding() # IOBinding は作り直す必要がある
for name, value in zip(input_names, style_input_tensor):
gpu_tensor = onnxruntime.OrtValue.ortvalue_from_numpy(
value, device_type, device_id
)
io_binding.bind_ortvalue_input(name, gpu_tensor)
# assist_text から BERT 特徴量を抽出
io_binding.bind_output(output_name, device_type)
session.run_with_iobinding(io_binding, run_options=run_options)
style_res = io_binding.get_outputs()[0].numpy()
style_res_mean = np.mean(style_res, axis=0)
assert len(word2ph) == len(text) + 2

59
style_bert_vits2/nlp/english/bert_feature.py
View File

@@ -3,10 +3,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Sequence, Union
import numpy as np
import onnxruntime
from numpy.typing import NDArray
from style_bert_vits2.constants import Languages
from style_bert_vits2.nlp import bert_models, onnx_bert_models
from style_bert_vits2.utils import get_onnx_device_options
if TYPE_CHECKING:
@@ -97,32 +99,57 @@ def extract_bert_feature_onnx(
NDArray[Any]: BERT の特徴量
"""
# トークナイザーとモデルの読み込み
tokenizer = onnx_bert_models.load_tokenizer(Languages.EN)
inputs = tokenizer(text, return_tensors="np")
session = onnx_bert_models.load_model(
language=Languages.EN,
onnx_providers=onnx_providers,
)
input_names = [input.name for input in session.get_inputs()]
output_name = session.get_outputs()[0].name
res = session.run(
[output_name],
{
"input_ids": inputs["input_ids"].astype(np.int64), # type: ignore
"attention_mask": inputs["attention_mask"].astype(np.int64), # type: ignore
},
)[0]
# 入力テンソルの転送に使用するデバイス種別, デバイス ID, 実行オプションを取得
device_type, device_id, run_options = get_onnx_device_options(session, onnx_providers) # fmt: skip
# 入力をテンソルに変換
inputs = tokenizer(text, return_tensors="np")
input_tensor = [
inputs["input_ids"].astype(np.int64), # type: ignore
inputs["attention_mask"].astype(np.int64), # type: ignore
]
# 推論デバイスに入力テンソルを割り当て
## GPU 推論の場合、device_type + device_id に対応する GPU デバイスに入力テンソルが割り当てられる
io_binding = session.io_binding()
for name, value in zip(input_names, input_tensor):
gpu_tensor = onnxruntime.OrtValue.ortvalue_from_numpy(
value, device_type, device_id
)
io_binding.bind_ortvalue_input(name, gpu_tensor)
# text から BERT 特徴量を抽出
io_binding.bind_output(output_name, device_type)
session.run_with_iobinding(io_binding, run_options=run_options)
res = io_binding.get_outputs()[0].numpy()
style_res_mean = None
if assist_text:
# 入力をテンソルに変換
style_inputs = tokenizer(assist_text, return_tensors="np")
style_res = session.run(
[output_name],
{
"input_ids": style_inputs["input_ids"].astype(np.int64), # type: ignore
"attention_mask": style_inputs["attention_mask"].astype(np.int64), # type: ignore
},
)[0]
style_input_tensor = [
style_inputs["input_ids"].astype(np.int64), # type: ignore
style_inputs["attention_mask"].astype(np.int64), # type: ignore
]
# 推論デバイスに入力テンソルを割り当て
## GPU 推論の場合、device_type + device_id に対応する GPU デバイスに入力テンソルが割り当てられる
io_binding = session.io_binding() # IOBinding は作り直す必要がある
for name, value in zip(input_names, style_input_tensor):
gpu_tensor = onnxruntime.OrtValue.ortvalue_from_numpy(
value, device_type, device_id
)
io_binding.bind_ortvalue_input(name, gpu_tensor)
# assist_text から BERT 特徴量を抽出
io_binding.bind_output(output_name, device_type)
session.run_with_iobinding(io_binding, run_options=run_options)
style_res = io_binding.get_outputs()[0].numpy()
style_res_mean = np.mean(style_res, axis=0)
assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))

59
style_bert_vits2/nlp/japanese/bert_feature.py
View File

@@ -3,11 +3,13 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Sequence, Union
import numpy as np
import onnxruntime
from numpy.typing import NDArray
from style_bert_vits2.constants import Languages
from style_bert_vits2.nlp import bert_models, onnx_bert_models
from style_bert_vits2.nlp.japanese.g2p import text_to_sep_kata
from style_bert_vits2.utils import get_onnx_device_options
if TYPE_CHECKING:
@@ -110,32 +112,57 @@ def extract_bert_feature_onnx(
if assist_text:
assist_text = "".join(text_to_sep_kata(assist_text, raise_yomi_error=False)[0])
# トークナイザーとモデルの読み込み
tokenizer = onnx_bert_models.load_tokenizer(Languages.JP)
inputs = tokenizer(text, return_tensors="np")
session = onnx_bert_models.load_model(
language=Languages.JP,
onnx_providers=onnx_providers,
)
input_names = [input.name for input in session.get_inputs()]
output_name = session.get_outputs()[0].name
res = session.run(
[output_name],
{
"input_ids": inputs["input_ids"].astype(np.int64), # type: ignore
"attention_mask": inputs["attention_mask"].astype(np.int64), # type: ignore
},
)[0]
# 入力テンソルの転送に使用するデバイス種別, デバイス ID, 実行オプションを取得
device_type, device_id, run_options = get_onnx_device_options(session, onnx_providers) # fmt: skip
# 入力をテンソルに変換
inputs = tokenizer(text, return_tensors="np")
input_tensor = [
inputs["input_ids"].astype(np.int64), # type: ignore
inputs["attention_mask"].astype(np.int64), # type: ignore
]
# 推論デバイスに入力テンソルを割り当て
## GPU 推論の場合、device_type + device_id に対応する GPU デバイスに入力テンソルが割り当てられる
io_binding = session.io_binding()
for name, value in zip(input_names, input_tensor):
gpu_tensor = onnxruntime.OrtValue.ortvalue_from_numpy(
value, device_type, device_id
)
io_binding.bind_ortvalue_input(name, gpu_tensor)
# text から BERT 特徴量を抽出
io_binding.bind_output(output_name, device_type)
session.run_with_iobinding(io_binding, run_options=run_options)
res = io_binding.get_outputs()[0].numpy()
style_res_mean = None
if assist_text:
# 入力をテンソルに変換
style_inputs = tokenizer(assist_text, return_tensors="np")
style_res = session.run(
[output_name],
{
"input_ids": style_inputs["input_ids"].astype(np.int64), # type: ignore
"attention_mask": style_inputs["attention_mask"].astype(np.int64), # type: ignore
},
)[0]
style_input_tensor = [
style_inputs["input_ids"].astype(np.int64), # type: ignore
style_inputs["attention_mask"].astype(np.int64), # type: ignore
]
# 推論デバイスに入力テンソルを割り当て
## GPU 推論の場合、device_type + device_id に対応する GPU デバイスに入力テンソルが割り当てられる
io_binding = session.io_binding() # IOBinding は作り直す必要がある
for name, value in zip(input_names, style_input_tensor):
gpu_tensor = onnxruntime.OrtValue.ortvalue_from_numpy(
value, device_type, device_id
)
io_binding.bind_ortvalue_input(name, gpu_tensor)
# assist_text から BERT 特徴量を抽出
io_binding.bind_output(output_name, device_type)
session.run_with_iobinding(io_binding, run_options=run_options)
style_res = io_binding.get_outputs()[0].numpy()
style_res_mean = np.mean(style_res, axis=0)
assert len(word2ph) == len(text) + 2, text