From c474bbe5af18e301ea297249fbcc9125356cc001 Mon Sep 17 00:00:00 2001
From: litagin02 <litagin02@gmail.com>
Date: Thu, 15 Feb 2024 22:55:08 +0900
Subject: [PATCH] Feat: pitch intonation changer

---
 app.py              | 20 ++++++++++++++++++
 common/tts_model.py | 51 +++++++++++++++++++++++++++++++++++++++------
 requirements.txt    |  1 +
 3 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/app.py b/app.py
index f9ceb5e..d2e9c19 100644
--- a/app.py
+++ b/app.py
@@ -57,6 +57,8 @@ def tts_fn(
     kata_tone_json_str,
     use_tone,
     speaker,
+    pitch_scale,
+    intonation_scale,
 ):
     model_holder.load_model_gr(model_name, model_path)
 
@@ -111,6 +113,8 @@ def tts_fn(
             style_weight=style_weight,
             given_tone=tone,
             sid=speaker_id,
+            pitch_scale=pitch_scale,
+            intonation_scale=intonation_scale,
         )
     except InvalidToneError as e:
         logger.error(f"Tone error: {e}")
@@ -306,6 +310,20 @@ if __name__ == "__main__":
                     refresh_button = gr.Button("更新", scale=1, visible=True)
                     load_button = gr.Button("ロード", scale=1, variant="primary")
                 text_input = gr.TextArea(label="テキスト", value=initial_text)
+                pitch_scale = gr.Slider(
+                    minimum=0,
+                    maximum=2,
+                    value=1,
+                    step=0.1,
+                    label="音程",
+                )
+                intonation_scale = gr.Slider(
+                    minimum=0,
+                    maximum=2,
+                    value=1,
+                    step=0.1,
+                    label="抑揚",
+                )
 
                 line_split = gr.Checkbox(
                     label="改行で分けて生成（分けたほうが感情が乗ります）",
@@ -441,6 +459,8 @@ if __name__ == "__main__":
                 tone,
                 use_tone,
                 speaker,
+                pitch_scale,
+                intonation_scale,
             ],
             outputs=[text_output, audio_output, tone],
         )
diff --git a/common/tts_model.py b/common/tts_model.py
index c148598..1bf5898 100644
--- a/common/tts_model.py
+++ b/common/tts_model.py
@@ -1,17 +1,18 @@
-import numpy as np
-import gradio as gr
-import torch
 import os
 import warnings
-from gradio.processing_utils import convert_to_16_bit_wav
 from typing import Dict, List, Optional, Union
 
+import gradio as gr
+import numpy as np
+import pyworld
+import torch
+from gradio.processing_utils import convert_to_16_bit_wav
+
 import utils
 from infer import get_net_g, infer
 from models import SynthesizerTrn
 from models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra
 
-from .log import logger
 from .constants import (
     DEFAULT_ASSIST_TEXT_WEIGHT,
     DEFAULT_LENGTH,
@@ -23,6 +24,36 @@ from .constants import (
     DEFAULT_STYLE,
     DEFAULT_STYLE_WEIGHT,
 )
+from .log import logger
+from torchfcpe import spawn_bundled_infer_model
+import librosa
+
+
+def adjust_voice(fs, wave, pitch_scale, intonation_scale):
+    if pitch_scale == 1.0 and intonation_scale == 1.0:
+        return fs, wave
+    # pyworldでf0を加工して合成
+    # pyworldよりもよいのがあるかもしれないが……
+
+    wave = wave.astype(np.double)
+    logger.debug(f"wave: shape={wave.shape}, max={max(wave)}, min={min(wave)}")
+    f0, t = pyworld.harvest(wave, fs)
+    # 質が高そうだしとりあえずharvestにしておく
+
+    sp = pyworld.cheaptrick(wave, f0, t, fs)
+    ap = pyworld.d4c(wave, f0, t, fs)
+
+    non_zero_f0 = [f for f in f0 if f != 0]
+    f0_mean = sum(non_zero_f0) / len(non_zero_f0)
+    logger.debug(f"f0: shape={f0.shape}, mean={f0_mean}, max={max(f0)}")
+
+    for i, f in enumerate(f0):
+        if f == 0:
+            continue
+        f0[i] = pitch_scale * f0_mean + intonation_scale * (f - f0_mean)
+
+    wave = pyworld.synthesize(f0, sp, ap, fs)
+    return fs, wave
 
 
 class Model:
@@ -97,6 +128,8 @@ class Model:
         style: str = DEFAULT_STYLE,
         style_weight: float = DEFAULT_STYLE_WEIGHT,
         given_tone: Optional[list[int]] = None,
+        pitch_scale: float = 1.0,
+        intonation_scale: float = 1.0,
     ) -> tuple[int, np.ndarray]:
         logger.info(f"Start generating audio data from text:\n{text}")
         if language != "JP" and self.hps.version.endswith("JP-Extra"):
@@ -165,7 +198,13 @@ class Model:
                 warnings.simplefilter("ignore")
                 audio = convert_to_16_bit_wav(audio)
         logger.info("Audio data generated successfully")
-        return (self.hps.data.sampling_rate, audio)
+        fs, audio = adjust_voice(
+            fs=self.hps.data.sampling_rate,
+            wave=audio,
+            pitch_scale=pitch_scale,
+            intonation_scale=intonation_scale,
+        )
+        return (fs, audio)
 
 
 class ModelHolder:
diff --git a/requirements.txt b/requirements.txt
index aa84daf..62b4d60 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,6 +19,7 @@ pyannote.audio>=3.1.0
 pyloudnorm
 pyopenjtalk-prebuilt
 pypinyin
+pyworld
 PyYAML
 requests
 safetensors