diff --git a/dict_data/.gitignore b/dict_data/.gitignore new file mode 100644 index 0000000..0b60cb6 --- /dev/null +++ b/dict_data/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!default.csv diff --git a/dict_data/default.csv b/dict_data/default.csv new file mode 100644 index 0000000..8869f97 --- /dev/null +++ b/dict_data/default.csv @@ -0,0 +1,3 @@ +Ｂｅｒｔ,,,8609,名詞,固有名詞,一般,*,*,*,Ｂｅｒｔ,バアト,バアト,0/3,* +ＶＩＴＳ,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ,ビッツ,ビッツ,0/3,* +ＶＩＴＳ２,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ２,ビッツツウ,ビッツツウ,4/5,* diff --git a/text/user_dict/LGPL_LICENSE b/text/user_dict/LGPL_LICENSE new file mode 100644 index 0000000..153d416 --- /dev/null +++ b/text/user_dict/LGPL_LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. \ No newline at end of file diff --git a/text/user_dict/README.md b/text/user_dict/README.md new file mode 100644 index 0000000..c7190ea --- /dev/null +++ b/text/user_dict/README.md @@ -0,0 +1,16 @@ +このモジュールは、[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)を使わせていただいています。 + +引用元: + +https://github.com/VOICEVOX/voicevox_engine/tree/709527be089c0410c08e989df95a4a1d78439423/voicevox_engine/user_dict + +改変部分は以下のとおりです。 +- ファイル名の書き換え、それに伴うimport文の書き換え +- VOICEVOX固有の他のモジュールへの依存のコメントアウト +- mutexを使用している部分のコメントアウト +- 参照しているpyopenjtalkの違いによるメソッド名書き換え +- UserDictWordのmora_countのデフォルト値をNoneに指定 + +ライセンス: LGPL + +[LGPL_LICENSE](LGPL_LICENSE)を参照してください。 diff --git a/text/user_dict/__init__.py b/text/user_dict/__init__.py new file mode 100644 index 0000000..d258359 --- /dev/null +++ b/text/user_dict/__init__.py @@ -0,0 +1,462 @@ +import json +import sys +import threading +import traceback +from pathlib import Path +from typing import Dict, List, Optional +from uuid import UUID, uuid4 + +import numpy as np +import pyopenjtalk +from fastapi import HTTPException + +from .word_model import UserDictWord, WordTypes + +# from ..utility.mutex_utility import mutex_wrapper +# from ..utility.path_utility import engine_root, get_save_dir +from .part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data +from common.constants import USER_DICT_DIR + +# root_dir = engine_root() +# save_dir = get_save_dir() +root_dir = Path(USER_DICT_DIR) +save_dir = Path(USER_DICT_DIR) + + +if not save_dir.is_dir(): + save_dir.mkdir(parents=True) + +default_dict_path = root_dir / "default.csv" # VOICEVOXデフォルト辞書ファイルのパス +user_dict_path = save_dir / "user_dict.json" # ユーザー辞書ファイルのパス +compiled_dict_path = save_dir / "user.dic" # コンパイル済み辞書ファイルのパス + + +# # 同時書き込みの制御 +# mutex_user_dict = threading.Lock() +# mutex_openjtalk_dict = threading.Lock() + + +# @mutex_wrapper(mutex_user_dict) +def _write_to_json(user_dict: Dict[str, UserDictWord], user_dict_path: Path) -> None: + """ + ユーザー辞書ファイルへのユーザー辞書データ書き込み + Parameters + ---------- + user_dict : Dict[str, UserDictWord] + ユーザー辞書データ + user_dict_path : Path + ユーザー辞書ファイルのパス + """ + converted_user_dict = {} + for word_uuid, word in user_dict.items(): + word_dict = word.dict() + word_dict["cost"] = _priority2cost( + word_dict["context_id"], word_dict["priority"] + ) + del word_dict["priority"] + converted_user_dict[word_uuid] = word_dict + # 予めjsonに変換できることを確かめる + user_dict_json = json.dumps(converted_user_dict, ensure_ascii=False) + + # ユーザー辞書ファイルへの書き込み + user_dict_path.write_text(user_dict_json, encoding="utf-8") + + +# @mutex_wrapper(mutex_openjtalk_dict) +def update_dict( + default_dict_path: Path = default_dict_path, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + 辞書の更新 + Parameters + ---------- + default_dict_path : Path + デフォルト辞書ファイルのパス + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + random_string = uuid4() + tmp_csv_path = compiled_dict_path.with_suffix( + f".dict_csv-{random_string}.tmp" + ) # csv形式辞書データの一時保存ファイル + tmp_compiled_path = compiled_dict_path.with_suffix( + f".dict_compiled-{random_string}.tmp" + ) # コンパイル済み辞書データの一時保存ファイル + + try: + # 辞書.csvを作成 + csv_text = "" + + # デフォルト辞書データの追加 + if not default_dict_path.is_file(): + print("Warning: Cannot find default dictionary.", file=sys.stderr) + return + default_dict = default_dict_path.read_text(encoding="utf-8") + if default_dict == default_dict.rstrip(): + default_dict += "\n" + csv_text += default_dict + + # ユーザー辞書データの追加 + user_dict = read_dict(user_dict_path=user_dict_path) + for word_uuid in user_dict: + word = user_dict[word_uuid] + csv_text += ( + "{surface},{context_id},{context_id},{cost},{part_of_speech}," + + "{part_of_speech_detail_1},{part_of_speech_detail_2}," + + "{part_of_speech_detail_3},{inflectional_type}," + + "{inflectional_form},{stem},{yomi},{pronunciation}," + + "{accent_type}/{mora_count},{accent_associative_rule}\n" + ).format( + surface=word.surface, + context_id=word.context_id, + cost=_priority2cost(word.context_id, word.priority), + part_of_speech=word.part_of_speech, + part_of_speech_detail_1=word.part_of_speech_detail_1, + part_of_speech_detail_2=word.part_of_speech_detail_2, + part_of_speech_detail_3=word.part_of_speech_detail_3, + inflectional_type=word.inflectional_type, + inflectional_form=word.inflectional_form, + stem=word.stem, + yomi=word.yomi, + pronunciation=word.pronunciation, + accent_type=word.accent_type, + mora_count=word.mora_count, + accent_associative_rule=word.accent_associative_rule, + ) + # 辞書データを辞書.csv へ一時保存 + tmp_csv_path.write_text(csv_text, encoding="utf-8") + + # 辞書.csvをOpenJTalk用にコンパイル + # pyopenjtalk.create_user_dict(str(tmp_csv_path), str(tmp_compiled_path)) + pyopenjtalk.mecab_dict_index(str(tmp_csv_path), str(tmp_compiled_path)) + if not tmp_compiled_path.is_file(): + raise RuntimeError("辞書のコンパイル時にエラーが発生しました。") + + # コンパイル済み辞書の置き換え・読み込み + pyopenjtalk.unset_user_dict() + tmp_compiled_path.replace(compiled_dict_path) + if compiled_dict_path.is_file(): + # pyopenjtalk.set_user_dict(str(compiled_dict_path.resolve(strict=True))) + pyopenjtalk.update_global_jtalk_with_user_dict(str(compiled_dict_path)) + + except Exception as e: + print("Error: Failed to update dictionary.", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + raise e + + finally: + # 後処理 + if tmp_csv_path.exists(): + tmp_csv_path.unlink() + if tmp_compiled_path.exists(): + tmp_compiled_path.unlink() + + +# @mutex_wrapper(mutex_user_dict) +def read_dict(user_dict_path: Path = user_dict_path) -> Dict[str, UserDictWord]: + """ + ユーザー辞書の読み出し + Parameters + ---------- + user_dict_path : Path + ユーザー辞書ファイルのパス + Returns + ------- + result : Dict[str, UserDictWord] + ユーザー辞書 + """ + # 指定ユーザー辞書が存在しない場合、空辞書を返す + if not user_dict_path.is_file(): + return {} + + with user_dict_path.open(encoding="utf-8") as f: + result: Dict[str, UserDictWord] = {} + for word_uuid, word in json.load(f).items(): + # cost2priorityで変換を行う際にcontext_idが必要となるが、 + # 0.12以前の辞書は、context_idがハードコーディングされていたためにユーザー辞書内に保管されていない + # ハードコーディングされていたcontext_idは固有名詞を意味するものなので、固有名詞のcontext_idを補完する + if word.get("context_id") is None: + word["context_id"] = part_of_speech_data[ + WordTypes.PROPER_NOUN + ].context_id + word["priority"] = _cost2priority(word["context_id"], word["cost"]) + del word["cost"] + result[str(UUID(word_uuid))] = UserDictWord(**word) + + return result + + +def _create_word( + surface: str, + pronunciation: str, + accent_type: int, + word_type: Optional[WordTypes] = None, + priority: Optional[int] = None, +) -> UserDictWord: + """ + 単語オブジェクトの生成 + Parameters + ---------- + surface : str + 単語情報 + pronunciation : str + 単語情報 + accent_type : int + 単語情報 + word_type : Optional[WordTypes] + 品詞 + priority : Optional[int] + 優先度 + Returns + ------- + : UserDictWord + 単語オブジェクト + """ + if word_type is None: + word_type = WordTypes.PROPER_NOUN + if word_type not in part_of_speech_data.keys(): + raise HTTPException(status_code=422, detail="不明な品詞です") + if priority is None: + priority = 5 + if not MIN_PRIORITY <= priority <= MAX_PRIORITY: + raise HTTPException(status_code=422, detail="優先度の値が無効です") + pos_detail = part_of_speech_data[word_type] + return UserDictWord( + surface=surface, + context_id=pos_detail.context_id, + priority=priority, + part_of_speech=pos_detail.part_of_speech, + part_of_speech_detail_1=pos_detail.part_of_speech_detail_1, + part_of_speech_detail_2=pos_detail.part_of_speech_detail_2, + part_of_speech_detail_3=pos_detail.part_of_speech_detail_3, + inflectional_type="*", + inflectional_form="*", + stem="*", + yomi=pronunciation, + pronunciation=pronunciation, + accent_type=accent_type, + accent_associative_rule="*", + ) + + +def apply_word( + surface: str, + pronunciation: str, + accent_type: int, + word_type: Optional[WordTypes] = None, + priority: Optional[int] = None, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> str: + """ + 新規単語の追加 + Parameters + ---------- + surface : str + 単語情報 + pronunciation : str + 単語情報 + accent_type : int + 単語情報 + word_type : Optional[WordTypes] + 品詞 + priority : Optional[int] + 優先度 + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + Returns + ------- + word_uuid : UserDictWord + 追加された単語に発行されたUUID + """ + # 新規単語の追加による辞書データの更新 + word = _create_word( + surface=surface, + pronunciation=pronunciation, + accent_type=accent_type, + word_type=word_type, + priority=priority, + ) + user_dict = read_dict(user_dict_path=user_dict_path) + word_uuid = str(uuid4()) + user_dict[word_uuid] = word + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict, user_dict_path) + update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path) + + return word_uuid + + +def rewrite_word( + word_uuid: str, + surface: str, + pronunciation: str, + accent_type: int, + word_type: Optional[WordTypes] = None, + priority: Optional[int] = None, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + 既存単語の上書き更新 + Parameters + ---------- + word_uuid : str + 単語UUID + surface : str + 単語情報 + pronunciation : str + 単語情報 + accent_type : int + 単語情報 + word_type : Optional[WordTypes] + 品詞 + priority : Optional[int] + 優先度 + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + word = _create_word( + surface=surface, + pronunciation=pronunciation, + accent_type=accent_type, + word_type=word_type, + priority=priority, + ) + + # 既存単語の上書きによる辞書データの更新 + user_dict = read_dict(user_dict_path=user_dict_path) + if word_uuid not in user_dict: + raise HTTPException( + status_code=422, detail="UUIDに該当するワードが見つかりませんでした" + ) + user_dict[word_uuid] = word + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict, user_dict_path) + update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path) + + +def delete_word( + word_uuid: str, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + 単語の削除 + Parameters + ---------- + word_uuid : str + 単語UUID + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + # 既存単語の削除による辞書データの更新 + user_dict = read_dict(user_dict_path=user_dict_path) + if word_uuid not in user_dict: + raise HTTPException( + status_code=422, detail="IDに該当するワードが見つかりませんでした" + ) + del user_dict[word_uuid] + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict, user_dict_path) + update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path) + + +def import_user_dict( + dict_data: Dict[str, UserDictWord], + override: bool = False, + user_dict_path: Path = user_dict_path, + default_dict_path: Path = default_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + ユーザー辞書のインポート + Parameters + ---------- + dict_data : Dict[str, UserDictWord] + インポートするユーザー辞書のデータ + override : bool + 重複したエントリがあった場合、上書きするかどうか + user_dict_path : Path + ユーザー辞書ファイルのパス + default_dict_path : Path + デフォルト辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + # インポートする辞書データのバリデーション + for word_uuid, word in dict_data.items(): + UUID(word_uuid) + assert isinstance(word, UserDictWord) + for pos_detail in part_of_speech_data.values(): + if word.context_id == pos_detail.context_id: + assert word.part_of_speech == pos_detail.part_of_speech + assert ( + word.part_of_speech_detail_1 == pos_detail.part_of_speech_detail_1 + ) + assert ( + word.part_of_speech_detail_2 == pos_detail.part_of_speech_detail_2 + ) + assert ( + word.part_of_speech_detail_3 == pos_detail.part_of_speech_detail_3 + ) + assert ( + word.accent_associative_rule in pos_detail.accent_associative_rules + ) + break + else: + raise ValueError("対応していない品詞です") + + # 既存辞書の読み出し + old_dict = read_dict(user_dict_path=user_dict_path) + + # 辞書データの更新 + # 重複エントリの上書き + if override: + new_dict = {**old_dict, **dict_data} + # 重複エントリの保持 + else: + new_dict = {**dict_data, **old_dict} + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict=new_dict, user_dict_path=user_dict_path) + update_dict( + default_dict_path=default_dict_path, + user_dict_path=user_dict_path, + compiled_dict_path=compiled_dict_path, + ) + + +def _search_cost_candidates(context_id: int) -> List[int]: + for value in part_of_speech_data.values(): + if value.context_id == context_id: + return value.cost_candidates + raise HTTPException(status_code=422, detail="品詞IDが不正です") + + +def _cost2priority(context_id: int, cost: int) -> int: + assert -32768 <= cost <= 32767 + cost_candidates = _search_cost_candidates(context_id) + # cost_candidatesの中にある値で最も近い値を元にpriorityを返す + # 参考: https://qiita.com/Krypf/items/2eada91c37161d17621d + # この関数とpriority2cost関数によって、辞書ファイルのcostを操作しても最も近いpriorityのcostに上書きされる + return MAX_PRIORITY - np.argmin(np.abs(np.array(cost_candidates) - cost)).item() + + +def _priority2cost(context_id: int, priority: int) -> int: + assert MIN_PRIORITY <= priority <= MAX_PRIORITY + cost_candidates = _search_cost_candidates(context_id) + return cost_candidates[MAX_PRIORITY - priority] diff --git a/text/user_dict/part_of_speech_data.py b/text/user_dict/part_of_speech_data.py new file mode 100644 index 0000000..3e80115 --- /dev/null +++ b/text/user_dict/part_of_speech_data.py @@ -0,0 +1,144 @@ +from typing import Dict + +from .word_model import ( + USER_DICT_MAX_PRIORITY, + USER_DICT_MIN_PRIORITY, + PartOfSpeechDetail, + WordTypes, +) + +MIN_PRIORITY = USER_DICT_MIN_PRIORITY +MAX_PRIORITY = USER_DICT_MAX_PRIORITY + +part_of_speech_data: Dict[WordTypes, PartOfSpeechDetail] = { + WordTypes.PROPER_NOUN: PartOfSpeechDetail( + part_of_speech="名詞", + part_of_speech_detail_1="固有名詞", + part_of_speech_detail_2="一般", + part_of_speech_detail_3="*", + context_id=1348, + cost_candidates=[ + -988, + 3488, + 4768, + 6048, + 7328, + 8609, + 8734, + 8859, + 8984, + 9110, + 14176, + ], + accent_associative_rules=[ + "*", + "C1", + "C2", + "C3", + "C4", + "C5", + ], + ), + WordTypes.COMMON_NOUN: PartOfSpeechDetail( + part_of_speech="名詞", + part_of_speech_detail_1="一般", + part_of_speech_detail_2="*", + part_of_speech_detail_3="*", + context_id=1345, + cost_candidates=[ + -4445, + 49, + 1473, + 2897, + 4321, + 5746, + 6554, + 7362, + 8170, + 8979, + 15001, + ], + accent_associative_rules=[ + "*", + "C1", + "C2", + "C3", + "C4", + "C5", + ], + ), + WordTypes.VERB: PartOfSpeechDetail( + part_of_speech="動詞", + part_of_speech_detail_1="自立", + part_of_speech_detail_2="*", + part_of_speech_detail_3="*", + context_id=642, + cost_candidates=[ + 3100, + 6160, + 6360, + 6561, + 6761, + 6962, + 7414, + 7866, + 8318, + 8771, + 13433, + ], + accent_associative_rules=[ + "*", + ], + ), + WordTypes.ADJECTIVE: PartOfSpeechDetail( + part_of_speech="形容詞", + part_of_speech_detail_1="自立", + part_of_speech_detail_2="*", + part_of_speech_detail_3="*", + context_id=20, + cost_candidates=[ + 1527, + 3266, + 3561, + 3857, + 4153, + 4449, + 5149, + 5849, + 6549, + 7250, + 10001, + ], + accent_associative_rules=[ + "*", + ], + ), + WordTypes.SUFFIX: PartOfSpeechDetail( + part_of_speech="名詞", + part_of_speech_detail_1="接尾", + part_of_speech_detail_2="一般", + part_of_speech_detail_3="*", + context_id=1358, + cost_candidates=[ + 4399, + 5373, + 6041, + 6710, + 7378, + 8047, + 9440, + 10834, + 12228, + 13622, + 15847, + ], + accent_associative_rules=[ + "*", + "C1", + "C2", + "C3", + "C4", + "C5", + ], + ), +} diff --git a/text/user_dict/word_model.py b/text/user_dict/word_model.py new file mode 100644 index 0000000..fda5576 --- /dev/null +++ b/text/user_dict/word_model.py @@ -0,0 +1,124 @@ +from enum import Enum +from re import findall, fullmatch +from typing import List, Optional + +from pydantic import BaseModel, Field, validator + +USER_DICT_MIN_PRIORITY = 0 +USER_DICT_MAX_PRIORITY = 10 + + +class UserDictWord(BaseModel): + """ + 辞書のコンパイルに使われる情報 + """ + + surface: str = Field(title="表層形") + priority: int = Field( + title="優先度", ge=USER_DICT_MIN_PRIORITY, le=USER_DICT_MAX_PRIORITY + ) + context_id: int = Field(title="文脈ID", default=1348) + part_of_speech: str = Field(title="品詞") + part_of_speech_detail_1: str = Field(title="品詞細分類1") + part_of_speech_detail_2: str = Field(title="品詞細分類2") + part_of_speech_detail_3: str = Field(title="品詞細分類3") + inflectional_type: str = Field(title="活用型") + inflectional_form: str = Field(title="活用形") + stem: str = Field(title="原形") + yomi: str = Field(title="読み") + pronunciation: str = Field(title="発音") + accent_type: int = Field(title="アクセント型") + mora_count: Optional[int] = Field(title="モーラ数", default=None) + accent_associative_rule: str = Field(title="アクセント結合規則") + + class Config: + validate_assignment = True + + @validator("surface") + def convert_to_zenkaku(cls, surface): + return surface.translate( + str.maketrans( + "".join(chr(0x21 + i) for i in range(94)), + "".join(chr(0xFF01 + i) for i in range(94)), + ) + ) + + @validator("pronunciation", pre=True) + def check_is_katakana(cls, pronunciation): + if not fullmatch(r"[ァ-ヴー]+", pronunciation): + raise ValueError("発音は有効なカタカナでなくてはいけません。") + sutegana = ["ァ", "ィ", "ゥ", "ェ", "ォ", "ャ", "ュ", "ョ", "ヮ", "ッ"] + for i in range(len(pronunciation)): + if pronunciation[i] in sutegana: + # 「キャット」のように、捨て仮名が連続する可能性が考えられるので、 + # 「ッ」に関しては「ッ」そのものが連続している場合と、「ッ」の後にほかの捨て仮名が連続する場合のみ無効とする + if i < len(pronunciation) - 1 and ( + pronunciation[i + 1] in sutegana[:-1] + or ( + pronunciation[i] == sutegana[-1] + and pronunciation[i + 1] == sutegana[-1] + ) + ): + raise ValueError("無効な発音です。(捨て仮名の連続)") + if pronunciation[i] == "ヮ": + if i != 0 and pronunciation[i - 1] not in ["ク", "グ"]: + raise ValueError( + "無効な発音です。(「くゎ」「ぐゎ」以外の「ゎ」の使用)" + ) + return pronunciation + + @validator("mora_count", pre=True, always=True) + def check_mora_count_and_accent_type(cls, mora_count, values): + if "pronunciation" not in values or "accent_type" not in values: + # 適切な場所でエラーを出すようにする + return mora_count + + if mora_count is None: + rule_others = ( + "[イ][ェ]|[ヴ][ャュョ]|[トド][ゥ]|[テデ][ィャュョ]|[デ][ェ]|[クグ][ヮ]" + ) + rule_line_i = "[キシチニヒミリギジビピ][ェャュョ]" + rule_line_u = "[ツフヴ][ァ]|[ウスツフヴズ][ィ]|[ウツフヴ][ェォ]" + rule_one_mora = "[ァ-ヴー]" + mora_count = len( + findall( + f"(?:{rule_others}|{rule_line_i}|{rule_line_u}|{rule_one_mora})", + values["pronunciation"], + ) + ) + + if not 0 <= values["accent_type"] <= mora_count: + raise ValueError( + "誤ったアクセント型です({})。 expect: 0 <= accent_type <= {}".format( + values["accent_type"], mora_count + ) + ) + return mora_count + + +class PartOfSpeechDetail(BaseModel): + """ + 品詞ごとの情報 + """ + + part_of_speech: str = Field(title="品詞") + part_of_speech_detail_1: str = Field(title="品詞細分類1") + part_of_speech_detail_2: str = Field(title="品詞細分類2") + part_of_speech_detail_3: str = Field(title="品詞細分類3") + # context_idは辞書の左・右文脈IDのこと + # https://github.com/VOICEVOX/open_jtalk/blob/427cfd761b78efb6094bea3c5bb8c968f0d711ab/src/mecab-naist-jdic/_left-id.def # noqa + context_id: int = Field(title="文脈ID") + cost_candidates: List[int] = Field(title="コストのパーセンタイル") + accent_associative_rules: List[str] = Field(title="アクセント結合規則の一覧") + + +class WordTypes(str, Enum): + """ + fastapiでword_type引数を検証する時に使用するクラス + """ + + PROPER_NOUN = "PROPER_NOUN" + COMMON_NOUN = "COMMON_NOUN" + VERB = "VERB" + ADJECTIVE = "ADJECTIVE" + SUFFIX = "SUFFIX"