Refactor: rewrote Japanese natural language processing code imported from server_editor.py

The logic has not been changed, only renaming, splitting and moving modules on a per-function basis.
Existing code will be left in place for the time being to avoid breaking the training code, which is not subject to refactoring this time.
This commit is contained in:
tsukumi
2024-03-06 20:56:21 +00:00
parent f26def4369
commit 918d168ae7
11 changed files with 1312 additions and 13 deletions

View File

@@ -0,0 +1,47 @@
import sys
import tempfile
from typing import TextIO
class StdoutWrapper(TextIO):
"""
`sys.stdout` wrapper for both Google Colab and local environment.
"""
def __init__(self) -> None:
self.temp_file = tempfile.NamedTemporaryFile(
mode="w+", delete=False, encoding="utf-8"
)
self.original_stdout = sys.stdout
def write(self, message: str) -> int:
result = self.temp_file.write(message)
self.temp_file.flush()
print(message, end="", file=self.original_stdout)
return result
def flush(self) -> None:
self.temp_file.flush()
def read(self, n: int = -1) -> str:
self.temp_file.seek(0)
return self.temp_file.read(n)
def close(self) -> None:
self.temp_file.close()
def fileno(self) -> int:
return self.temp_file.fileno()
try:
import google.colab # type: ignore
SAFE_STDOUT = StdoutWrapper()
except ImportError:
SAFE_STDOUT = sys.stdout