24#if defined(__unix__) || defined(__APPLE__)
33 "yarp.device.kokoroSpeechSynthesizer",
38const char*
const kBase64Alphabet =
39 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
41const std::map<std::string, yarp::sig::VectorOfString> kokoroVoices = {
42 {
"a", yarp::sig::VectorOfString{
64 {
"b", yarp::sig::VectorOfString{
74 {
"j", yarp::sig::VectorOfString{
81 {
"e", yarp::sig::VectorOfString{
86 {
"z", yarp::sig::VectorOfString{
96 {
"f", yarp::sig::VectorOfString{
99 {
"h", yarp::sig::VectorOfString{
105 {
"i", yarp::sig::VectorOfString{
109 {
"p", yarp::sig::VectorOfString{
118 std::ifstream file(path);
122std::string lower(std::string value)
124 std::transform(value.begin(), value.end(), value.begin(), [](
unsigned char c) {
125 return static_cast<char>(std::tolower(c));
151 m_voiceExplicit = config.
check(
"voice") &&
m_voice !=
"auto";
153 if (m_workerScript.empty()) {
154#ifdef KOKORO_WORKER_SCRIPT_SOURCE
155 if (
fileExists(KOKORO_WORKER_SCRIPT_SOURCE)) {
156 m_workerScript = KOKORO_WORKER_SCRIPT_SOURCE;
160 if (m_workerScript.empty()) {
161#ifdef KOKORO_WORKER_SCRIPT_INSTALL
162 m_workerScript = KOKORO_WORKER_SCRIPT_INSTALL;
165 if (m_workerScript.empty() || !
fileExists(m_workerScript)) {
175 if (!startWorker()) {
181 << m_currentLanguage <<
"and voice" << m_currentVoice;
187 std::lock_guard<std::mutex> lock(m_mutex);
189#
if defined(__unix__) || defined(__APPLE__)
200bool KokoroSpeechSynthesizer::startWorker()
202#if defined(__unix__) || defined(__APPLE__)
203 int parentToChild[2]{};
204 int childToParent[2]{};
205 if (pipe(parentToChild) != 0 || pipe(childToParent) != 0) {
210 const pid_t pid =
fork();
221 dup2(parentToChild[0], STDIN_FILENO);
222 dup2(childToParent[1], STDOUT_FILENO);
228 execlp(m_pythonExecutable.c_str(),
229 m_pythonExecutable.c_str(),
231 m_workerScript.c_str(),
234 m_modelRepository.empty() ?
"hexgrad/Kokoro-82M" : m_modelRepository.c_str(),
237 static_cast<char*>(nullptr));
243 m_workerInput = parentToChild[1];
244 m_workerOutput = childToParent[0];
248 if (!readLine(line, 120000) || line !=
"READY") {
260void KokoroSpeechSynthesizer::stopWorker()
262#if defined(__unix__) || defined(__APPLE__)
263 if (m_workerInput >= 0) {
267 if (m_workerOutput >= 0) {
271 if (m_workerPid >= 0) {
273 if (waitpid(m_workerPid, &status, WNOHANG) == 0) {
274 kill(m_workerPid, SIGTERM);
275 for (
int i = 0; i < 20 && waitpid(m_workerPid, &status, WNOHANG) == 0; ++i) {
278 if (waitpid(m_workerPid, &status, WNOHANG) == 0) {
279 kill(m_workerPid, SIGKILL);
280 waitpid(m_workerPid, &status, 0);
285 m_readBuffer.clear();
289bool KokoroSpeechSynthesizer::writeAll(
const std::string& data)
291#if defined(__unix__) || defined(__APPLE__)
293 while (offset < data.size()) {
294 const ssize_t written =
write(m_workerInput, data.data() + offset, data.size() - offset);
295 if (written < 0 && errno == EINTR) {
301 offset +=
static_cast<size_t>(written);
310bool KokoroSpeechSynthesizer::readLine(std::string& line,
int timeoutMs)
312#if defined(__unix__) || defined(__APPLE__)
318 const auto newline = std::find(m_readBuffer.begin(), m_readBuffer.end(),
'\n');
319 if (newline != m_readBuffer.end()) {
320 line.assign(m_readBuffer.begin(), newline);
321 m_readBuffer.erase(m_readBuffer.begin(), newline + 1);
324 struct pollfd descriptor{};
325 descriptor.fd = m_workerOutput;
326 descriptor.events = POLLIN;
327 const int ready = poll(&descriptor, 1, timeoutMs);
328 if (ready <= 0 || (descriptor.revents & (POLLERR | POLLHUP | POLLNVAL))) {
335 m_readBuffer.append(
buffer,
static_cast<size_t>(count));
344std::string KokoroSpeechSynthesizer::base64Encode(
const std::string& input)
347 result.reserve(((input.size() + 2) / 3) * 4);
348 for (
size_t i = 0; i < input.size(); i += 3) {
349 const uint32_t a =
static_cast<unsigned char>(input[i]);
350 const uint32_t b = i + 1 < input.size() ?
static_cast<unsigned char>(input[i + 1]) : 0;
351 const uint32_t c = i + 2 < input.size() ?
static_cast<unsigned char>(input[i + 2]) : 0;
352 const uint32_t value = (a << 16) | (b << 8) | c;
353 result += kBase64Alphabet[(value >> 18) & 63];
354 result += kBase64Alphabet[(value >> 12) & 63];
355 result += i + 1 < input.size() ? kBase64Alphabet[(value >> 6) & 63] :
'=';
356 result += i + 2 < input.size() ? kBase64Alphabet[value & 63] :
'=';
361bool KokoroSpeechSynthesizer::base64Decode(
const std::string& input,
362 std::vector<unsigned char>& output)
365 if (input.empty() || input.size() % 4 != 0) {
368 auto valueOf = [](
char c) ->
int {
369 const char* position = std::strchr(kBase64Alphabet, c);
370 return position ?
static_cast<int>(position - kBase64Alphabet) : -1;
372 output.reserve((input.size() / 4) * 3);
373 for (
size_t i = 0; i < input.size(); i += 4) {
374 const int a = valueOf(input[i]);
375 const int b = valueOf(input[i + 1]);
376 const int c = input[i + 2] ==
'=' ? 0 : valueOf(input[i + 2]);
377 const int d = input[i + 3] ==
'=' ? 0 : valueOf(input[i + 3]);
378 if (a < 0 || b < 0 || c < 0 || d < 0 || (input[i + 2] ==
'=' && input[i + 3] !=
'=')) {
381 const uint32_t value = (
static_cast<uint32_t
>(a) << 18)
382 | (
static_cast<uint32_t
>(b) << 12)
383 | (
static_cast<uint32_t
>(c) << 6)
384 |
static_cast<uint32_t
>(d);
385 output.push_back(
static_cast<unsigned char>((value >> 16) & 0xff));
386 if (input[i + 2] !=
'=') {
387 output.push_back(
static_cast<unsigned char>((value >> 8) & 0xff));
389 if (input[i + 3] !=
'=') {
390 output.push_back(
static_cast<unsigned char>(value & 0xff));
396bool KokoroSpeechSynthesizer::workerSynthesize(
const std::string& text,
399#if defined(__unix__) || defined(__APPLE__)
409 std::ostringstream speedText;
410 speedText.imbue(std::locale::classic());
411 speedText << m_currentSpeed;
412 const std::string request =
"SYNTH " + m_kokoroLanguage +
" " + m_currentVoice +
" "
413 + speedText.str() +
" " + base64Encode(text) +
"\n";
414 if (!writeAll(request)) {
419 std::string response;
420 if (!readLine(response, 300000)) {
424 std::istringstream stream(response);
427 if (status ==
"ERR") {
428 std::string encodedError;
429 stream >> encodedError;
430 std::vector<unsigned char> errorBytes;
431 base64Decode(encodedError, errorBytes);
433 << std::string(errorBytes.begin(), errorBytes.end());
436 if (status !=
"OK") {
443 std::string encodedAudio;
444 stream >> sampleRate >> channels >> encodedAudio;
445 if (!stream || sampleRate <= 0 || channels <= 0 || channels > 2) {
449 std::vector<unsigned char> pcm;
450 if (!base64Decode(encodedAudio, pcm) || pcm.empty() || pcm.size() % (2 * channels) != 0) {
455 const size_t frames = pcm.size() / (2 *
static_cast<size_t>(channels));
457 result.setFrequency(sampleRate);
458 result.resize(frames, channels);
459 for (
size_t frame = 0; frame < frames; ++frame) {
460 for (
int channel = 0; channel < channels; ++channel) {
461 const size_t offset = (frame *
static_cast<size_t>(channels) +
static_cast<size_t>(channel)) * 2;
462 const auto sample =
static_cast<int16_t
>(
static_cast<uint16_t
>(pcm[offset])
463 | (
static_cast<uint16_t
>(pcm[offset + 1]) << 8));
464 result.set(sample, frame, channel);
476bool KokoroSpeechSynthesizer::knownLanguage(
const std::string& language,
477 std::string& kokoroCode)
479 const std::string value = lower(language);
480 if (value ==
"auto" || value ==
"a" || value ==
"en" || value ==
"en-us" || value ==
"eng") {
482 }
else if (value ==
"b" || value ==
"en-gb") {
484 }
else if (value ==
"e" || value ==
"es" || value ==
"es-es") {
486 }
else if (value ==
"f" || value ==
"fr" || value ==
"fr-fr") {
488 }
else if (value ==
"h" || value ==
"hi" || value ==
"hi-in") {
490 }
else if (value ==
"i" || value ==
"it" || value ==
"it-it") {
492 }
else if (value ==
"j" || value ==
"ja" || value ==
"ja-jp") {
494 }
else if (value ==
"p" || value ==
"pt" || value ==
"pt-br") {
496 }
else if (value ==
"z" || value ==
"zh" || value ==
"zh-cn") {
504std::string KokoroSpeechSynthesizer::defaultVoiceFor(
const std::string& kokoroCode)
506 if (kokoroCode ==
"b")
return "bf_emma";
507 if (kokoroCode ==
"e")
return "ef_dora";
508 if (kokoroCode ==
"f")
return "ff_siwis";
509 if (kokoroCode ==
"h")
return "hf_alpha";
510 if (kokoroCode ==
"i")
return "if_sara";
511 if (kokoroCode ==
"j")
return "jf_alpha";
512 if (kokoroCode ==
"p")
return "pf_dora";
513 if (kokoroCode ==
"z")
return "zf_xiaobei";
520 if (!knownLanguage(language, code)) {
524 std::lock_guard<std::mutex> lock(m_mutex);
525 m_currentLanguage = language;
526 m_kokoroLanguage = code;
527 if (!m_voiceExplicit) {
528 m_currentVoice = defaultVoiceFor(code);
535 std::lock_guard<std::mutex> lock(m_mutex);
536 language = m_currentLanguage;
542 std::lock_guard<std::mutex> lock(m_mutex);
543 m_voiceExplicit = voice_name !=
"auto";
544 m_currentVoice = m_voiceExplicit ? voice_name : defaultVoiceFor(m_kokoroLanguage);
545 if (m_currentVoice.empty()) {
553 std::lock_guard<std::mutex> lock(m_mutex);
554 voice_name = m_currentVoice;
560 std::lock_guard<std::mutex> lock(m_mutex);
561 auto it = kokoroVoices.find(m_kokoroLanguage);
562 if (it != kokoroVoices.end()) {
571 if (!std::isfinite(speed) || speed <= 0.0 || speed > 4.0) {
575 std::lock_guard<std::mutex> lock(m_mutex);
576 m_currentSpeed = speed;
582 std::lock_guard<std::mutex> lock(m_mutex);
583 speed = m_currentSpeed;
589 if (!std::isfinite(pitch)) {
592 std::lock_guard<std::mutex> lock(m_mutex);
595 m_currentPitch = pitch;
601 std::lock_guard<std::mutex> lock(m_mutex);
602 pitch = m_currentPitch;
609 std::lock_guard<std::mutex> lock(m_mutex);
614 return workerSynthesize(text, sound)
const yarp::os::LogComponent & KOKOROSPEECHSYNTHESIZER()
static bool fileExists(const char *fname)
std::string m_python_executable
std::string m_worker_script
bool parseParams(const yarp::os::Searchable &config) override
Parse the DeviceDriver parameters.
std::string m_torch_device
std::string m_model_repository
yarp::dev::ReturnValue getLanguage(std::string &language) override
Gets the current language set for speech synthesis.
yarp::dev::ReturnValue setVoice(const std::string &voice_name="auto") override
Sets the voice set for speech synthesis.
yarp::dev::ReturnValue getAvailableVoices(yarp::sig::VectorOfString &voices) override
yarp::dev::ReturnValue getVoice(std::string &voice_name) override
Gets the current voice set for speech synthesis.
yarp::dev::ReturnValue getPitch(double &pitch) override
Gets the current pitch set for speech synthesis.
yarp::dev::ReturnValue synthesize(const std::string &text, yarp::sig::Sound &sound) override
Performs the speech synthesis.
bool open(yarp::os::Searchable &config) override
Open the DeviceDriver.
yarp::dev::ReturnValue setLanguage(const std::string &language="auto") override
Sets the language for speech synthesis.
yarp::dev::ReturnValue setSpeed(double speed=0) override
Sets the voice speed for speech synthesis.
yarp::dev::ReturnValue setPitch(double pitch) override
Sets the pitch for speech synthesis.
bool close() override
Close the DeviceDriver.
yarp::dev::ReturnValue getSpeed(double &speed) override
Gets the current voice speed.
~KokoroSpeechSynthesizer() override
@ return_value_error_generic
Method was successfully executed.
A base class for nested structures that can be searched.
virtual bool check(const std::string &key) const =0
Check if there exists a property of the given name.
Class for storing sounds See Audio in YARP for additional documentation on YARP audio.
#define yCInfo(component,...)
#define yCError(component,...)
#define YARP_LOG_COMPONENT(name,...)
int fork()
Portable wrapper for the fork() function.
bool read(ImageOf< PixelRgb > &dest, const std::string &src, image_fileformat format=FORMAT_ANY)
bool write(const ImageOf< PixelRgb > &src, const std::string &dest, image_fileformat format=FORMAT_PPM)