YARP
Yet Another Robot Platform
 
Loading...
Searching...
No Matches
KokoroSpeechSynthesizer.cpp
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: 2026 Istituto Italiano di Tecnologia (IIT)
3 * SPDX-License-Identifier: BSD-3-Clause
4 */
5
7
9#include <yarp/os/LogStream.h>
10#include <yarp/os/Value.h>
11
12#include <algorithm>
13#include <cerrno>
14#include <cmath>
15#include <cstdint>
16#include <cctype>
17#include <cstring>
18#include <fstream>
19#include <locale>
20#include <sstream>
21#include <map>
22
23
24#if defined(__unix__) || defined(__APPLE__)
25# include <fcntl.h>
26# include <poll.h>
27# include <signal.h>
28# include <sys/wait.h>
29# include <unistd.h>
30#endif
31
33 "yarp.device.kokoroSpeechSynthesizer",
35
36namespace {
37
38const char* const kBase64Alphabet =
39 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
40
41const std::map<std::string, yarp::sig::VectorOfString> kokoroVoices = {
42 {"a", yarp::sig::VectorOfString{
43 "af_heart",
44 "af_alloy",
45 "af_aoede",
46 "af_bella",
47 "af_jessicas",
48 "af_kore",
49 "af_nicole",
50 "af_nova",
51 "af_river",
52 "af_sarah",
53 "af_sky",
54 "am_adam",
55 "am_echo",
56 "am_eric",
57 "am_fenrir",
58 "am_liam",
59 "am_michael",
60 "am_onyx",
61 "am_puck",
62 "am_santa"
63 }},
64 {"b", yarp::sig::VectorOfString{
65 "bf_alice",
66 "bf_emma",
67 "bf_isabella",
68 "bf_lily",
69 "bm_daniel",
70 "bm_fable",
71 "bm_george",
72 "bm_lewis"
73 }},
74 {"j", yarp::sig::VectorOfString{
75 "jf_alpha",
76 "jf_gongitsune",
77 "jf_nezumi",
78 "jf_tebukuro",
79 "jm_kumo"
80 }},
81 {"e", yarp::sig::VectorOfString{
82 "ef_dora",
83 "em_alex",
84 "em_santa"
85 }},
86 {"z", yarp::sig::VectorOfString{
87 "zf_xiaobei",
88 "zf_xiaoni",
89 "zf_xiaoxiao",
90 "zf_xiaoyi",
91 "zm_yunjian",
92 "zm_yunxi",
93 "zm_yunxia",
94 "zm_yunyang"
95 }},
96 {"f", yarp::sig::VectorOfString{
97 "ff_siwis"
98 }},
99 {"h", yarp::sig::VectorOfString{
100 "hf_alpha",
101 "hf_beta",
102 "hm_omega",
103 "hm_psi"
104 }},
105 {"i", yarp::sig::VectorOfString{
106 "if_sara",
107 "im_nicola"
108 }},
109 {"p", yarp::sig::VectorOfString{
110 "pf_dora",
111 "pm_alex",
112 "pm_santa"
113 }}
114};
115
116bool fileExists(const std::string& path)
117{
118 std::ifstream file(path);
119 return file.good();
120}
121
122std::string lower(std::string value)
123{
124 std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
125 return static_cast<char>(std::tolower(c));
126 });
127 return value;
128}
129
130} // namespace
131
136
138{
139 if (m_open) {
140 return true;
141 }
142
143 if (!parseParams(config)) {
144 yCError(KOKOROSPEECHSYNTHESIZER) << "Failed to parse device parameters";
145 return false;
146 }
147 m_pythonExecutable = m_python_executable;
148 m_workerScript = m_worker_script;
149 m_modelRepository = m_model_repository;
150 m_device = m_torch_device;
151 m_voiceExplicit = config.check("voice") && m_voice != "auto";
152
153 if (m_workerScript.empty()) {
154#ifdef KOKORO_WORKER_SCRIPT_SOURCE
155 if (fileExists(KOKORO_WORKER_SCRIPT_SOURCE)) {
156 m_workerScript = KOKORO_WORKER_SCRIPT_SOURCE;
157 }
158#endif
159 }
160 if (m_workerScript.empty()) {
161#ifdef KOKORO_WORKER_SCRIPT_INSTALL
162 m_workerScript = KOKORO_WORKER_SCRIPT_INSTALL;
163#endif
164 }
165 if (m_workerScript.empty() || !fileExists(m_workerScript)) {
166 yCError(KOKOROSPEECHSYNTHESIZER) << "Kokoro worker script not found. Set the 'worker_script' parameter.";
167 return false;
168 }
169
171 || !setSpeed(m_speed) || !setPitch(m_pitch)) {
172 return false;
173 }
174
175 if (!startWorker()) {
176 yCError(KOKOROSPEECHSYNTHESIZER) << "Unable to start Kokoro worker";
177 return false;
178 }
179 m_open = true;
180 yCInfo(KOKOROSPEECHSYNTHESIZER) << "Kokoro speech synthesizer opened with language"
181 << m_currentLanguage << "and voice" << m_currentVoice;
182 return true;
183}
184
186{
187 std::lock_guard<std::mutex> lock(m_mutex);
188 if (!m_open
189#if defined(__unix__) || defined(__APPLE__)
190 && m_workerPid < 0
191#endif
192 ) {
193 return true;
194 }
195 stopWorker();
196 m_open = false;
197 return true;
198}
199
200bool KokoroSpeechSynthesizer::startWorker()
201{
202#if defined(__unix__) || defined(__APPLE__)
203 int parentToChild[2]{};
204 int childToParent[2]{};
205 if (pipe(parentToChild) != 0 || pipe(childToParent) != 0) {
206 yCError(KOKOROSPEECHSYNTHESIZER) << "pipe() failed:" << std::strerror(errno);
207 return false;
208 }
209
210 const pid_t pid = fork();
211 if (pid < 0) {
212 yCError(KOKOROSPEECHSYNTHESIZER) << "fork() failed:" << std::strerror(errno);
213 ::close(parentToChild[0]);
214 ::close(parentToChild[1]);
215 ::close(childToParent[0]);
216 ::close(childToParent[1]);
217 return false;
218 }
219
220 if (pid == 0) {
221 dup2(parentToChild[0], STDIN_FILENO);
222 dup2(childToParent[1], STDOUT_FILENO);
223 ::close(parentToChild[0]);
224 ::close(parentToChild[1]);
225 ::close(childToParent[0]);
226 ::close(childToParent[1]);
227
228 execlp(m_pythonExecutable.c_str(),
229 m_pythonExecutable.c_str(),
230 "-u",
231 m_workerScript.c_str(),
232 "--stdio",
233 "--repo-id",
234 m_modelRepository.empty() ? "hexgrad/Kokoro-82M" : m_modelRepository.c_str(),
235 "--device",
236 m_device.c_str(),
237 static_cast<char*>(nullptr));
238 _exit(127);
239 }
240
241 ::close(parentToChild[0]);
242 ::close(childToParent[1]);
243 m_workerInput = parentToChild[1];
244 m_workerOutput = childToParent[0];
245 m_workerPid = pid;
246
247 std::string line;
248 if (!readLine(line, 120000) || line != "READY") {
249 yCError(KOKOROSPEECHSYNTHESIZER) << "Kokoro worker did not become ready:" << line;
250 stopWorker();
251 return false;
252 }
253 return true;
254#else
255 yCError(KOKOROSPEECHSYNTHESIZER) << "The Kokoro device currently requires POSIX process support";
256 return false;
257#endif
258}
259
260void KokoroSpeechSynthesizer::stopWorker()
261{
262#if defined(__unix__) || defined(__APPLE__)
263 if (m_workerInput >= 0) {
264 ::close(m_workerInput);
265 m_workerInput = -1;
266 }
267 if (m_workerOutput >= 0) {
268 ::close(m_workerOutput);
269 m_workerOutput = -1;
270 }
271 if (m_workerPid >= 0) {
272 int status = 0;
273 if (waitpid(m_workerPid, &status, WNOHANG) == 0) {
274 kill(m_workerPid, SIGTERM);
275 for (int i = 0; i < 20 && waitpid(m_workerPid, &status, WNOHANG) == 0; ++i) {
276 usleep(10000);
277 }
278 if (waitpid(m_workerPid, &status, WNOHANG) == 0) {
279 kill(m_workerPid, SIGKILL);
280 waitpid(m_workerPid, &status, 0);
281 }
282 }
283 m_workerPid = -1;
284 }
285 m_readBuffer.clear();
286#endif
287}
288
289bool KokoroSpeechSynthesizer::writeAll(const std::string& data)
290{
291#if defined(__unix__) || defined(__APPLE__)
292 size_t offset = 0;
293 while (offset < data.size()) {
294 const ssize_t written = write(m_workerInput, data.data() + offset, data.size() - offset);
295 if (written < 0 && errno == EINTR) {
296 continue;
297 }
298 if (written <= 0) {
299 return false;
300 }
301 offset += static_cast<size_t>(written);
302 }
303 return true;
304#else
305 (void)data;
306 return false;
307#endif
308}
309
310bool KokoroSpeechSynthesizer::readLine(std::string& line, int timeoutMs)
311{
312#if defined(__unix__) || defined(__APPLE__)
313 line.clear();
314 char buffer[4096];
315 while (true) {
316 // Bytes after a newline belong to the next line: keep them for the
317 // next call instead of dropping them.
318 const auto newline = std::find(m_readBuffer.begin(), m_readBuffer.end(), '\n');
319 if (newline != m_readBuffer.end()) {
320 line.assign(m_readBuffer.begin(), newline);
321 m_readBuffer.erase(m_readBuffer.begin(), newline + 1);
322 return true;
323 }
324 struct pollfd descriptor{};
325 descriptor.fd = m_workerOutput;
326 descriptor.events = POLLIN;
327 const int ready = poll(&descriptor, 1, timeoutMs);
328 if (ready <= 0 || (descriptor.revents & (POLLERR | POLLHUP | POLLNVAL))) {
329 return false;
330 }
331 const ssize_t count = read(m_workerOutput, buffer, sizeof(buffer));
332 if (count <= 0) {
333 return false;
334 }
335 m_readBuffer.append(buffer, static_cast<size_t>(count));
336 }
337#else
338 (void)line;
339 (void)timeoutMs;
340 return false;
341#endif
342}
343
344std::string KokoroSpeechSynthesizer::base64Encode(const std::string& input)
345{
346 std::string result;
347 result.reserve(((input.size() + 2) / 3) * 4);
348 for (size_t i = 0; i < input.size(); i += 3) {
349 const uint32_t a = static_cast<unsigned char>(input[i]);
350 const uint32_t b = i + 1 < input.size() ? static_cast<unsigned char>(input[i + 1]) : 0;
351 const uint32_t c = i + 2 < input.size() ? static_cast<unsigned char>(input[i + 2]) : 0;
352 const uint32_t value = (a << 16) | (b << 8) | c;
353 result += kBase64Alphabet[(value >> 18) & 63];
354 result += kBase64Alphabet[(value >> 12) & 63];
355 result += i + 1 < input.size() ? kBase64Alphabet[(value >> 6) & 63] : '=';
356 result += i + 2 < input.size() ? kBase64Alphabet[value & 63] : '=';
357 }
358 return result;
359}
360
361bool KokoroSpeechSynthesizer::base64Decode(const std::string& input,
362 std::vector<unsigned char>& output)
363{
364 output.clear();
365 if (input.empty() || input.size() % 4 != 0) {
366 return false;
367 }
368 auto valueOf = [](char c) -> int {
369 const char* position = std::strchr(kBase64Alphabet, c);
370 return position ? static_cast<int>(position - kBase64Alphabet) : -1;
371 };
372 output.reserve((input.size() / 4) * 3);
373 for (size_t i = 0; i < input.size(); i += 4) {
374 const int a = valueOf(input[i]);
375 const int b = valueOf(input[i + 1]);
376 const int c = input[i + 2] == '=' ? 0 : valueOf(input[i + 2]);
377 const int d = input[i + 3] == '=' ? 0 : valueOf(input[i + 3]);
378 if (a < 0 || b < 0 || c < 0 || d < 0 || (input[i + 2] == '=' && input[i + 3] != '=')) {
379 return false;
380 }
381 const uint32_t value = (static_cast<uint32_t>(a) << 18)
382 | (static_cast<uint32_t>(b) << 12)
383 | (static_cast<uint32_t>(c) << 6)
384 | static_cast<uint32_t>(d);
385 output.push_back(static_cast<unsigned char>((value >> 16) & 0xff));
386 if (input[i + 2] != '=') {
387 output.push_back(static_cast<unsigned char>((value >> 8) & 0xff));
388 }
389 if (input[i + 3] != '=') {
390 output.push_back(static_cast<unsigned char>(value & 0xff));
391 }
392 }
393 return true;
394}
395
396bool KokoroSpeechSynthesizer::workerSynthesize(const std::string& text,
397 yarp::sig::Sound& sound)
398{
399#if defined(__unix__) || defined(__APPLE__)
400 if (text.empty()) {
401 yCError(KOKOROSPEECHSYNTHESIZER) << "Cannot synthesize empty text";
402 return false;
403 }
404 // Language and voice are restricted to the Kokoro alphabet and are sent
405 // as plain fields. Text is base64 encoded so arbitrary UTF-8 is safe.
406 // The speed is formatted with the classic locale: std::to_string follows
407 // the C locale and would emit "1,500000" under e.g. it_IT, which Python's
408 // float() rejects.
409 std::ostringstream speedText;
410 speedText.imbue(std::locale::classic());
411 speedText << m_currentSpeed;
412 const std::string request = "SYNTH " + m_kokoroLanguage + " " + m_currentVoice + " "
413 + speedText.str() + " " + base64Encode(text) + "\n";
414 if (!writeAll(request)) {
415 yCError(KOKOROSPEECHSYNTHESIZER) << "Unable to write to Kokoro worker";
416 return false;
417 }
418
419 std::string response;
420 if (!readLine(response, 300000)) {
421 yCError(KOKOROSPEECHSYNTHESIZER) << "Timed out waiting for Kokoro worker";
422 return false;
423 }
424 std::istringstream stream(response);
425 std::string status;
426 stream >> status;
427 if (status == "ERR") {
428 std::string encodedError;
429 stream >> encodedError;
430 std::vector<unsigned char> errorBytes;
431 base64Decode(encodedError, errorBytes);
432 yCError(KOKOROSPEECHSYNTHESIZER) << "Kokoro synthesis failed:"
433 << std::string(errorBytes.begin(), errorBytes.end());
434 return false;
435 }
436 if (status != "OK") {
437 yCError(KOKOROSPEECHSYNTHESIZER) << "Malformed response from Kokoro worker";
438 return false;
439 }
440
441 int sampleRate = 0;
442 int channels = 0;
443 std::string encodedAudio;
444 stream >> sampleRate >> channels >> encodedAudio;
445 if (!stream || sampleRate <= 0 || channels <= 0 || channels > 2) {
446 yCError(KOKOROSPEECHSYNTHESIZER) << "Invalid audio metadata from Kokoro worker";
447 return false;
448 }
449 std::vector<unsigned char> pcm;
450 if (!base64Decode(encodedAudio, pcm) || pcm.empty() || pcm.size() % (2 * channels) != 0) {
451 yCError(KOKOROSPEECHSYNTHESIZER) << "Invalid PCM data from Kokoro worker";
452 return false;
453 }
454
455 const size_t frames = pcm.size() / (2 * static_cast<size_t>(channels));
456 yarp::sig::Sound result(2);
457 result.setFrequency(sampleRate);
458 result.resize(frames, channels);
459 for (size_t frame = 0; frame < frames; ++frame) {
460 for (int channel = 0; channel < channels; ++channel) {
461 const size_t offset = (frame * static_cast<size_t>(channels) + static_cast<size_t>(channel)) * 2;
462 const auto sample = static_cast<int16_t>(static_cast<uint16_t>(pcm[offset])
463 | (static_cast<uint16_t>(pcm[offset + 1]) << 8));
464 result.set(sample, frame, channel);
465 }
466 }
467 sound = result;
468 return true;
469#else
470 (void)text;
471 (void)sound;
472 return false;
473#endif
474}
475
476bool KokoroSpeechSynthesizer::knownLanguage(const std::string& language,
477 std::string& kokoroCode)
478{
479 const std::string value = lower(language);
480 if (value == "auto" || value == "a" || value == "en" || value == "en-us" || value == "eng") {
481 kokoroCode = "a";
482 } else if (value == "b" || value == "en-gb") {
483 kokoroCode = "b";
484 } else if (value == "e" || value == "es" || value == "es-es") {
485 kokoroCode = "e";
486 } else if (value == "f" || value == "fr" || value == "fr-fr") {
487 kokoroCode = "f";
488 } else if (value == "h" || value == "hi" || value == "hi-in") {
489 kokoroCode = "h";
490 } else if (value == "i" || value == "it" || value == "it-it") {
491 kokoroCode = "i";
492 } else if (value == "j" || value == "ja" || value == "ja-jp") {
493 kokoroCode = "j";
494 } else if (value == "p" || value == "pt" || value == "pt-br") {
495 kokoroCode = "p";
496 } else if (value == "z" || value == "zh" || value == "zh-cn") {
497 kokoroCode = "z";
498 } else {
499 return false;
500 }
501 return true;
502}
503
504std::string KokoroSpeechSynthesizer::defaultVoiceFor(const std::string& kokoroCode)
505{
506 if (kokoroCode == "b") return "bf_emma";
507 if (kokoroCode == "e") return "ef_dora";
508 if (kokoroCode == "f") return "ff_siwis";
509 if (kokoroCode == "h") return "hf_alpha";
510 if (kokoroCode == "i") return "if_sara";
511 if (kokoroCode == "j") return "jf_alpha";
512 if (kokoroCode == "p") return "pf_dora";
513 if (kokoroCode == "z") return "zf_xiaobei";
514 return "af_heart";
515}
516
518{
519 std::string code;
520 if (!knownLanguage(language, code)) {
521 yCError(KOKOROSPEECHSYNTHESIZER) << "Unsupported Kokoro language:" << language;
523 }
524 std::lock_guard<std::mutex> lock(m_mutex);
525 m_currentLanguage = language;
526 m_kokoroLanguage = code;
527 if (!m_voiceExplicit) {
528 m_currentVoice = defaultVoiceFor(code);
529 }
531}
532
534{
535 std::lock_guard<std::mutex> lock(m_mutex);
536 language = m_currentLanguage;
538}
539
541{
542 std::lock_guard<std::mutex> lock(m_mutex);
543 m_voiceExplicit = voice_name != "auto";
544 m_currentVoice = m_voiceExplicit ? voice_name : defaultVoiceFor(m_kokoroLanguage);
545 if (m_currentVoice.empty()) {
547 }
549}
550
552{
553 std::lock_guard<std::mutex> lock(m_mutex);
554 voice_name = m_currentVoice;
556}
557
559{
560 std::lock_guard<std::mutex> lock(m_mutex);
561 auto it = kokoroVoices.find(m_kokoroLanguage);
562 if (it != kokoroVoices.end()) {
563 voices = it->second;
565 }
567}
568
570{
571 if (!std::isfinite(speed) || speed <= 0.0 || speed > 4.0) {
572 yCError(KOKOROSPEECHSYNTHESIZER) << "Speed must be in the range (0, 4]";
574 }
575 std::lock_guard<std::mutex> lock(m_mutex);
576 m_currentSpeed = speed;
578}
579
581{
582 std::lock_guard<std::mutex> lock(m_mutex);
583 speed = m_currentSpeed;
585}
586
588{
589 if (!std::isfinite(pitch)) {
591 }
592 std::lock_guard<std::mutex> lock(m_mutex);
593 // Kokoro does not expose pitch as a synthesis parameter. Keep the value
594 // for interface compatibility, but document that it has no audio effect.
595 m_currentPitch = pitch;
597}
598
600{
601 std::lock_guard<std::mutex> lock(m_mutex);
602 pitch = m_currentPitch;
604}
605
607 yarp::sig::Sound& sound)
608{
609 std::lock_guard<std::mutex> lock(m_mutex);
610 if (!m_open) {
611 yCError(KOKOROSPEECHSYNTHESIZER) << "Device is not open";
613 }
614 return workerSynthesize(text, sound)
617}
const yarp::os::LogComponent & KOKOROSPEECHSYNTHESIZER()
static bool fileExists(const char *fname)
Definition RunServer.cpp:86
bool parseParams(const yarp::os::Searchable &config) override
Parse the DeviceDriver parameters.
yarp::dev::ReturnValue getLanguage(std::string &language) override
Gets the current language set for speech synthesis.
yarp::dev::ReturnValue setVoice(const std::string &voice_name="auto") override
Sets the voice set for speech synthesis.
yarp::dev::ReturnValue getAvailableVoices(yarp::sig::VectorOfString &voices) override
yarp::dev::ReturnValue getVoice(std::string &voice_name) override
Gets the current voice set for speech synthesis.
yarp::dev::ReturnValue getPitch(double &pitch) override
Gets the current pitch set for speech synthesis.
yarp::dev::ReturnValue synthesize(const std::string &text, yarp::sig::Sound &sound) override
Performs the speech synthesis.
bool open(yarp::os::Searchable &config) override
Open the DeviceDriver.
yarp::dev::ReturnValue setLanguage(const std::string &language="auto") override
Sets the language for speech synthesis.
yarp::dev::ReturnValue setSpeed(double speed=0) override
Sets the voice speed for speech synthesis.
yarp::dev::ReturnValue setPitch(double pitch) override
Sets the pitch for speech synthesis.
bool close() override
Close the DeviceDriver.
yarp::dev::ReturnValue getSpeed(double &speed) override
Gets the current voice speed.
@ return_value_error_generic
Method was successfully executed.
@ TraceType
Definition Log.h:92
A base class for nested structures that can be searched.
Definition Searchable.h:31
virtual bool check(const std::string &key) const =0
Check if there exists a property of the given name.
Class for storing sounds See Audio in YARP for additional documentation on YARP audio.
Definition Sound.h:42
#define yCInfo(component,...)
#define yCError(component,...)
#define YARP_LOG_COMPONENT(name,...)
::ssize_t ssize_t
Definition numeric.h:87
int fork()
Portable wrapper for the fork() function.
bool read(ImageOf< PixelRgb > &dest, const std::string &src, image_fileformat format=FORMAT_ANY)
bool write(const ImageOf< PixelRgb > &src, const std::string &dest, image_fileformat format=FORMAT_PPM)