YARP
Yet Another Robot Platform
 
Loading...
Searching...
No Matches
VoiceBoxSynthesizer.cpp
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: 2023 Istituto Italiano di Tecnologia (IIT)
3 * SPDX-License-Identifier: BSD-3-Clause
4 */
5
6#ifndef _USE_MATH_DEFINES
7#define _USE_MATH_DEFINES
8#endif
9
10#include "VoiceBoxSynthesizer.h"
11
12#include <curl/curl.h>
13#include <nlohmann/json.hpp>
14#define DR_WAV_IMPLEMENTATION
15#include "dr_wav.h"
16
18#include <yarp/os/LogStream.h>
19
20using namespace yarp::os;
21using namespace yarp::dev;
22using json = nlohmann::json;
23
25
26namespace {
27
28size_t writeStringCallback(void* contents, size_t size, size_t nmemb, void* userp)
29{
30 const size_t total = size * nmemb;
31 auto* out = static_cast<std::string*>(userp);
32 out->append(static_cast<const char*>(contents), total);
33 return total;
34}
35
36size_t writeBytesCallback(void* contents, size_t size, size_t nmemb, void* userp)
37{
38 const size_t total = size * nmemb;
39 auto* out = static_cast<std::vector<unsigned char>*>(userp);
40 const auto* begin = static_cast<const unsigned char*>(contents);
41 out->insert(out->end(), begin, begin + total);
42 return total;
43}
44
45struct HttpResponse
46{
47 long status = 0;
48 std::string body;
49};
50
51struct HttpBytesResponse
52{
53 long status = 0;
54 std::vector<unsigned char> body;
55};
56
57HttpResponse httpPostJson(const std::string& url, const json& payload)
58{
60 if (!curl) {
61 throw std::runtime_error("curl_easy_init failed");
62 }
63
64 HttpResponse resp;
65 std::string body = payload.dump();
66
67 struct curl_slist* headers = nullptr;
68 headers = curl_slist_append(headers, "Content-Type: application/json");
69
70 curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
77
79 if (rc != CURLE_OK) {
80 curl_slist_free_all(headers);
82 throw std::runtime_error(std::string("HTTP POST failed: ") + curl_easy_strerror(rc));
83 }
84
86
87 curl_slist_free_all(headers);
89 return resp;
90}
91
92HttpResponse httpGetText(const std::string& url)
93{
95 if (!curl) {
96 throw std::runtime_error("curl_easy_init failed");
97 }
98
99 HttpResponse resp;
100 curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
103
105 if (rc != CURLE_OK) {
107 throw std::runtime_error(std::string("HTTP GET failed: ") + curl_easy_strerror(rc));
108 }
109
112 return resp;
113}
114
115HttpBytesResponse httpGetBytes(const std::string& url)
116{
118 if (!curl) {
119 throw std::runtime_error("curl_easy_init failed");
120 }
121
122 HttpBytesResponse resp;
123 curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
126
128 if (rc != CURLE_OK) {
130 throw std::runtime_error(std::string("HTTP GET bytes failed: ") + curl_easy_strerror(rc));
131 }
132
134 yCDebug(VOICEBOXSYNTHESIZER) << "HTTP GET bytes completed with status " << resp.status << " and " << resp.body.size() << " bytes received";
136 return resp;
137}
138
139struct DecodedAudio
140{
141 uint32_t sampleRate = 0;
142 uint32_t channels = 0;
143 std::vector<int16_t> interleavedPcm16;
144};
145
146DecodedAudio decodeWav16FromMemory(const std::vector<unsigned char>& wavBytes)
147{
148 unsigned int channels = 0;
149 unsigned int sampleRate = 0;
151
153 wavBytes.data(),
154 wavBytes.size(),
155 &channels,
156 &sampleRate,
158 nullptr);
159
160 if (!pcm) {
161 throw std::runtime_error("Unable to decode WAV from memory");
162 }
163
164 const size_t totalSamples = static_cast<size_t>(totalFrames) * channels;
165
166 DecodedAudio out;
167 out.sampleRate = sampleRate;
168 out.channels = channels;
169 out.interleavedPcm16.assign(pcm, pcm + totalSamples);
170
171 drwav_free(pcm, nullptr);
172 return out;
173}
174
175profilesList parseProfiles(const std::string& profilesJson)
176{
178
179 json j = json::parse(profilesJson);
180
181 // Handle both array and object responses
182 json profilesArray = j.is_array() ? j : json::array();
183
184 for (const auto& profileJson : profilesArray) {
185 std::map<std::string, std::string> profile;
186
187 // Extract required fields, validating that they are not null
188 for (auto& [key, value] : profileJson.items()) {
189 if (!value.is_null()) {
190 profile[key] = value.is_string() ? value.get<std::string>() : value.dump();
191 }
192 }
193
194 // Only add profile if it has required fields
195 if (profile.count("id") && profile.count("name")) {
196 profiles.push_back(profile);
197 }
198 }
199
200 return profiles;
201}
202
203}
204
206{
207 if (audio.channels == 0 || audio.sampleRate == 0) {
208 throw std::runtime_error("Invalid decoded audio metadata");
209 }
210
211 if (audio.interleavedPcm16.size() % audio.channels != 0) {
212 throw std::runtime_error("Invalid PCM layout");
213 }
214
215 const size_t samplesPerChannel = audio.interleavedPcm16.size() / audio.channels;
216
217 yarp::sig::Sound snd(2); // 16-bit
218 snd.setFrequency(static_cast<int>(audio.sampleRate));
219 snd.resize(samplesPerChannel, audio.channels);
220
221 for (size_t s = 0; s < samplesPerChannel; ++s) {
222 for (size_t ch = 0; ch < audio.channels; ++ch) {
223 const int16_t sample = audio.interleavedPcm16[s * audio.channels + ch];
224 snd.set(sample, s, ch);
225 }
226 }
227
228 return snd;
229}
230
231static profilesList getProfiles(const std::string& baseUrl)
232{
233 HttpResponse resp = httpGetText(baseUrl + "/profiles");
234
235 if (resp.status < 200 || resp.status >= 300) {
236 throw std::runtime_error("Get profiles failed. HTTP " + std::to_string(resp.status) + ": " + resp.body);
237 }
238
239 return parseProfiles(resp.body);
240}
241
242static bool findProfileByName(const profilesList& profiles, const std::string& name, profileMap& outProfile)
243{
244 for (const auto& profile : profiles) {
245 if (profile.at("name") == name) {
247 return true;
248 }
249 }
250 return false;
251}
252
253static std::string startGeneration(
254 const std::string& baseUrl,
255 const profileMap& profile,
256 const std::string& text,
257 const std::string& language,
258 double speed)
259{
260 std::string profileId = profile.at("id");
261 yCDebug(VOICEBOXSYNTHESIZER) << "Starting generation with profile ID: " << profileId
262 << " text='" << text << "'"
263 << " language='" << language << "'"
264 << " speed=" << speed;
265 std::string engine = profile.at("default_engine");
266 yCDebug(VOICEBOXSYNTHESIZER) << "Using engine: " << engine;
267 json payload = {
268 {"profile_id", profileId},
269 {"engine", engine},
270 {"text", text},
271 {"language", language},
272 {"speed", speed},
273 {"seed", 0},
274 {"max_chunk_chars", 800},
275 {"crossfade_ms", 50},
276 {"normalize", true}
277 };
278
279 HttpResponse resp = httpPostJson(baseUrl + "/generate", payload);
280
281 yCInfo(VOICEBOXSYNTHESIZER) << "Generation request sent. HTTP " << resp.status << ": " << resp.body;
282
283 if (resp.status < 200 || resp.status >= 300) {
284 throw std::runtime_error("Generate failed. HTTP " + std::to_string(resp.status) + ": " + resp.body);
285 }
286
287 json j = json::parse(resp.body);
288
289 if (!j.contains("id") || j.at("id").is_null()) {
290 throw std::runtime_error("Generate response has no valid id: " + resp.body);
291 }
292
293 std::string id = j.at("id").get<std::string>();
294 if (id.empty()) {
295 throw std::runtime_error("Generate response has empty id: " + resp.body);
296 }
297
298 yCInfo(VOICEBOXSYNTHESIZER) << "Generation started with ID: " << id;
299
300 return id;
301}
302
303// static void waitUntilFinished(
304// const std::string& baseUrl,
305// const std::string& generationId,
306// std::chrono::milliseconds pollInterval = std::chrono::milliseconds(1000),
307// std::chrono::seconds timeout = std::chrono::seconds(120))
308// {
309// const auto deadline = std::chrono::steady_clock::now() + timeout;
310
311// while (std::chrono::steady_clock::now() < deadline) {
312// HttpResponse resp = httpGetText(baseUrl + "/history/" + generationId);
313
314// if (resp.status < 200 || resp.status >= 300) {
315// throw std::runtime_error("History failed. HTTP " + std::to_string(resp.status) + ": " + resp.body);
316// }
317
318// json j = json::parse(resp.body);
319
320// // Safely extract status and error, handling null values
321// std::string status;
322// if (j.contains("status") && !j.at("status").is_null()) {
323// status = j.at("status").is_string() ? j.at("status").get<std::string>() : j.at("status").dump();
324// }
325
326// std::string error;
327// if (j.contains("error") && !j.at("error").is_null()) {
328// error = j.at("error").is_string() ? j.at("error").get<std::string>() : j.at("error").dump();
329// }
330
331// if (status == "completed" || status == "done" || status == "finished") {
332// yCDebug(VOICEBOXSYNTHESIZER) << "Generation completed with status: " << status;
333// return;
334// }
335
336// if (status == "error" || status == "failed" || !error.empty()) {
337// throw std::runtime_error("Generation failed. Status=" + status + " error=" + error);
338// }
339
340// std::this_thread::sleep_for(pollInterval);
341// }
342
343// throw std::runtime_error("Timeout waiting for Voicebox generation");
344// }
346 const std::string& baseUrl,
347 const std::string& generationId,
348 std::chrono::milliseconds pollInterval = std::chrono::milliseconds(1000),
349 std::chrono::seconds timeout = std::chrono::seconds(120))
350{
351 const auto deadline = std::chrono::steady_clock::now() + timeout;
352
353 while (std::chrono::steady_clock::now() < deadline) {
354 HttpResponse resp = httpGetText(baseUrl + "/history/" + generationId);
355
356 if (resp.status < 200 || resp.status >= 300) {
357 throw std::runtime_error("History failed. HTTP " + std::to_string(resp.status) + ": " + resp.body);
358 }
359
360 json j = json::parse(resp.body);
361
362 std::string status;
363 if (j.contains("status") && !j.at("status").is_null()) {
364 status = j.at("status").is_string() ? j.at("status").get<std::string>() : j.at("status").dump();
365 }
366
367 std::string error;
368 if (j.contains("error") && !j.at("error").is_null()) {
369 error = j.at("error").is_string() ? j.at("error").get<std::string>() : j.at("error").dump();
370 }
371
372 std::string audioPath;
373 if (j.contains("audio_path") && !j.at("audio_path").is_null()) {
374 audioPath = j.at("audio_path").is_string() ? j.at("audio_path").get<std::string>() : j.at("audio_path").dump();
375 }
376
377 double duration = 0.0;
378 if (j.contains("duration") && !j.at("duration").is_null()) {
379 duration = j.at("duration").get<double>();
380 }
381
383 << "Generation status=" << status
384 << " audio_path='" << audioPath
385 << "' duration=" << duration
386 << " error='" << error << "'";
387
388 if (status == "error" || status == "failed" || !error.empty()) {
389 throw std::runtime_error("Generation failed. Status=" + status + " error=" + error);
390 }
391
392 // Considera pronto solo se c'è davvero un file audio associato
393 if ((status == "completed" || status == "done" || status == "finished") &&
394 !audioPath.empty() &&
395 audioPath != ".") {
396 return;
397 }
398
399 std::this_thread::sleep_for(pollInterval);
400 }
401
402 throw std::runtime_error("Timeout waiting for Voicebox generation");
403}
404
405// static yarp::sig::Sound downloadAudioAsYarpSound(
406// const std::string& baseUrl,
407// const std::string& generationId)
408// {
409// yCInfo(VOICEBOXSYNTHESIZER) << "Downloading audio for generation ID: " << generationId;
410
411// HttpBytesResponse resp = httpGetBytes(baseUrl + "/audio/" + generationId);
412
413// if (resp.status < 200 || resp.status >= 300) {
414// std::string errorBody;
415// if (!resp.body.empty()) {
416// errorBody = std::string(resp.body.begin(), resp.body.end());
417// }
418// yCError(VOICEBOXSYNTHESIZER) << "Audio download failed. HTTP " << resp.status << " received " << resp.body.size() << " bytes. Response: " << errorBody;
419// throw std::runtime_error("Audio download failed. HTTP " + std::to_string(resp.status) + ": " + errorBody);
420// }
421
422// if (resp.body.empty()) {
423// throw std::runtime_error("Audio download returned empty response");
424// }
425
426// yCInfo(VOICEBOXSYNTHESIZER) << "Downloaded " << resp.body.size() << " bytes";
427
428// DecodedAudio audio = decodeWav16FromMemory(resp.body);
429// return decodedAudioToYarpSound(audio);
430// }
432 const std::string& baseUrl,
433 const std::string& generationId)
434{
435 HttpResponse hist = httpGetText(baseUrl + "/history/" + generationId);
436 yCInfo(VOICEBOXSYNTHESIZER) << "History before audio download: " << hist.body;
437
438 yCInfo(VOICEBOXSYNTHESIZER) << "Downloading audio for generation ID: " << generationId;
439 HttpBytesResponse resp = httpGetBytes(baseUrl + "/audio/" + generationId);
440
441 if (resp.status < 200 || resp.status >= 300) {
442 std::string errorBody;
443 if (!resp.body.empty()) {
444 errorBody = std::string(resp.body.begin(), resp.body.end());
445 }
446 yCError(VOICEBOXSYNTHESIZER) << "Audio download failed. HTTP " << resp.status
447 << " received " << resp.body.size()
448 << " bytes. Response: " << errorBody;
449 throw std::runtime_error("Audio download failed. HTTP " + std::to_string(resp.status) + ": " + errorBody);
450 }
451
452 if (resp.body.empty()) {
453 throw std::runtime_error("Audio download returned empty response");
454 }
455
456 DecodedAudio audio = decodeWav16FromMemory(resp.body);
458}
459
461 const std::string& baseUrl,
462 const profileMap& profile,
463 const std::string& text,
464 const std::string& language,
465 double speed)
466{
467 const std::string id = startGeneration(baseUrl, profile, text, language, speed);
468 yCInfo(VOICEBOXSYNTHESIZER) << "Waiting for generation to complete...";
470 yCInfo(VOICEBOXSYNTHESIZER) << "Generation completed. Downloading audio...";
472}
473
477
479{
480 if (!parseParams(config)) {
481 yCError(VOICEBOXSYNTHESIZER) << "Failed to parse parameters";
482 return false;
483 }
484
485 m_baseUrl = "http://" + m_base_ip + ":" + m_base_port;
486
487 if(!findProfileByName(getProfiles(m_baseUrl), m_voice, m_profile)) {
488 yCError(VOICEBOXSYNTHESIZER) << "Voice profile not found: " << m_voice;
489 return false;
490 }
491
492
494 yCError(VOICEBOXSYNTHESIZER) << "curl_global_init failed";
495 return false;
496 }
497
498 setLanguage(m_language);
499
500 yCInfo(VOICEBOXSYNTHESIZER) << "Open";
501 return true;
502}
503
505{
507 yCInfo(VOICEBOXSYNTHESIZER) << "Close";
508 return true;
509}
510
512{
513 if (language.empty()) {
515 }
516 std::string languageCopy = language;
517 if(language.find("-")!= std::string::npos) {
518 yCDebug(VOICEBOXSYNTHESIZER) << "Language '" << language << "' contains region code. Voicebox may not support this format. Consider using just the language code (e.g. 'en' instead of 'en-US')";
519 languageCopy = language.substr(0, language.find("-"));
520 m_region = language.substr(language.find("-")+1);
521 }
522 m_language = languageCopy;
523 return ReturnValue_ok;
524}
525
527{
528 language = m_language+"-"+m_region;
529 return ReturnValue_ok;
530}
531
533{
534 if (voice_name.empty()) {
536 }
537
538 try {
541 yCError(VOICEBOXSYNTHESIZER) << "Voice profile not found: " << voice_name;
543 }
544
545 // Verify profile has required fields
546 if (profile.find("id") == profile.end() || profile["id"].empty()) {
547 yCError(VOICEBOXSYNTHESIZER) << "Profile missing required 'id' field: " << voice_name;
549 }
550
552 m_profile = profile;
553 return ReturnValue_ok;
554 } catch (const std::exception& ex) {
555 yCError(VOICEBOXSYNTHESIZER) << "setVoice failed: " << ex.what();
557 }
558}
559
565
567{
568 try {
569 profilesList profiles = getProfiles(m_baseUrl);
570 voices.clear();
571 for (const auto& profile : profiles) {
572 if (profile.find("name") != profile.end()) {
573 voices.push_back(profile.at("name"));
574 }
575 }
576 return ReturnValue_ok;
577 } catch (const std::exception& ex) {
578 yCError(VOICEBOXSYNTHESIZER) << "getAvailableVoices failed: " << ex.what();
580 }
581}
582
584{
585 if (speed <= 0.0) {
587 }
588 m_speed = speed;
589 return ReturnValue_ok;
590}
591
593{
594 speed = m_speed;
595 return ReturnValue_ok;
596}
597
599{
600 if (pitch <= 0.0) {
602 }
603 m_pitch = pitch;
604 return ReturnValue_ok;
605}
606
608{
609 pitch = m_pitch;
610 return ReturnValue_ok;
611}
612
614{
615 if (text.empty()) {
617 }
618
619 try {
620 sound = synthesizeVoiceboxToYarpSound(m_baseUrl, m_profile, text, m_language, m_speed);
621 return ReturnValue_ok;
622 } catch (const std::exception& ex) {
623 yCError(VOICEBOXSYNTHESIZER) << "synthesize failed: " << ex.what();
625 }
626}
627
#define ReturnValue_ok
Definition ReturnValue.h:82
nlohmann::json json
Definition Utils.h:15
static std::string startGeneration(const std::string &baseUrl, const profileMap &profile, const std::string &text, const std::string &language, double speed)
const yarp::os::LogComponent & VOICEBOXSYNTHESIZER()
static void waitUntilFinished(const std::string &baseUrl, const std::string &generationId, std::chrono::milliseconds pollInterval=std::chrono::milliseconds(1000), std::chrono::seconds timeout=std::chrono::seconds(120))
static yarp::sig::Sound synthesizeVoiceboxToYarpSound(const std::string &baseUrl, const profileMap &profile, const std::string &text, const std::string &language, double speed)
static bool findProfileByName(const profilesList &profiles, const std::string &name, profileMap &outProfile)
static profilesList getProfiles(const std::string &baseUrl)
static yarp::sig::Sound downloadAudioAsYarpSound(const std::string &baseUrl, const std::string &generationId)
static yarp::sig::Sound decodedAudioToYarpSound(const DecodedAudio &audio)
std::map< std::string, std::string > profileMap
std::vector< std::map< std::string, std::string > > profilesList
bool parseParams(const yarp::os::Searchable &config) override
Parse the DeviceDriver parameters.
yarp::dev::ReturnValue getPitch(double &pitch) override
Gets the current pitch set for speech synthesis.
yarp::dev::ReturnValue getLanguage(std::string &language) override
Gets the current language set for speech synthesis.
yarp::dev::ReturnValue setSpeed(const double speed) override
Sets the voice speed for speech synthesis.
yarp::dev::ReturnValue getSpeed(double &speed) override
Gets the current voice speed.
yarp::dev::ReturnValue getVoice(std::string &voice_name) override
Gets the current voice set for speech synthesis.
bool close() override
Close the DeviceDriver.
yarp::dev::ReturnValue setVoice(const std::string &voice_name) override
Sets the voice set for speech synthesis.
yarp::dev::ReturnValue setLanguage(const std::string &language) override
Sets the language for speech synthesis.
yarp::dev::ReturnValue getAvailableVoices(yarp::sig::VectorOfString &voices) override
bool open(yarp::os::Searchable &config) override
Open the DeviceDriver.
yarp::dev::ReturnValue synthesize(const std::string &text, yarp::sig::Sound &sound) override
Performs the speech synthesis.
yarp::dev::ReturnValue setPitch(const double pitch) override
Sets the pitch for speech synthesis.
@ return_value_error_method_failed
Method is deprecated.
A mini-server for performing network communication in the background.
@ TraceType
Definition Log.h:92
A base class for nested structures that can be searched.
Definition Searchable.h:31
Class for storing sounds See Audio in YARP for additional documentation on YARP audio.
Definition Sound.h:42
drwav_int16 * drwav_open_memory_and_read_pcm_frames_s16(const void *data, size_t dataSize, unsigned int *channelsOut, unsigned int *sampleRateOut, drwav_uint64 *totalFrameCountOut, const drwav_allocation_callbacks *pAllocationCallbacks)
void drwav_free(void *p, const drwav_allocation_callbacks *pAllocationCallbacks)
#define yCInfo(component,...)
#define yCError(component,...)
#define yCDebug(component,...)
#define YARP_LOG_COMPONENT(name,...)
For streams capable of holding different kinds of content, check what they actually have.
An interface to the operating system, including Port based communication.
Definition OSTestMsg.cpp:13