YARP
Yet Another Robot Platform
 
Loading...
Searching...
No Matches
whisperSpeechTranscription.cpp
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: 2023-2023 Istituto Italiano di Tecnologia (IIT)
3 * SPDX-License-Identifier: BSD-3-Clause
4 */
5
7
8#include <yarp/os/Log.h>
9#include <yarp/os/LogStream.h>
11
12#include <cstdio>
13#include <cstdlib>
14#include <algorithm>
15#include <thread>
16#include <regex>
17
18using namespace yarp::os;
19using namespace yarp::dev;
20
21namespace {
22YARP_LOG_COMPONENT(WHISPER_SPEECHTR, "yarp.device.WhisperSpeechTranscription")
23}
24
26{
27 m_language = "en";
28 m_wparams.language=m_language.c_str();
30}
31
36
38{
39 if(!parseParams(config))
40 {
41 yCError(WHISPER_SPEECHTR) << "Could not parse the parameters correctly";
42 return false;
43 }
44 if (config.check("threads","number of threads")){
45 m_wparams.n_threads = config.find("threads").asInt16();}
46 if (config.check("processors", "number of processors")){
47 m_wparams.n_threads = config.find("processors").asInt16();}
48 if (config.check("initial_prompt")) {
49 m_wparams.initial_prompt = config.find("initial_prompt").asString().c_str();}
50 if (config.check("duration","duration of audio to process in milliseconds")) {
51 m_wparams.duration_ms = config.find("duration").asInt32();}
52 if (config.check("offset_ms")) {
53 m_wparams.offset_ms = config.find("offset_ms").asInt32();}
54 // if (config.check("speed_up")) {
55 // m_wparams.speed_up = config.find("speed_up").asBool();}
56 if (config.check("thold_pt","word timestamp probability threshold")) {
57 m_wparams.thold_pt = config.find("thold_pt").asFloat32();}
58 if (config.check("entropy_thold","entropy threshold for decoder fail")) {
59 m_wparams.entropy_thold = config.find("entropy_thold").asFloat32();}
60 if (config.check("logprob_thold","log probability threshold for decoder fail")) {
61 m_wparams.logprob_thold = config.find("logprob_thold").asFloat32();}
62 if (config.check("print_timestamps", "print_timestamps")) {
63 m_wparams.logprob_thold = config.find("print_timestamps").asFloat32();}
64 if (config.check("model", "file containing the model")) {
65 m_model = config.find("model").asString();}
66 if (config.check("translate", "translate from source language to English")) {
67 m_wparams.translate = config.find("translate").asBool();}
68// if (config.check("diarize", "stereo audio diarization")) {
69// m_wparams.diarize = config.find("diarize").asBool();}
70 if(config.check("print_realtime", "print_realtime")) {
71 m_wparams.print_realtime = config.find("print_realtime").asBool();}
72 if(config.check("print_progress", "print_progress")) {
73 m_wparams.print_progress = config.find("print_progress").asBool();}
74 if (config.check("split_on_word", "split on word rather than on token")) {
75 m_wparams.split_on_word = config.find("split_on_word").asBool();}
76 if (config.check("best_of", "number of best candidates to keep")) {
77 m_wparams.greedy.best_of = config.find("best_of").asInt32();}
78 if (config.check("detect-language", "exit after automatically detecting language")) {
79 m_wparams.detect_language = config.find("detect-language").asBool();}
80 if (config.check("beam_size", " beam size for beam search")) {
81 m_wparams.beam_search.beam_size = config.find("beam_size").asInt32();
82 m_wparams.strategy = m_wparams.beam_search.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;
83 }
85 int32_t max_len = 0;
86 bool no_fallback = false;
87 if (config.check("max-context", "maximum number of text context tokens to store")) {
88 max_context = config.find("max-context").asInt32();}
89 if (config.check("max-len", "maximum segment length in characters")) {
90 max_len = config.find("max-len").asInt32();}
91 if (config.check("no-fallback", "do not use temperature fallback while decoding")) {
92 no_fallback = config.find("no-fallback").asBool();}
93 m_no_symbols = m_remove_symbols==1;
94 m_wparams.n_max_text_ctx = max_context >= 0 ? max_context : m_wparams.n_max_text_ctx;
95 m_wparams.token_timestamps = false || max_len > 0;
96 m_wparams.max_len = false && max_len == 0 ? 60 : max_len;
97 m_wparams.temperature_inc = no_fallback ? 0.0f : m_wparams.temperature_inc;
98
99 if (m_language != "auto" && whisper_lang_id(m_language.c_str()) == -1)
100 {
101 yCError(WHISPER_SPEECHTR, "error: unknown language '%s'\n", m_language.c_str());
102 return false;
103 }
104 m_wparams.language = config.find("language").asString().c_str();
105
106 // whisper init
107 std::string model_path;
111 {
112 resource_finder.setDefaultContext(m_model_context);
113 model_path = resource_finder.findFileByName(m_model_name);
114 }
115 else
116 {
117 model_path = m_model_name;
118 }
119 m_ctx = whisper_init_from_file(model_path.c_str());
120 if (m_ctx == nullptr)
121 {
122 yCError(WHISPER_SPEECHTR, "Failed to initialize whisper context\n");
123 return false;
124 }
125
126 // print system information
127 {
128 yCInfo(WHISPER_SPEECHTR, "system_info: n_threads = %d / %d | %s\n",
129 m_wparams.n_threads * n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());
130 }
131
132 // print some info about the processing
133 {
134 if (!whisper_is_multilingual(m_ctx))
135 {
136 if (m_wparams.language != "en" || m_wparams.translate)
137 {
138 m_wparams.language = "en";
139 m_wparams.translate = false;
140 yCWarning(WHISPER_SPEECHTR,"model is not multilingual, ignoring language and translation options");
141 }
142 }
143 if (m_wparams.detect_language)
144 {
145 m_wparams.language = "auto";
146 }
147 yCDebug(WHISPER_SPEECHTR, "%s: processing (%d samples, %.1f sec), %d threads, %d processors, lang = %s, task = %s, timestamps = %d ...\n",
148 __func__, int(m_pcmf32.size()), float(m_pcmf32.size()) / WHISPER_SAMPLE_RATE,
149 m_wparams.n_threads, n_processors,
150 m_wparams.language,
151 m_wparams.translate ? "translate" : "transcribe",
152 m_wparams.print_timestamps);
153 }
154 return true;
155}
156
158{
159 if (m_ctx)
160 {
161 whisper_free(m_ctx);
162 m_ctx=nullptr;
163 }
164 return true;
165}
166
168{
169 m_language=language;
170 yCInfo(WHISPER_SPEECHTR) << "Language set to" << language;
171 return ReturnValue_ok;
172}
173
175{
176 language = m_language;
177 return ReturnValue_ok;
178}
179
180ReturnValue WhisperSpeechTranscription::transcribe(const yarp::sig::Sound& sound, std::string& transcription, double& score)
181{
182 yCDebug(WHISPER_SPEECHTR)<< "received audio" << sound.getSamples();
183
184 score=0;
185 transcription="";
186
187 if (sound.getSamples() == 0 ||
188 sound.getChannels() == 0)
189 {
190 yCError(WHISPER_SPEECHTR) << "Invalid Sound sample received";
191 return ReturnValue::return_code::return_value_error_method_failed;
192 }
193
194 //copy the audio data;
195#if 0
196 m_pcmf32.resize(1000);
197 m_pcmf32s.clear();
198#else
199 m_pcmf32.resize(sound.getSamples());
200 size_t channel=0;
201 for (size_t i = 0; i < m_pcmf32.size(); i++)
202 {
203 m_pcmf32[i]=float(sound.get(i, channel))/65535.0;
204 }
205 m_pcmf32s.clear();
206#endif
207
208 // run the inference
209 {
210 if (whisper_full_parallel(m_ctx, m_wparams, m_pcmf32.data(), m_pcmf32.size(), n_processors) != 0)
211 {
212 yCError(WHISPER_SPEECHTR, "failed to process audio");
213 return ReturnValue::return_code::return_value_error_method_failed;
214 }
215 }
216
217 // output stuff
218 {
219 const int n_segments = whisper_full_n_segments(m_ctx);
220 for (int i = 0; i < n_segments; ++i) {
221 const char* text = whisper_full_get_segment_text(m_ctx, i);
222 transcription += std::string(text);
224 }
225 }
226
227 //remove symbols such as [bla bla]
228 if (m_no_symbols)
229 {
230 std::regex pattern1("\\[[^\\]]*\\]");
231 std::string input = transcription;
232 transcription = std::regex_replace(input, pattern1, "");
233 }
234
235 //assign the score
236 score = 1.0;
237 if (transcription.empty()) {score = 0.0;}
238 return ReturnValue_ok;
239}
#define ReturnValue_ok
Definition ReturnValue.h:82
bool parseParams(const yarp::os::Searchable &config) override
Parse the DeviceDriver parameters.
virtual yarp::dev::ReturnValue setLanguage(const std::string &language) override
Sets the language for speech transcription.
bool open(yarp::os::Searchable &config) override
Open the DeviceDriver.
bool close() override
Close the DeviceDriver.
virtual yarp::dev::ReturnValue getLanguage(std::string &language) override
Gets the current language set for speech transcription.
virtual yarp::dev::ReturnValue transcribe(const yarp::sig::Sound &sound, std::string &transcription, double &score) override
Performs the speech transcription.
A mini-server for performing network communication in the background.
Helper class for finding config files and other external resources.
A base class for nested structures that can be searched.
Definition Searchable.h:31
virtual bool check(const std::string &key) const =0
Check if there exists a property of the given name.
virtual Value & find(const std::string &key) const =0
Gets a value corresponding to a given keyword.
Class for storing sounds See Audio in YARP for additional documentation on YARP audio.
Definition Sound.h:42
size_t getChannels() const
Get the number of channels of the sound.
Definition Sound.cpp:669
audio_sample get(size_t sample, size_t channel=0) const
Definition Sound.cpp:291
size_t getSamples() const
Get the number of samples contained in the sound.
Definition Sound.cpp:664
#define yCInfo(component,...)
#define yCError(component,...)
#define yCWarning(component,...)
#define yCDebug(component,...)
#define YARP_LOG_COMPONENT(name,...)
For streams capable of holding different kinds of content, check what they actually have.
An interface to the operating system, including Port based communication.
Definition OSTestMsg.cpp:13