Skip to content

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
singalsu wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7d89764
Audio: MFCC: Add Voice Activity Detection based on Mel spectrum
singalsu May 12, 2026
86388a7
Audio: MFCC: Add Python script for speech to text with Whisper model
singalsu May 19, 2026
ca7aa07
Audio: MFCC: Add VAD switch control notification to user space
singalsu May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/audio/mfcc/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@ if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
add_subdirectory(llext ${PROJECT_BINARY_DIR}/mfcc_llext)
add_dependencies(app mfcc)
else()
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c)
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c mfcc_vad.c)
if(CONFIG_IPC_MAJOR_4)
add_local_sources(sof mfcc_ipc4.c)
endif()
endif()
44 changes: 41 additions & 3 deletions src/audio/mfcc/mfcc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <ipc/control.h>
#include <ipc/stream.h>
#include <ipc/topology.h>
#include <ipc4/header.h>
#include <user/mfcc.h>
#include <user/trace.h>
#include <rtos/init.h>
Expand Down Expand Up @@ -68,6 +69,16 @@ static mfcc_func mfcc_find_func(enum sof_ipc_frame source_format,
* End of MFCC setup code. Next the standard component methods.
*/

/* Weak stubs for IPC notification, overridden by mfcc_ipc4.c */
__weak int mfcc_ipc_notification_init(struct processing_module *mod)
{
return 0;
}

__weak void mfcc_send_vad_notification(struct processing_module *mod, uint32_t val)
{
}

static int mfcc_init(struct processing_module *mod)
{
struct module_data *md = &mod->priv;
Expand Down Expand Up @@ -97,6 +108,7 @@ static int mfcc_free(struct processing_module *mod)
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");
ipc_msg_free(cd->msg);
mod_data_blob_handler_free(mod, cd->model_handler);
mfcc_free_buffers(mod);
mod_free(mod, cd);
Expand All @@ -109,10 +121,21 @@ static int mfcc_get_config(struct processing_module *mod,
{
struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment;
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct sof_ipc4_control_msg_payload *ctl;

comp_info(mod->dev, "entry");

return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
switch (config_id) {
case SOF_IPC4_SWITCH_CONTROL_PARAM_ID:
ctl = (struct sof_ipc4_control_msg_payload *)fragment;
if (ctl->id == MFCC_CTRL_INDEX_VAD && ctl->num_elems == 1) {
ctl->chanv[0].value = cd->vad_prev ? 1 : 0;
*data_offset_size = sizeof(*ctl) + sizeof(ctl->chanv[0]);
}
return 0;
default:
return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
}
}

static int mfcc_set_config(struct processing_module *mod, uint32_t config_id,
Expand All @@ -124,8 +147,14 @@ static int mfcc_set_config(struct processing_module *mod, uint32_t config_id,

comp_info(mod->dev, "entry");

return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
fragment, fragment_size);
switch (config_id) {
case SOF_IPC4_SWITCH_CONTROL_PARAM_ID:
/* VAD switch is read-only, ignore set requests */
return 0;
default:
return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
fragment, fragment_size);
}
}

static int mfcc_process(struct processing_module *mod,
Expand Down Expand Up @@ -198,6 +227,15 @@ static int mfcc_prepare(struct processing_module *mod,
goto err;
}

/* Initialize VAD switch control notification if enabled */
if (cd->config && cd->config->update_controls) {
ret = mfcc_ipc_notification_init(mod);
if (ret < 0)
goto err;

cd->vad_prev = false;
}

return 0;

err:
Expand Down
88 changes: 57 additions & 31 deletions src/audio/mfcc/mfcc_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,17 @@
#include <stddef.h>
#include <stdint.h>

#include <sof/audio/mfcc/mfcc_vad.h>

LOG_MODULE_REGISTER(mfcc_common, CONFIG_SOF_LOG_LEVEL);

/*
* The main processing function for MFCC
*/

static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *cd)
static int mfcc_stft_process(struct processing_module *mod, struct mfcc_comp_data *cd)
{
const struct comp_dev *dev = mod->dev;
struct sof_mfcc_config *config = cd->config;
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &state->buf;
Expand Down Expand Up @@ -169,6 +172,28 @@ static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *

cc_count += state->dct.num_out;
}

/* Run VAD on the mel log spectrum (available in both modes) */
if (config->enable_vad)
mfcc_vad_update(&cd->vad, state->mel_log_32);

/* Populate data header for this output frame */
state->header.magic = MFCC_MAGIC;
state->header.frame_number = cd->vad.frame_count;
state->header.reserved = 0;
state->header.energy = cd->vad.energy;
state->header.noise_energy = cd->vad.noise_energy;
state->header.vad_flag = cd->vad.is_speech ? 1 : 0;

/* Send notification when VAD state changes */
if (config->update_controls) {
bool vad_now = cd->vad.is_speech;

if (vad_now != cd->vad_prev) {
mfcc_send_vad_notification(mod, vad_now ? 1 : 0);
cd->vad_prev = vad_now;
}
}
}

return cc_count;
Expand Down Expand Up @@ -267,9 +292,8 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int16_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 2;
const int num_header_s16 = sizeof(state->header) / sizeof(int16_t);
int num_ceps;
int sink_samples;
int to_copy;
Expand All @@ -278,27 +302,29 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s16(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT: Mel auditory filter and DCT. */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data and mark magic pending */
/* If new output produced, set up pointer into scratch data and mark header pending */
if (num_ceps > 0) {
if (state->mel_only)
if (state->mel_only) {
state->out_data_ptr = state->mel_spectra->data;
else
} else {
state->out_data_ptr = state->cepstral_coef->data;
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_magic, (int16_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s16) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_header_s16,
(int16_t *)&state->header);
sink_samples -= num_header_s16;
state->header_pending = false;
}

/* Write cepstral/mel data from scratch buffer */
Expand Down Expand Up @@ -363,9 +389,8 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -376,7 +401,7 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s24(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -391,17 +416,18 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s32) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down Expand Up @@ -443,9 +469,8 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -455,7 +480,7 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s32(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -466,17 +491,18 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s32) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down
97 changes: 97 additions & 0 deletions src/audio/mfcc/mfcc_ipc4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// SPDX-License-Identifier: BSD-3-Clause
//
// Copyright(c) 2026 Intel Corporation.
//
// Author: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>

/**
* \file mfcc_ipc4.c
* \brief IPC4-specific functions for MFCC component.
*
* Provides VAD switch control notification to user space via the
* IPC4 module notification mechanism.
*/

#include <sof/audio/mfcc/mfcc_comp.h>
#include <sof/audio/module_adapter/module/generic.h>
#include <sof/audio/component.h>
#include <sof/ipc/msg.h>
#include <sof/trace/trace.h>
#include <ipc4/base-config.h>
#include <ipc4/header.h>
#include <ipc4/module.h>
#include <ipc4/notification.h>
#include <rtos/string.h>
#include <errno.h>
#include <stdint.h>

LOG_MODULE_DECLARE(mfcc, CONFIG_SOF_LOG_LEVEL);

/**
* \brief Initialize IPC notification message for VAD switch control.
*
* Allocates and configures the IPC message used to send VAD state
* change notifications to user space via a switch control.
*/
int mfcc_ipc_notification_init(struct processing_module *mod)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct ipc_msg msg_proto;
struct comp_dev *dev = mod->dev;
struct comp_ipc_config *ipc_config = &dev->ipc_config;
union ipc4_notification_header *primary =
(union ipc4_notification_header *)&msg_proto.header;
struct sof_ipc4_notify_module_data *msg_module_data;
struct sof_ipc4_control_msg_payload *msg_payload;

memset_s(&msg_proto, sizeof(msg_proto), 0, sizeof(msg_proto));
primary->r.notif_type = SOF_IPC4_MODULE_NOTIFICATION;
primary->r.type = SOF_IPC4_GLB_NOTIFICATION;
primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST;
primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG;
cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension,
sizeof(struct sof_ipc4_notify_module_data) +
sizeof(struct sof_ipc4_control_msg_payload) +
sizeof(struct sof_ipc4_ctrl_value_chan));
if (!cd->msg) {
comp_err(dev, "Failed to initialize VAD notification");
return -ENOMEM;
}

msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data;
msg_module_data->instance_id = IPC4_INST_ID(ipc_config->id);
msg_module_data->module_id = IPC4_MOD_ID(ipc_config->id);
msg_module_data->event_id = SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL |
SOF_IPC4_SWITCH_CONTROL_PARAM_ID;
msg_module_data->event_data_size = sizeof(struct sof_ipc4_control_msg_payload) +
sizeof(struct sof_ipc4_ctrl_value_chan);

msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data;
msg_payload->id = MFCC_CTRL_INDEX_VAD;
msg_payload->num_elems = 1;
msg_payload->chanv[0].channel = 0;

comp_dbg(dev, "VAD notification init: instance_id = 0x%08x, module_id = 0x%08x",
msg_module_data->instance_id, msg_module_data->module_id);
return 0;
}

/**
* \brief Send VAD switch control notification to user space.
* \param mod Processing module.
* \param val VAD value: 1 = speech, 0 = silence.
*/
void mfcc_send_vad_notification(struct processing_module *mod, uint32_t val)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct sof_ipc4_notify_module_data *msg_module_data;
struct sof_ipc4_control_msg_payload *msg_payload;

if (!cd->msg)
return;

msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data;
msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data;
msg_payload->chanv[0].value = val;
ipc_msg_send(cd->msg, NULL, false);
}
Loading
Loading