fe_noise.c
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 2013 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* This work was supported in part by funding from the Defense Advanced
* Research Projects Agency and the National Science Foundation of the
* United States of America, and the CMU Sphinx Speech Consortium.
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
/* This noise removal algorithm is inspired by the following papers
* Computationally Efficient Speech Enchancement by Spectral Minina Tracking
* by G. Doblinger
*
* Power-Normalized Cepstral Coefficients (PNCC) for Robust Speech Recognition
* by C. Kim.
*
* For the recent research and state of art see papers about IMRCA and
* A Minimum-Mean-Square-Error Noise Reduction Algorithm On Mel-Frequency
* Cepstra For Robust Speech Recognition by Dong Yu and others
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <math.h>
#include "sphinxbase/prim_type.h"
#include "sphinxbase/ckd_alloc.h"
#include "sphinxbase/strfuncs.h"
#include "sphinxbase/err.h"
#include "fe_noise.h"
#include "fe_internal.h"
/* Noise supression constants */
#define SMOOTH_WINDOW 4
#define LAMBDA_POWER 0.7
#define LAMBDA_A 0.995
#define LAMBDA_B 0.5
#define LAMBDA_T 0.85
#define MU_T 0.2
#define MAX_GAIN 20
#define SLOW_PEAK_FORGET_FACTOR 0.9995
#define SLOW_PEAK_LEARN_FACTOR 0.9
#define SPEECH_VOLUME_RANGE 8.0
/* define VAD_DEBUG 1 */
#ifdef VAD_DEBUG
static FILE *vad_stats;
static int64 low_snr = 0;
static int64 low_volume = 0;
#endif
struct noise_stats_s {
/* Smoothed power */
powspec_t *power;
/* Noise estimate */
powspec_t *noise;
/* Signal floor estimate */
powspec_t *floor;
/* Peak for temporal masking */
powspec_t *peak;
/* Initialize it next time */
uint8 undefined;
/* Number of items to process */
uint32 num_filters;
/* Sum of slow peaks for VAD */
powspec_t slow_peak_sum;
/* Precomputed constants */
powspec_t lambda_power;
powspec_t comp_lambda_power;
powspec_t lambda_a;
powspec_t comp_lambda_a;
powspec_t lambda_b;
powspec_t comp_lambda_b;
powspec_t lambda_t;
powspec_t mu_t;
powspec_t max_gain;
powspec_t inv_max_gain;
powspec_t smooth_scaling[2 * SMOOTH_WINDOW + 3];
};
static void
fe_lower_envelope(noise_stats_t *noise_stats, powspec_t * buf, powspec_t * floor_buf, int32 num_filt)
{
int i;
for (i = 0; i < num_filt; i++) {
#ifndef FIXED_POINT
if (buf[i] >= floor_buf[i]) {
floor_buf[i] =
noise_stats->lambda_a * floor_buf[i] + noise_stats->comp_lambda_a * buf[i];
}
else {
floor_buf[i] =
noise_stats->lambda_b * floor_buf[i] + noise_stats->comp_lambda_b * buf[i];
}
#else
if (buf[i] >= floor_buf[i]) {
floor_buf[i] = fe_log_add(noise_stats->lambda_a + floor_buf[i],
noise_stats->comp_lambda_a + buf[i]);
}
else {
floor_buf[i] = fe_log_add(noise_stats->lambda_b + floor_buf[i],
noise_stats->comp_lambda_b + buf[i]);
}
#endif
}
}
/* update slow peaks, check if max signal level big enough compared to peak */
static int16
fe_is_frame_quiet(noise_stats_t *noise_stats, powspec_t *buf, int32 num_filt)
{
int i;
int16 is_quiet;
powspec_t sum;
double smooth_factor;
sum = 0.0;
for (i = 0; i < num_filt; i++) {
#ifndef FIXED_POINT
sum += buf[i];
#else
sum = fe_log_add(sum, buf[i]);
#endif
}
#ifndef FIXED_POINT
sum = log(sum);
#endif
smooth_factor = (sum > noise_stats->slow_peak_sum) ? SLOW_PEAK_LEARN_FACTOR : SLOW_PEAK_FORGET_FACTOR;
noise_stats->slow_peak_sum = noise_stats->slow_peak_sum * smooth_factor +
sum * (1 - smooth_factor);
#ifdef VAD_DEBUG
#ifndef FIXED_POINT
fprintf(vad_stats, "%.3f %.3f ", noise_stats->slow_peak_sum, sum);
#else
fprintf(vad_stats, "%d %d ", noise_stats->slow_peak_sum, sum);
#endif
#endif
#ifndef FIXED_POINT
is_quiet = noise_stats->slow_peak_sum - SPEECH_VOLUME_RANGE > sum;
#else
is_quiet = noise_stats->slow_peak_sum - FLOAT2FIX(SPEECH_VOLUME_RANGE) > sum;
#endif
return is_quiet;
}
/* temporal masking */
static void
fe_temp_masking(noise_stats_t *noise_stats, powspec_t * buf, powspec_t * peak, int32 num_filt)
{
powspec_t cur_in;
int i;
for (i = 0; i < num_filt; i++) {
cur_in = buf[i];
#ifndef FIXED_POINT
peak[i] *= noise_stats->lambda_t;
if (buf[i] < noise_stats->lambda_t * peak[i])
buf[i] = peak[i] * noise_stats->mu_t;
#else
peak[i] += noise_stats->lambda_t;
if (buf[i] < noise_stats->lambda_t + peak[i])
buf[i] = peak[i] + noise_stats->mu_t;
#endif
if (cur_in > peak[i])
peak[i] = cur_in;
}
}
/* spectral weight smoothing */
static void
fe_weight_smooth(noise_stats_t *noise_stats, powspec_t * buf, powspec_t * coefs, int32 num_filt)
{
int i, j;
int l1, l2;
powspec_t coef;
for (i = 0; i < num_filt; i++) {
l1 = ((i - SMOOTH_WINDOW) > 0) ? (i - SMOOTH_WINDOW) : 0;
l2 = ((i + SMOOTH_WINDOW) <
(num_filt - 1)) ? (i + SMOOTH_WINDOW) : (num_filt - 1);
#ifndef FIXED_POINT
coef = 0;
for (j = l1; j <= l2; j++) {
coef += coefs[j];
}
buf[i] = buf[i] * (coef / (l2 - l1 + 1));
#else
coef = MIN_FIXLOG;
for (j = l1; j <= l2; j++) {
coef = fe_log_add(coef, coefs[j]);
}
buf[i] = buf[i] + coef - noise_stats->smooth_scaling[l2 - l1 + 1];
#endif
}
}
noise_stats_t *
fe_init_noisestats(int num_filters)
{
int i;
noise_stats_t *noise_stats;
noise_stats = (noise_stats_t *) ckd_calloc(1, sizeof(noise_stats_t));
noise_stats->power =
(powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
noise_stats->noise =
(powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
noise_stats->floor =
(powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
noise_stats->peak =
(powspec_t *) ckd_calloc(num_filters, sizeof(powspec_t));
noise_stats->undefined = TRUE;
noise_stats->num_filters = num_filters;
#ifndef FIXED_POINT
noise_stats->lambda_power = LAMBDA_POWER;
noise_stats->comp_lambda_power = 1 - LAMBDA_POWER;
noise_stats->lambda_a = LAMBDA_A;
noise_stats->comp_lambda_a = 1 - LAMBDA_A;
noise_stats->lambda_b = LAMBDA_B;
noise_stats->comp_lambda_b = 1 - LAMBDA_B;
noise_stats->lambda_t = LAMBDA_T;
noise_stats->mu_t = MU_T;
noise_stats->max_gain = MAX_GAIN;
noise_stats->inv_max_gain = 1.0 / MAX_GAIN;
for (i = 1; i < 2 * SMOOTH_WINDOW + 1; i++) {
noise_stats->smooth_scaling[i] = 1.0 / i;
}
#else
noise_stats->lambda_power = FLOAT2FIX(log(LAMBDA_POWER));
noise_stats->comp_lambda_power = FLOAT2FIX(log(1 - LAMBDA_POWER));
noise_stats->lambda_a = FLOAT2FIX(log(LAMBDA_A));
noise_stats->comp_lambda_a = FLOAT2FIX(log(1 - LAMBDA_A));
noise_stats->lambda_b = FLOAT2FIX(log(LAMBDA_B));
noise_stats->comp_lambda_b = FLOAT2FIX(log(1 - LAMBDA_B));
noise_stats->lambda_t = FLOAT2FIX(log(LAMBDA_T));
noise_stats->mu_t = FLOAT2FIX(log(MU_T));
noise_stats->max_gain = FLOAT2FIX(log(MAX_GAIN));
noise_stats->inv_max_gain = FLOAT2FIX(log(1.0 / MAX_GAIN));
for (i = 1; i < 2 * SMOOTH_WINDOW + 3; i++) {
noise_stats->smooth_scaling[i] = FLOAT2FIX(log(i));
}
#endif
#ifdef VAD_DEBUG
vad_stats = fopen("vad_debug", "w");
#endif
return noise_stats;
}
void
fe_reset_noisestats(noise_stats_t * noise_stats)
{
if (noise_stats)
noise_stats->undefined = TRUE;
}
void
fe_free_noisestats(noise_stats_t * noise_stats)
{
ckd_free(noise_stats->power);
ckd_free(noise_stats->noise);
ckd_free(noise_stats->floor);
ckd_free(noise_stats->peak);
ckd_free(noise_stats);
#ifdef VAD_DEBUG
fclose(vad_stats);
E_INFO("Low SNR [%ld] frames; Low volume [%ld] frames\n", (long)low_snr, (long)low_volume);
#endif
}
void
fe_track_snr(fe_t * fe, int32 *in_speech)
{
powspec_t *signal;
powspec_t *gain;
noise_stats_t *noise_stats;
powspec_t *mfspec;
int32 i, num_filts;
int16 is_quiet;
powspec_t lrt, snr;
if (!(fe->remove_noise || fe->remove_silence)) {
*in_speech = TRUE;
return;
}
noise_stats = fe->noise_stats;
mfspec = fe->mfspec;
num_filts = noise_stats->num_filters;
signal = (powspec_t *) ckd_calloc(num_filts, sizeof(powspec_t));
if (noise_stats->undefined) {
noise_stats->slow_peak_sum = FIX2FLOAT(0.0);
for (i = 0; i < num_filts; i++) {
noise_stats->power[i] = mfspec[i];
#ifndef FIXED_POINT
noise_stats->noise[i] = mfspec[i] / noise_stats->max_gain;
noise_stats->floor[i] = mfspec[i] / noise_stats->max_gain;
noise_stats->peak[i] = 0.0;
#else
noise_stats->noise[i] = mfspec[i] - noise_stats->max_gain;;
noise_stats->floor[i] = mfspec[i] - noise_stats->max_gain;
noise_stats->peak[i] = MIN_FIXLOG;
#endif
}
noise_stats->undefined = FALSE;
}
/* Calculate smoothed power */
for (i = 0; i < num_filts; i++) {
#ifndef FIXED_POINT
noise_stats->power[i] =
noise_stats->lambda_power * noise_stats->power[i] + noise_stats->comp_lambda_power * mfspec[i];
#else
noise_stats->power[i] = fe_log_add(noise_stats->lambda_power + noise_stats->power[i],
noise_stats->comp_lambda_power + mfspec[i]);
#endif
}
/* Noise estimation and vad decision */
fe_lower_envelope(noise_stats, noise_stats->power, noise_stats->noise, num_filts);
lrt = FLOAT2FIX(0.0);
for (i = 0; i < num_filts; i++) {
#ifndef FIXED_POINT
signal[i] = noise_stats->power[i] - noise_stats->noise[i];
if (signal[i] < 1.0)
signal[i] = 1.0;
snr = log(noise_stats->power[i] / noise_stats->noise[i]);
#else
signal[i] = fe_log_sub(noise_stats->power[i], noise_stats->noise[i]);
snr = noise_stats->power[i] - noise_stats->noise[i];
#endif
if (snr > lrt)
lrt = snr;
}
is_quiet = fe_is_frame_quiet(noise_stats, signal, num_filts);
#ifdef VAD_DEBUG
if (lrt < fe->vad_threshold)
low_snr++;
else if (is_quiet)
low_volume++;
#endif
#ifndef FIXED_POINT
if (fe->remove_silence && (lrt < fe->vad_threshold || is_quiet)) {
#else
if (fe->remove_silence && (lrt < FLOAT2FIX(fe->vad_threshold) || is_quiet)) {
#endif
*in_speech = FALSE;
} else {
*in_speech = TRUE;
}
#ifdef VAD_DEBUG
#ifndef FIXED_POINT
fprintf(vad_stats, "%.3f %d\n", lrt, *in_speech);
#else
fprintf(vad_stats, "%d %d\n", lrt, *in_speech);
#endif
#endif
fe_lower_envelope(noise_stats, signal, noise_stats->floor, num_filts);
fe_temp_masking(noise_stats, signal, noise_stats->peak, num_filts);
if (!fe->remove_noise) {
/* no need for further calculations if noise cancellation disabled */
ckd_free(signal);
return;
}
for (i = 0; i < num_filts; i++) {
if (signal[i] < noise_stats->floor[i])
signal[i] = noise_stats->floor[i];
}
gain = (powspec_t *) ckd_calloc(num_filts, sizeof(powspec_t));
#ifndef FIXED_POINT
for (i = 0; i < num_filts; i++) {
if (signal[i] < noise_stats->max_gain * noise_stats->power[i])
gain[i] = signal[i] / noise_stats->power[i];
else
gain[i] = noise_stats->max_gain;
if (gain[i] < noise_stats->inv_max_gain)
gain[i] = noise_stats->inv_max_gain;
}
#else
for (i = 0; i < num_filts; i++) {
gain[i] = signal[i] - noise_stats->power[i];
if (gain[i] > noise_stats->max_gain)
gain[i] = noise_stats->max_gain;
if (gain[i] < noise_stats->inv_max_gain)
gain[i] = noise_stats->inv_max_gain;
}
#endif
/* Weight smoothing and time frequency normalization */
fe_weight_smooth(noise_stats, mfspec, gain, num_filts);
ckd_free(gain);
ckd_free(signal);
}
void
fe_vad_hangover(fe_t * fe, mfcc_t * feat, int32 is_speech, int32 store_pcm)
{
if (!fe->vad_data->in_speech) {
fe_prespch_write_cep(fe->vad_data->prespch_buf, feat);
if (store_pcm)
fe_prespch_write_pcm(fe->vad_data->prespch_buf, fe->spch);
}
/* track vad state and deal with cepstrum prespeech buffer */
if (is_speech) {
fe->vad_data->post_speech_frames = 0;
if (!fe->vad_data->in_speech) {
fe->vad_data->pre_speech_frames++;
/* check for transition sil->speech */
if (fe->vad_data->pre_speech_frames >= fe->start_speech) {
fe->vad_data->pre_speech_frames = 0;
fe->vad_data->in_speech = 1;
}
}
} else {
fe->vad_data->pre_speech_frames = 0;
if (fe->vad_data->in_speech) {
fe->vad_data->post_speech_frames++;
/* check for transition speech->sil */
if (fe->vad_data->post_speech_frames >= fe->post_speech) {
fe->vad_data->post_speech_frames = 0;
fe->vad_data->in_speech = 0;
fe_prespch_reset_cep(fe->vad_data->prespch_buf);
fe_prespch_reset_pcm(fe->vad_data->prespch_buf);
}
}
}
}
ckd_alloc.h
Sphinx's memory allocation/deallocation routines.
ckd_free
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
Definition ckd_alloc.c:244
ckd_calloc
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
Definition ckd_alloc.h:248
err.h
Implementation of logging routines.
E_INFO
#define E_INFO(...)
Print logging information to standard error stream.
Definition err.h:114
prim_type.h
Basic type definitions used in Sphinx.
strfuncs.h
Miscellaneous useful string functions.
noise_stats_s
Definition fe_noise.c:83