state_align_search.c
Go to the documentation of this file.
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 2010 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* This work was supported in part by funding from the Defense Advanced
* Research Projects Agency and the National Science Foundation of the
* United States of America, and the CMU Sphinx Speech Consortium.
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
#include "state_align_search.h"
static int
state_align_search_start(ps_search_t *search)
{
state_align_search_t *sas = (state_align_search_t *)search;
/* Activate the initial state. */
hmm_enter(sas->hmms, 0, 0, 0);
return 0;
}
static void
renormalize_hmms(state_align_search_t *sas, int frame_idx, int32 norm)
{
int i;
for (i = 0; i < sas->n_phones; ++i)
hmm_normalize(sas->hmms + i, norm);
}
static int32
evaluate_hmms(state_align_search_t *sas, int16 const *senscr, int frame_idx)
{
int32 bs = WORST_SCORE;
int i;
hmm_context_set_senscore(sas->hmmctx, senscr);
for (i = 0; i < sas->n_phones; ++i) {
hmm_t *hmm = sas->hmms + i;
int32 score;
if (hmm_frame(hmm) < frame_idx)
continue;
score = hmm_vit_eval(hmm);
if (score BETTER_THAN bs) {
bs = score;
}
}
return bs;
}
static void
prune_hmms(state_align_search_t *sas, int frame_idx)
{
int nf = frame_idx + 1;
int i;
/* Check all phones to see if they remain active in the next frame. */
for (i = 0; i < sas->n_phones; ++i) {
hmm_t *hmm = sas->hmms + i;
if (hmm_frame(hmm) < frame_idx)
continue;
hmm_frame(hmm) = nf;
}
}
static void
phone_transition(state_align_search_t *sas, int frame_idx)
{
int nf = frame_idx + 1;
int i;
for (i = 0; i < sas->n_phones - 1; ++i) {
hmm_t *hmm, *nhmm;
int32 newphone_score;
hmm = sas->hmms + i;
if (hmm_frame(hmm) != nf)
continue;
newphone_score = hmm_out_score(hmm);
/* Transition into next phone using the usual Viterbi rule. */
nhmm = hmm + 1;
if (hmm_frame(nhmm) < frame_idx
|| newphone_score BETTER_THAN hmm_in_score(nhmm)) {
hmm_enter(nhmm, newphone_score, hmm_out_history(hmm), nf);
}
}
}
#define TOKEN_STEP 20
static void
extend_tokenstack(state_align_search_t *sas, int frame_idx)
{
if (frame_idx >= sas->n_fr_alloc) {
sas->n_fr_alloc = frame_idx + TOKEN_STEP + 1;
sas->tokens = ckd_realloc(sas->tokens,
sas->n_emit_state * sas->n_fr_alloc
* sizeof(*sas->tokens));
}
memset(sas->tokens + frame_idx * sas->n_emit_state, 0xff,
sas->n_emit_state * sizeof(*sas->tokens));
}
static void
record_transitions(state_align_search_t *sas, int frame_idx)
{
state_align_hist_t *tokens;
int i;
/* Push another frame of tokens on the stack. */
extend_tokenstack(sas, frame_idx);
tokens = sas->tokens + frame_idx * sas->n_emit_state;
/* Scan all active HMMs */
for (i = 0; i < sas->n_phones; ++i) {
hmm_t *hmm = sas->hmms + i;
int j;
if (hmm_frame(hmm) < frame_idx)
continue;
for (j = 0; j < sas->hmmctx->n_emit_state; ++j) {
int state_idx = i * sas->hmmctx->n_emit_state + j;
/* Record their backpointers on the token stack. */
tokens[state_idx].id = hmm_history(hmm, j);
tokens[state_idx].score = hmm_score(hmm, j);
/* Update backpointer fields with state index. */
hmm_history(hmm, j) = state_idx;
}
}
}
static int
state_align_search_step(ps_search_t *search, int frame_idx)
{
state_align_search_t *sas = (state_align_search_t *)search;
acmod_t *acmod = ps_search_acmod(search);
int16 const *senscr;
int i;
/* Calculate senone scores. */
for (i = 0; i < sas->n_phones; ++i)
acmod_activate_hmm(acmod, sas->hmms + i);
senscr = acmod_score(acmod, &frame_idx);
/* Renormalize here if needed. */
/* FIXME: Make sure to (unit-)test this!!! */
if ((sas->best_score - 0x300000) WORSE_THAN WORST_SCORE) {
E_INFO("Renormalizing Scores at frame %d, best score %d\n",
frame_idx, sas->best_score);
renormalize_hmms(sas, frame_idx, sas->best_score);
}
/* Viterbi step. */
sas->best_score = evaluate_hmms(sas, senscr, frame_idx);
prune_hmms(sas, frame_idx);
/* Transition out of non-emitting states. */
phone_transition(sas, frame_idx);
/* Generate new tokens from best path results. */
record_transitions(sas, frame_idx);
/* Update frame counter */
sas->frame = frame_idx;
return 0;
}
static int
state_align_search_finish(ps_search_t *search)
{
state_align_search_t *sas = (state_align_search_t *)search;
hmm_t *final_phone = sas->hmms + sas->n_phones - 1;
ps_alignment_iter_t *itor;
ps_alignment_entry_t *ent;
int last_frame, cur_frame;
state_align_hist_t last, cur;
/* Best state exiting the last cur_frame. */
last.id = cur.id = hmm_out_history(final_phone);
last.score = hmm_out_score(final_phone);
if (last.id == 0xffff) {
E_ERROR("Failed to reach final state in alignment\n");
return -1;
}
itor = ps_alignment_states(sas->al);
last_frame = sas->frame + 1;
for (cur_frame = sas->frame - 1; cur_frame >= 0; --cur_frame) {
cur = sas->tokens[cur_frame * sas->n_emit_state + cur.id];
/* State boundary, update alignment entry for next state. */
if (cur.id != last.id) {
itor = ps_alignment_iter_goto(itor, last.id);
assert(itor != NULL);
ent = ps_alignment_iter_get(itor);
ent->start = cur_frame + 1;
ent->duration = last_frame - ent->start;
ent->score = last.score - cur.score;
E_DEBUG(1,("state %d start %d end %d\n", last.id,
ent->start, last_frame));
last = cur;
last_frame = cur_frame + 1;
}
}
/* Update alignment entry for initial state. */
itor = ps_alignment_iter_goto(itor, 0);
assert(itor != NULL);
ent = ps_alignment_iter_get(itor);
ent->start = 0;
ent->duration = last_frame;
E_DEBUG(1,("state %d start %d end %d\n", 0,
ent->start, last_frame));
ps_alignment_iter_free(itor);
ps_alignment_propagate(sas->al);
return 0;
}
static int
state_align_search_reinit(ps_search_t *search, dict_t *dict, dict2pid_t *d2p)
{
/* This does nothing. */
return 0;
}
static void
state_align_search_free(ps_search_t *search)
{
state_align_search_t *sas = (state_align_search_t *)search;
ps_search_base_free(search);
ckd_free(sas->hmms);
ckd_free(sas->tokens);
hmm_context_free(sas->hmmctx);
ckd_free(sas);
}
static ps_searchfuncs_t state_align_search_funcs = {
/* start: */ state_align_search_start,
/* step: */ state_align_search_step,
/* finish: */ state_align_search_finish,
/* reinit: */ state_align_search_reinit,
/* free: */ state_align_search_free,
/* lattice: */ NULL,
/* hyp: */ NULL,
/* prob: */ NULL,
/* seg_iter: */ NULL,
};
ps_search_t *
state_align_search_init(const char *name,
cmd_ln_t *config,
acmod_t *acmod,
ps_alignment_t *al)
{
state_align_search_t *sas;
ps_alignment_iter_t *itor;
hmm_t *hmm;
sas = ckd_calloc(1, sizeof(*sas));
ps_search_init(ps_search_base(sas), &state_align_search_funcs,
PS_SEARCH_TYPE_STATE_ALIGN, name,
config, acmod, al->d2p->dict, al->d2p);
sas->hmmctx = hmm_context_init(bin_mdef_n_emit_state(acmod->mdef),
acmod->tmat->tp, NULL, acmod->mdef->sseq);
if (sas->hmmctx == NULL) {
ckd_free(sas);
return NULL;
}
sas->al = al;
/* Generate HMM vector from phone level of alignment. */
sas->n_phones = ps_alignment_n_phones(al);
sas->n_emit_state = ps_alignment_n_states(al);
sas->hmms = ckd_calloc(sas->n_phones, sizeof(*sas->hmms));
for (hmm = sas->hmms, itor = ps_alignment_phones(al); itor;
++hmm, itor = ps_alignment_iter_next(itor)) {
ps_alignment_entry_t *ent = ps_alignment_iter_get(itor);
hmm_init(sas->hmmctx, hmm, FALSE,
ent->id.pid.ssid, ent->id.pid.tmatid);
}
return ps_search_base(sas);
}
acmod_activate_hmm
void acmod_activate_hmm(acmod_t *acmod, hmm_t *hmm)
Activate senones associated with an HMM.
Definition acmod.c:1213
acmod_score
int16 const * acmod_score(acmod_t *acmod, int *inout_frame_idx)
Score one frame of data.
Definition acmod.c:1106
BETTER_THAN
#define BETTER_THAN
Is one score better than another?
Definition hmm.h:95
hmm_context_set_senscore
#define hmm_context_set_senscore(ctx, senscr)
Change the senone score array for a context.
Definition hmm.h:227
WORST_SCORE
#define WORST_SCORE
Large "bad" score.
Definition hmm.h:84
WORSE_THAN
#define WORSE_THAN
Is one score worse than another?
Definition hmm.h:100
ps_alignment_states
ps_alignment_iter_t * ps_alignment_states(ps_alignment_t *al)
Iterate over the alignment starting at the first state.
Definition ps_alignment.c:397
ps_alignment_iter_next
ps_alignment_iter_t * ps_alignment_iter_next(ps_alignment_iter_t *itor)
Move an alignment iterator forward.
Definition ps_alignment.c:437
ps_alignment_iter_goto
ps_alignment_iter_t * ps_alignment_iter_goto(ps_alignment_iter_t *itor, int pos)
Move alignment iterator to given index.
Definition ps_alignment.c:424
ps_alignment_iter_get
ps_alignment_entry_t * ps_alignment_iter_get(ps_alignment_iter_t *itor)
Get the alignment entry pointed to by an iterator.
Definition ps_alignment.c:411
ps_alignment_iter_free
int ps_alignment_iter_free(ps_alignment_iter_t *itor)
Release an iterator before completing all iterations.
Definition ps_alignment.c:417
ps_alignment_n_phones
int ps_alignment_n_phones(ps_alignment_t *al)
Number of phones.
Definition ps_alignment.c:357
ps_alignment_n_states
int ps_alignment_n_states(ps_alignment_t *al)
Number of states.
Definition ps_alignment.c:363
ps_alignment_phones
ps_alignment_iter_t * ps_alignment_phones(ps_alignment_t *al)
Iterate over the alignment starting at the first phone.
Definition ps_alignment.c:383
ps_alignment_propagate
int ps_alignment_propagate(ps_alignment_t *al)
Propagate timing information up from state sequence.
Definition ps_alignment.c:313
state_align_search.h
State (and phone and word) alignment search.
acmod_s
Acoustic model structure.
Definition acmod.h:148
acmod_s::mdef
bin_mdef_t * mdef
Model definition.
Definition acmod.h:159
acmod_s::tmat
tmat_t * tmat
Transition matrices.
Definition acmod.h:160
bin_mdef_s::sseq
uint16 ** sseq
Unique senone sequences (2D array built at load time)
Definition bin_mdef.h:134
dict2pid_t
Building composite triphone (as well as word internal triphones) with the dictionary.
Definition dict2pid.h:84
dict2pid_t::dict
dict_t * dict
Dictionary this table refers to.
Definition dict2pid.h:89
dict_t
a structure for a dictionary.
Definition dict.h:76
hmm_t
An individual HMM among the HMM search space.
ps_alignment_entry_s
Definition ps_alignment.h:56
ps_alignment_iter_s
Definition ps_alignment.h:88
ps_alignment_s
Definition ps_alignment.h:80
ps_search_s
Base structure for search module.
Definition pocketsphinx_internal.h:98
ps_searchfuncs_s
V-table for search algorithm.
Definition pocketsphinx_internal.h:82
state_align_hist_s
History structure.
Definition state_align_search.h:57
state_align_search_s
Phone loop search structure.
Definition state_align_search.h:66
state_align_search_s::best_score
int32 best_score
Best score in current frame.
Definition state_align_search.h:74
state_align_search_s::tokens
state_align_hist_t * tokens
Tokens (backpointers) for state alignment.
Definition state_align_search.h:77
state_align_search_s::al
ps_alignment_t * al
Alignment structure being operated on.
Definition state_align_search.h:69
state_align_search_s::hmmctx
hmm_context_t * hmmctx
HMM context structure.
Definition state_align_search.h:68
state_align_search_s::hmms
hmm_t * hmms
Vector of HMMs corresponding to phone level.
Definition state_align_search.h:70
state_align_search_s::n_emit_state
int n_emit_state
Number of emitting states (tokens per frame)
Definition state_align_search.h:76
state_align_search_s::n_fr_alloc
int n_fr_alloc
Number of frames of tokens allocated.
Definition state_align_search.h:78
state_align_search_s::frame
int frame
Current frame being processed.
Definition state_align_search.h:73
state_align_search_s::n_phones
int n_phones
Number of HMMs (phones).
Definition state_align_search.h:71
tmat_t::tp
uint8 *** tp
The transition matrices; kept in the same scale as acoustic scores; tp[tmatid][from-state][to-state].
Definition tmat.h:56