fsg_model.c
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 1999-2004 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
#include <stdio.h>
#include <string.h>
#include <assert.h>
/* SphinxBase headers. */
#include "sphinxbase/err.h"
#include "sphinxbase/pio.h"
#include "sphinxbase/ckd_alloc.h"
#include "sphinxbase/prim_type.h"
#include "sphinxbase/strfuncs.h"
#include "sphinxbase/hash_table.h"
#include "sphinxbase/fsg_model.h"
#include "sphinxbase/bitvec.h"
struct trans_list_s {
hash_table_t *null_trans; /* Null transitions keyed by state. */
hash_table_t *trans; /* Lists of non-null transitions keyed by state. */
};
struct fsg_arciter_s {
hash_iter_t *itor, *null_itor;
gnode_t *gn;
};
#define FSG_MODEL_BEGIN_DECL "FSG_BEGIN"
#define FSG_MODEL_END_DECL "FSG_END"
#define FSG_MODEL_N_DECL "N"
#define FSG_MODEL_NUM_STATES_DECL "NUM_STATES"
#define FSG_MODEL_S_DECL "S"
#define FSG_MODEL_START_STATE_DECL "START_STATE"
#define FSG_MODEL_F_DECL "F"
#define FSG_MODEL_FINAL_STATE_DECL "FINAL_STATE"
#define FSG_MODEL_T_DECL "T"
#define FSG_MODEL_TRANSITION_DECL "TRANSITION"
#define FSG_MODEL_COMMENT_CHAR '#'
static int32
nextline_str2words(FILE * fp, int32 * lineno,
char **lineptr, char ***wordptr)
{
for (;;) {
size_t len;
int32 n;
ckd_free(*lineptr);
if ((*lineptr = fread_line(fp, &len)) == NULL)
return -1;
(*lineno)++;
if ((*lineptr)[0] == FSG_MODEL_COMMENT_CHAR)
continue; /* Skip comment lines */
n = str2words(*lineptr, NULL, 0);
if (n == 0)
continue; /* Skip blank lines */
/* Abuse of realloc(), but this doesn't have to be fast. */
if (*wordptr == NULL)
*wordptr = ckd_calloc(n, sizeof(**wordptr));
else
*wordptr = ckd_realloc(*wordptr, n * sizeof(**wordptr));
return str2words(*lineptr, *wordptr, n);
}
}
void
fsg_model_trans_add(fsg_model_t * fsg,
int32 from, int32 to, int32 logp, int32 wid)
{
fsg_link_t *link;
glist_t gl;
gnode_t *gn;
if (fsg->trans[from].trans == NULL)
fsg->trans[from].trans = hash_table_new(5, HASH_CASE_YES);
/* Check for duplicate link (i.e., link already exists with label=wid) */
for (gn = gl = fsg_model_trans(fsg, from, to); gn; gn = gnode_next(gn)) {
link = (fsg_link_t *) gnode_ptr(gn);
if (link->wid == wid) {
if (link->logs2prob < logp)
link->logs2prob = logp;
return;
}
}
/* Create transition object */
link = listelem_malloc(fsg->link_alloc);
link->from_state = from;
link->to_state = to;
link->logs2prob = logp;
link->wid = wid;
/* Add it to the list of transitions and update the hash table */
gl = glist_add_ptr(gl, (void *) link);
hash_table_replace_bkey(fsg->trans[from].trans,
(char const *) &link->to_state,
sizeof(link->to_state), gl);
}
int32
fsg_model_tag_trans_add(fsg_model_t * fsg, int32 from, int32 to,
int32 logp, int32 wid)
{
fsg_link_t *link, *link2;
/* Check for transition probability */
if (logp > 0) {
E_FATAL("Null transition prob must be <= 1.0 (state %d -> %d)\n",
from, to);
}
/* Self-loop null transitions (with prob <= 1.0) are redundant */
if (from == to)
return -1;
if (fsg->trans[from].null_trans == NULL)
fsg->trans[from].null_trans = hash_table_new(5, HASH_CASE_YES);
/* Check for a duplicate link; if found, keep the higher prob */
link = fsg_model_null_trans(fsg, from, to);
if (link) {
if (link->logs2prob < logp) {
link->logs2prob = logp;
return 0;
}
else
return -1;
}
/* Create null transition object */
link = listelem_malloc(fsg->link_alloc);
link->from_state = from;
link->to_state = to;
link->logs2prob = logp;
link->wid = -1;
link2 = (fsg_link_t *)
hash_table_enter_bkey(fsg->trans[from].null_trans,
(char const *) &link->to_state,
sizeof(link->to_state), link);
assert(link == link2);
return 1;
}
int32
fsg_model_null_trans_add(fsg_model_t * fsg, int32 from, int32 to,
int32 logp)
{
return fsg_model_tag_trans_add(fsg, from, to, logp, -1);
}
glist_t
fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
{
gnode_t *gn1;
int updated;
fsg_link_t *tl1, *tl2;
int32 k, n;
E_INFO("Computing transitive closure for null transitions\n");
/* If our caller didn't give us a list of null-transitions,
make such a list. Just loop through all the FSG states,
and all the null-transitions in that state (which are kept in
their own hash table). */
if (nulls == NULL) {
int i;
for (i = 0; i < fsg->n_state; ++i) {
hash_iter_t *itor;
hash_table_t *null_trans = fsg->trans[i].null_trans;
if (null_trans == NULL)
continue;
for (itor = hash_table_iter(null_trans);
itor != NULL; itor = hash_table_iter_next(itor)) {
nulls = glist_add_ptr(nulls, hash_entry_val(itor->ent));
}
}
}
/*
* Probably not the most efficient closure implementation, in general, but
* probably reasonably efficient for a sparse null transition matrix.
*/
n = 0;
do {
updated = FALSE;
for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
hash_iter_t *itor;
tl1 = (fsg_link_t *) gnode_ptr(gn1);
assert(tl1->wid < 0);
if (fsg->trans[tl1->to_state].null_trans == NULL)
continue;
for (itor =
hash_table_iter(fsg->trans[tl1->to_state].null_trans);
itor; itor = hash_table_iter_next(itor)) {
tl2 = (fsg_link_t *) hash_entry_val(itor->ent);
k = fsg_model_null_trans_add(fsg,
tl1->from_state,
tl2->to_state,
tl1->logs2prob +
tl2->logs2prob);
if (k >= 0) {
updated = TRUE;
if (k > 0) {
nulls = glist_add_ptr(nulls, (void *)
fsg_model_null_trans
(fsg, tl1->from_state,
tl2->to_state));
n++;
}
}
}
}
} while (updated);
E_INFO("%d null transitions added\n", n);
return nulls;
}
glist_t
fsg_model_trans(fsg_model_t * fsg, int32 i, int32 j)
{
void *val;
if (fsg->trans[i].trans == NULL)
return NULL;
if (hash_table_lookup_bkey(fsg->trans[i].trans, (char const *) &j,
sizeof(j), &val) < 0)
return NULL;
return (glist_t) val;
}
fsg_link_t *
fsg_model_null_trans(fsg_model_t * fsg, int32 i, int32 j)
{
void *val;
if (fsg->trans[i].null_trans == NULL)
return NULL;
if (hash_table_lookup_bkey(fsg->trans[i].null_trans, (char const *) &j,
sizeof(j), &val) < 0)
return NULL;
return (fsg_link_t *) val;
}
fsg_arciter_t *
fsg_model_arcs(fsg_model_t * fsg, int32 i)
{
fsg_arciter_t *itor;
if (fsg->trans[i].trans == NULL && fsg->trans[i].null_trans == NULL)
return NULL;
itor = ckd_calloc(1, sizeof(*itor));
if (fsg->trans[i].null_trans)
itor->null_itor = hash_table_iter(fsg->trans[i].null_trans);
if (fsg->trans[i].trans)
itor->itor = hash_table_iter(fsg->trans[i].trans);
if (itor->itor != NULL)
itor->gn = hash_entry_val(itor->itor->ent);
return itor;
}
fsg_link_t *
fsg_arciter_get(fsg_arciter_t * itor)
{
/* Iterate over non-null arcs first. */
if (itor->gn)
return (fsg_link_t *) gnode_ptr(itor->gn);
else if (itor->null_itor)
return (fsg_link_t *) hash_entry_val(itor->null_itor->ent);
else
return NULL;
}
fsg_arciter_t *
fsg_arciter_next(fsg_arciter_t * itor)
{
/* Iterate over non-null arcs first. */
if (itor->gn) {
itor->gn = gnode_next(itor->gn);
/* Move to the next destination arc. */
if (itor->gn == NULL) {
itor->itor = hash_table_iter_next(itor->itor);
if (itor->itor != NULL)
itor->gn = hash_entry_val(itor->itor->ent);
else if (itor->null_itor == NULL)
goto stop_iteration;
}
}
else {
if (itor->null_itor == NULL)
goto stop_iteration;
itor->null_itor = hash_table_iter_next(itor->null_itor);
if (itor->null_itor == NULL)
goto stop_iteration;
}
return itor;
stop_iteration:
fsg_arciter_free(itor);
return NULL;
}
void
fsg_arciter_free(fsg_arciter_t * itor)
{
if (itor == NULL)
return;
hash_table_iter_free(itor->null_itor);
hash_table_iter_free(itor->itor);
ckd_free(itor);
}
int
fsg_model_word_id(fsg_model_t * fsg, char const *word)
{
int wid;
/* Search for an existing word matching this. */
for (wid = 0; wid < fsg->n_word; ++wid) {
if (0 == strcmp(fsg->vocab[wid], word))
break;
}
/* If not found, add this to the vocab. */
if (wid == fsg->n_word)
return -1;
return wid;
}
int
fsg_model_word_add(fsg_model_t * fsg, char const *word)
{
int wid, old_size;
/* Search for an existing word matching this. */
wid = fsg_model_word_id(fsg, word);
/* If not found, add this to the vocab. */
if (wid == -1) {
wid = fsg->n_word;
if (fsg->n_word == fsg->n_word_alloc) {
old_size = fsg->n_word_alloc;
fsg->n_word_alloc += 10;
fsg->vocab = ckd_realloc(fsg->vocab,
fsg->n_word_alloc *
sizeof(*fsg->vocab));
if (fsg->silwords)
fsg->silwords =
bitvec_realloc(fsg->silwords, old_size,
fsg->n_word_alloc);
if (fsg->altwords)
fsg->altwords =
bitvec_realloc(fsg->altwords, old_size,
fsg->n_word_alloc);
}
++fsg->n_word;
fsg->vocab[wid] = ckd_salloc(word);
}
return wid;
}
int
fsg_model_add_silence(fsg_model_t * fsg, char const *silword,
int state, float32 silprob)
{
int32 logsilp;
int n_trans, silwid, src;
E_INFO("Adding silence transitions for %s to FSG\n", silword);
silwid = fsg_model_word_add(fsg, silword);
logsilp = (int32) (logmath_log(fsg->lmath, silprob) * fsg->lw);
if (fsg->silwords == NULL)
fsg->silwords = bitvec_alloc(fsg->n_word_alloc);
bitvec_set(fsg->silwords, silwid);
n_trans = 0;
if (state == -1) {
for (src = 0; src < fsg->n_state; src++) {
fsg_model_trans_add(fsg, src, src, logsilp, silwid);
++n_trans;
}
}
else {
fsg_model_trans_add(fsg, state, state, logsilp, silwid);
++n_trans;
}
E_INFO("Added %d silence word transitions\n", n_trans);
return n_trans;
}
int
fsg_model_add_alt(fsg_model_t * fsg, char const *baseword,
char const *altword)
{
int i, basewid, altwid;
int ntrans;
/* FIXME: This will get slow, eventually... */
for (basewid = 0; basewid < fsg->n_word; ++basewid)
if (0 == strcmp(fsg->vocab[basewid], baseword))
break;
if (basewid == fsg->n_word) {
E_ERROR("Base word %s not present in FSG vocabulary!\n", baseword);
return -1;
}
altwid = fsg_model_word_add(fsg, altword);
if (fsg->altwords == NULL)
fsg->altwords = bitvec_alloc(fsg->n_word_alloc);
bitvec_set(fsg->altwords, altwid);
if (fsg_model_is_filler(fsg, basewid)) {
if (fsg->silwords == NULL)
fsg->silwords = bitvec_alloc(fsg->n_word_alloc);
bitvec_set(fsg->silwords, altwid);
}
E_DEBUG(2, ("Adding alternate word transitions (%s,%s) to FSG\n",
baseword, altword));
/* Look for all transitions involving baseword and duplicate them. */
/* FIXME: This will also get slow, eventually... */
ntrans = 0;
for (i = 0; i < fsg->n_state; ++i) {
hash_iter_t *itor;
if (fsg->trans[i].trans == NULL)
continue;
for (itor = hash_table_iter(fsg->trans[i].trans); itor;
itor = hash_table_iter_next(itor)) {
glist_t trans;
gnode_t *gn;
trans = hash_entry_val(itor->ent);
for (gn = trans; gn; gn = gnode_next(gn)) {
fsg_link_t *fl = gnode_ptr(gn);
if (fl->wid == basewid) {
fsg_link_t *link;
/* Create transition object */
link = listelem_malloc(fsg->link_alloc);
link->from_state = fl->from_state;
link->to_state = fl->to_state;
link->logs2prob = fl->logs2prob; /* FIXME!!!??? */
link->wid = altwid;
trans = glist_add_ptr(trans, (void *) link);
++ntrans;
}
}
hash_entry_val(itor->ent) = trans;
}
}
E_DEBUG(2, ("Added %d alternate word transitions\n", ntrans));
return ntrans;
}
fsg_model_t *
fsg_model_init(char const *name, logmath_t * lmath, float32 lw,
int32 n_state)
{
fsg_model_t *fsg;
/* Allocate basic stuff. */
fsg = ckd_calloc(1, sizeof(*fsg));
fsg->refcount = 1;
fsg->link_alloc = listelem_alloc_init(sizeof(fsg_link_t));
fsg->lmath = lmath;
fsg->name = name ? ckd_salloc(name) : NULL;
fsg->n_state = n_state;
fsg->lw = lw;
fsg->trans = ckd_calloc(fsg->n_state, sizeof(*fsg->trans));
return fsg;
}
fsg_model_t *
fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw)
{
fsg_model_t *fsg;
hash_table_t *vocab;
hash_iter_t *itor;
int32 lastwid;
char **wordptr;
char *lineptr;
char *fsgname;
int32 lineno;
int32 n, i, j;
int n_state, n_trans, n_null_trans;
glist_t nulls;
float32 p;
lineno = 0;
vocab = hash_table_new(32, FALSE);
wordptr = NULL;
lineptr = NULL;
nulls = NULL;
fsgname = NULL;
fsg = NULL;
/* Scan upto FSG_BEGIN header */
for (;;) {
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if (n < 0) {
E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL);
goto parse_error;
}
if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) {
if (n > 2) {
E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n",
lineno);
goto parse_error;
}
break;
}
}
/* Save FSG name, or it will get clobbered below :(.
* If name is missing, try the default.
*/
if (n == 2) {
fsgname = ckd_salloc(wordptr[1]);
}
else {
E_WARN("FSG name is missing\n");
fsgname = ckd_salloc("unknown");
}
/* Read #states */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &n_state) != 1)
|| (n_state <= 0)) {
E_ERROR
("Line[%d]: #states declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Now create the FSG. */
fsg = fsg_model_init(fsgname, lmath, lw, n_state);
ckd_free(fsgname);
fsgname = NULL;
/* Read start state */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1)
|| (fsg->start_state < 0)
|| (fsg->start_state >= fsg->n_state)) {
E_ERROR
("Line[%d]: start state declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Read final state */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1)
|| (fsg->final_state < 0)
|| (fsg->final_state >= fsg->n_state)) {
E_ERROR
("Line[%d]: final state declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Read transitions */
lastwid = 0;
n_trans = n_null_trans = 0;
for (;;) {
int32 wid, tprob;
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if (n <= 0) {
E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
lineno);
goto parse_error;
}
if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) {
break;
}
if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0)
|| (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) {
if (((n != 4) && (n != 5))
|| (sscanf(wordptr[1], "%d", &i) != 1)
|| (sscanf(wordptr[2], "%d", &j) != 1)
|| (i < 0) || (i >= fsg->n_state)
|| (j < 0) || (j >= fsg->n_state)) {
E_ERROR
("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n",
lineno);
goto parse_error;
}
p = atof_c(wordptr[3]);
if ((p <= 0.0) || (p > 1.0)) {
E_ERROR
("Line[%d]: transition spec malformed; Expecting float as transition probability\n",
lineno);
goto parse_error;
}
}
else {
E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
lineno);
goto parse_error;
}
tprob = (int32) (logmath_log(lmath, p) * fsg->lw);
/* Add word to "dictionary". */
if (n > 4) {
if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) {
(void) hash_table_enter_int32(vocab,
ckd_salloc(wordptr[4]),
lastwid);
wid = lastwid;
++lastwid;
}
fsg_model_trans_add(fsg, i, j, tprob, wid);
++n_trans;
}
else {
if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) {
++n_null_trans;
nulls =
glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j));
}
}
}
E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n",
fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans);
/* Now create a string table from the "dictionary" */
fsg->n_word = hash_table_inuse(vocab);
fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */
fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab));
for (itor = hash_table_iter(vocab); itor;
itor = hash_table_iter_next(itor)) {
char const *word = hash_entry_key(itor->ent);
int32 wid = (int32) (long) hash_entry_val(itor->ent);
fsg->vocab[wid] = (char *) word;
}
hash_table_free(vocab);
/* Do transitive closure on null transitions */
nulls = fsg_model_null_trans_closure(fsg, nulls);
glist_free(nulls);
ckd_free(lineptr);
ckd_free(wordptr);
return fsg;
parse_error:
for (itor = hash_table_iter(vocab); itor;
itor = hash_table_iter_next(itor))
ckd_free((char *) hash_entry_key(itor->ent));
glist_free(nulls);
hash_table_free(vocab);
ckd_free(fsgname);
ckd_free(lineptr);
ckd_free(wordptr);
fsg_model_free(fsg);
return NULL;
}
fsg_model_t *
fsg_model_readfile(const char *file, logmath_t * lmath, float32 lw)
{
FILE *fp;
fsg_model_t *fsg;
if ((fp = fopen(file, "r")) == NULL) {
E_ERROR_SYSTEM("Failed to open FSG file '%s' for reading", file);
return NULL;
}
fsg = fsg_model_read(fp, lmath, lw);
fclose(fp);
return fsg;
}
fsg_model_t *
fsg_model_retain(fsg_model_t * fsg)
{
++fsg->refcount;
return fsg;
}
static void
trans_list_free(fsg_model_t * fsg, int32 i)
{
hash_iter_t *itor;
/* FIXME (maybe): FSG links will all get freed when we call
* listelem_alloc_free() so don't bother freeing them explicitly
* here. */
if (fsg->trans[i].trans) {
for (itor = hash_table_iter(fsg->trans[i].trans);
itor; itor = hash_table_iter_next(itor)) {
glist_t gl = (glist_t) hash_entry_val(itor->ent);
glist_free(gl);
}
}
hash_table_free(fsg->trans[i].trans);
hash_table_free(fsg->trans[i].null_trans);
}
int
fsg_model_free(fsg_model_t * fsg)
{
int i;
if (fsg == NULL)
return 0;
if (--fsg->refcount > 0)
return fsg->refcount;
for (i = 0; i < fsg->n_word; ++i)
ckd_free(fsg->vocab[i]);
for (i = 0; i < fsg->n_state; ++i)
trans_list_free(fsg, i);
ckd_free(fsg->trans);
ckd_free(fsg->vocab);
listelem_alloc_free(fsg->link_alloc);
bitvec_free(fsg->silwords);
bitvec_free(fsg->altwords);
ckd_free(fsg->name);
ckd_free(fsg);
return 0;
}
void
fsg_model_write(fsg_model_t * fsg, FILE * fp)
{
int32 i;
fprintf(fp, "%s %s\n", FSG_MODEL_BEGIN_DECL,
fsg->name ? fsg->name : "");
fprintf(fp, "%s %d\n", FSG_MODEL_NUM_STATES_DECL, fsg->n_state);
fprintf(fp, "%s %d\n", FSG_MODEL_START_STATE_DECL, fsg->start_state);
fprintf(fp, "%s %d\n", FSG_MODEL_FINAL_STATE_DECL, fsg->final_state);
for (i = 0; i < fsg->n_state; i++) {
fsg_arciter_t *itor;
for (itor = fsg_model_arcs(fsg, i); itor;
itor = fsg_arciter_next(itor)) {
fsg_link_t *tl = fsg_arciter_get(itor);
fprintf(fp, "%s %d %d %f %s\n", FSG_MODEL_TRANSITION_DECL,
tl->from_state, tl->to_state,
logmath_exp(fsg->lmath,
(int32) (tl->logs2prob / fsg->lw)),
(tl->wid < 0) ? "" : fsg_model_word_str(fsg, tl->wid));
}
}
fprintf(fp, "%s\n", FSG_MODEL_END_DECL);
fflush(fp);
}
void
fsg_model_writefile(fsg_model_t * fsg, char const *file)
{
FILE *fp;
assert(fsg);
E_INFO("Writing FSG file '%s'\n", file);
if ((fp = fopen(file, "w")) == NULL) {
E_ERROR_SYSTEM("Failed to open FSG file '%s' for reading", file);
return;
}
fsg_model_write(fsg, fp);
fclose(fp);
}
static void
fsg_model_write_fsm_trans(fsg_model_t * fsg, int i, FILE * fp)
{
fsg_arciter_t *itor;
for (itor = fsg_model_arcs(fsg, i); itor;
itor = fsg_arciter_next(itor)) {
fsg_link_t *tl = fsg_arciter_get(itor);
fprintf(fp, "%d %d %s %f\n",
tl->from_state, tl->to_state,
(tl->wid < 0) ? "<eps>" : fsg_model_word_str(fsg, tl->wid),
-logmath_log_to_ln(fsg->lmath, tl->logs2prob / fsg->lw));
}
}
void
fsg_model_write_fsm(fsg_model_t * fsg, FILE * fp)
{
int i;
/* Write transitions from initial state first. */
fsg_model_write_fsm_trans(fsg, fsg_model_start_state(fsg), fp);
/* Other states. */
for (i = 0; i < fsg->n_state; i++) {
if (i == fsg_model_start_state(fsg))
continue;
fsg_model_write_fsm_trans(fsg, i, fp);
}
/* Final state. */
fprintf(fp, "%d 0\n", fsg_model_final_state(fsg));
fflush(fp);
}
void
fsg_model_writefile_fsm(fsg_model_t * fsg, char const *file)
{
FILE *fp;
assert(fsg);
E_INFO("Writing FSM file '%s'\n", file);
if ((fp = fopen(file, "w")) == NULL) {
E_ERROR_SYSTEM("Failed to open fsm file '%s' for writing", file);
return;
}
fsg_model_write_fsm(fsg, fp);
fclose(fp);
}
void
fsg_model_write_symtab(fsg_model_t * fsg, FILE * file)
{
int i;
fprintf(file, "<eps> 0\n");
for (i = 0; i < fsg_model_n_word(fsg); ++i) {
fprintf(file, "%s %d\n", fsg_model_word_str(fsg, i), i + 1);
}
fflush(file);
}
void
fsg_model_writefile_symtab(fsg_model_t * fsg, char const *file)
{
FILE *fp;
assert(fsg);
E_INFO("Writing FSM symbol table '%s'\n", file);
if ((fp = fopen(file, "w")) == NULL) {
E_ERROR("Failed to open symbol table '%s' for writing", file);
return;
}
fsg_model_write_symtab(fsg, fp);
fclose(fp);
}
bitvec.h
An implementation of bit vectors.
bitvec_free
#define bitvec_free(v)
Free a bit vector.
Definition bitvec.h:87
bitvec_realloc
SPHINXBASE_EXPORT bitvec_t * bitvec_realloc(bitvec_t *vec, size_t old_len, size_t new_len)
Resize a bit vector, clear the remaining bits.
Definition bitvec.c:64
bitvec_alloc
#define bitvec_alloc(n)
Allocate a bit vector, all bits are clear.
Definition bitvec.h:75
bitvec_set
#define bitvec_set(v, b)
Set the b-th bit of bit vector v.
Definition bitvec.h:95
ckd_alloc.h
Sphinx's memory allocation/deallocation routines.
ckd_free
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
Definition ckd_alloc.c:244
ckd_calloc
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
Definition ckd_alloc.h:248
ckd_salloc
#define ckd_salloc(ptr)
Macro for ckd_salloc.
Definition ckd_alloc.h:264
ckd_realloc
#define ckd_realloc(ptr, sz)
Macro for ckd_realloc.
Definition ckd_alloc.h:258
err.h
Implementation of logging routines.
E_ERROR
#define E_ERROR(...)
Print error message to error log.
Definition err.h:104
E_INFO
#define E_INFO(...)
Print logging information to standard error stream.
Definition err.h:114
E_FATAL
#define E_FATAL(...)
Exit with non-zero status after error message.
Definition err.h:81
E_ERROR_SYSTEM
#define E_ERROR_SYSTEM(...)
Print error text; Call perror("");.
Definition err.h:99
E_WARN
#define E_WARN(...)
Print warning message to error log.
Definition err.h:109
E_DEBUG
#define E_DEBUG(level, x)
Print debugging information to standard error stream.
Definition err.h:144
gnode_t
struct gnode_s gnode_t
A node in a generic list.
glist_free
SPHINXBASE_EXPORT void glist_free(glist_t g)
Free the given generic list; user-defined data contained within is not automatically freed.
Definition glist.c:133
glist_add_ptr
SPHINXBASE_EXPORT glist_t glist_add_ptr(glist_t g, void *ptr)
Create and prepend a new list node, with the given user-defined data, at the HEAD of the given generi...
Definition glist.c:74
gnode_ptr
#define gnode_ptr(g)
Head of a list of gnodes.
Definition glist.h:109
hash_table.h
Hash table implementation.
hash_table_free
SPHINXBASE_EXPORT void hash_table_free(hash_table_t *h)
Free the specified hash table; the caller is responsible for freeing the key strings pointed to by th...
Definition hash_table.c:688
hash_table_iter_next
SPHINXBASE_EXPORT hash_iter_t * hash_table_iter_next(hash_iter_t *itor)
Get the next key-value pair in iteration.
Definition hash_table.c:656
hash_table_enter_int32
#define hash_table_enter_int32(h, k, v)
Add a 32-bit integer value to a hash table.
Definition hash_table.h:228
hash_table_enter_bkey
SPHINXBASE_EXPORT void * hash_table_enter_bkey(hash_table_t *h, const char *key, size_t len, void *val)
Like hash_table_enter, but with an explicitly specified key length, instead of a NULL-terminated,...
Definition hash_table.c:535
hash_table_replace_bkey
SPHINXBASE_EXPORT void * hash_table_replace_bkey(hash_table_t *h, const char *key, size_t len, void *val)
Like hash_table_replace, but with an explicitly specified key length, instead of a NULL-terminated,...
Definition hash_table.c:548
hash_table_iter_free
SPHINXBASE_EXPORT void hash_table_iter_free(hash_iter_t *itor)
Delete an unfinished iterator.
Definition hash_table.c:682
hash_table_lookup_bkey
SPHINXBASE_EXPORT int32 hash_table_lookup_bkey(hash_table_t *h, const char *key, size_t len, void **val)
Like hash_lookup, but with an explicitly specified key length, instead of a NULL-terminated,...
Definition hash_table.c:337
hash_entry_val
#define hash_entry_val(e)
Access macros.
Definition hash_table.h:175
hash_table_lookup_int32
SPHINXBASE_EXPORT int32 hash_table_lookup_int32(hash_table_t *h, const char *key, int32 *val)
Look up a 32-bit integer value in a hash table.
Definition hash_table.c:322
hash_table_iter
SPHINXBASE_EXPORT hash_iter_t * hash_table_iter(hash_table_t *h)
Start iterating over key-value pairs in a hash table.
Definition hash_table.c:646
hash_table_new
SPHINXBASE_EXPORT hash_table_t * hash_table_new(int32 size, int32 casearg)
Allocate a new hash table for a given expected size.
Definition hash_table.c:158
listelem_alloc_free
SPHINXBASE_EXPORT void listelem_alloc_free(listelem_alloc_t *le)
Finalize and release all memory associated with a list element allocator.
Definition listelem_alloc.c:121
listelem_alloc_init
SPHINXBASE_EXPORT listelem_alloc_t * listelem_alloc_init(size_t elemsize)
Initialize and return a list element allocator.
Definition listelem_alloc.c:87
listelem_malloc
#define listelem_malloc(le)
Allocate a list element and return pointer to it.
Definition listelem_alloc.h:86
logmath_log_to_ln
SPHINXBASE_EXPORT float64 logmath_log_to_ln(logmath_t *lmath, int logb_p)
Convert integer log in base B to natural log (in floating point).
Definition logmath.c:468
logmath_t
struct logmath_s logmath_t
Integer log math computation class.
Definition logmath.h:108
logmath_exp
SPHINXBASE_EXPORT float64 logmath_exp(logmath_t *lmath, int logb_p)
Convert integer log in base B to linear floating point.
Definition logmath.c:456
logmath_log
SPHINXBASE_EXPORT int logmath_log(logmath_t *lmath, float64 p)
Convert linear floating point number to integer log in base B.
Definition logmath.c:447
pio.h
file IO related operations.
fread_line
SPHINXBASE_EXPORT char * fread_line(FILE *stream, size_t *out_len)
Read a line of arbitrary length from a file and return it as a newly allocated string.
Definition pio.c:377
prim_type.h
Basic type definitions used in Sphinx.
strfuncs.h
Miscellaneous useful string functions.
str2words
SPHINXBASE_EXPORT int32 str2words(char *line, char **wptr, int32 n_wptr)
Convert a line to an array of "words", based on whitespace separators.
Definition strfuncs.c:123
atof_c
SPHINXBASE_EXPORT double atof_c(char const *str)
Locale independent version of atof().
Definition strfuncs.c:55
fsg_arciter_s
Implementation of arc iterator.
Definition fsg_model.c:64
fsg_link_s::logs2prob
int32 logs2prob
log(transition probability)*lw
Definition fsg_model.h:77
fsg_link_s::wid
int32 wid
Word-ID; <0 if epsilon or null transition.
Definition fsg_model.h:78
fsg_model_s::n_word_alloc
int32 n_word_alloc
Number of words allocated in vocab.
Definition fsg_model.h:103
fsg_model_s::start_state
int32 start_state
Must be in the range [0..n_state-1].
Definition fsg_model.h:109
fsg_model_s::vocab
char ** vocab
Vocabulary for this FSG.
Definition fsg_model.h:104
fsg_model_s::n_state
int32 n_state
number of states in FSG
Definition fsg_model.h:108
fsg_model_s::n_word
int32 n_word
Number of unique words in this FSG.
Definition fsg_model.h:102
fsg_model_s::lmath
logmath_t * lmath
Pointer to log math computation object.
Definition fsg_model.h:107
fsg_model_s::name
char * name
A unique string identifier for this FSG.
Definition fsg_model.h:101
fsg_model_s::silwords
bitvec_t * silwords
Indicates which words are silence/fillers.
Definition fsg_model.h:105
fsg_model_s::link_alloc
listelem_alloc_t * link_alloc
Allocator for FSG links.
Definition fsg_model.h:114
fsg_model_s::trans
trans_list_t * trans
Transitions out of each state, if any.
Definition fsg_model.h:113
fsg_model_s::final_state
int32 final_state
Must be in the range [0..n_state-1].
Definition fsg_model.h:110
fsg_model_s::altwords
bitvec_t * altwords
Indicates which words are pronunciation alternates.
Definition fsg_model.h:106
fsg_model_s::lw
float32 lw
Language weight that's been applied to transition logprobs.
Definition fsg_model.h:111
fsg_model_s::refcount
int refcount
Reference count.
Definition fsg_model.h:100
hash_iter_s::ent
hash_entry_t * ent
Current entry in that table.
Definition hash_table.h:170
trans_list_s
Adjacency list (opaque) for a state in an FSG.
Definition fsg_model.c:56