sphinx_lm_eval.c
Go to the documentation of this file.
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 2008 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* This work was supported in part by funding from the Defense Advanced
* Research Projects Agency and the National Science Foundation of the
* United States of America, and the CMU Sphinx Speech Consortium.
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
#include <sphinxbase/logmath.h>
#include <sphinxbase/ngram_model.h>
#include <sphinxbase/cmd_ln.h>
#include <sphinxbase/ckd_alloc.h>
#include <sphinxbase/err.h>
#include <sphinxbase/pio.h>
#include <sphinxbase/strfuncs.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
static const arg_t defn[] = {
{ "-help",
ARG_BOOLEAN,
"no",
"Shows the usage of the tool"},
{ "-logbase",
ARG_FLOAT64,
"1.0001",
"Base in which all log-likelihoods calculated" },
{ "-lm",
ARG_STRING,
NULL,
"Language model file"},
{ "-probdef",
ARG_STRING,
NULL,
"Probability definition file for classes in LM"},
{ "-lmctlfn",
ARG_STRING,
NULL,
"Control file listing a set of language models"},
{ "-lmname",
ARG_STRING,
NULL,
"Name of language model in -lmctlfn to use for all utterances" },
{ "-lsn",
ARG_STRING,
NULL,
"Transcription file to evaluate"},
{ "-text",
ARG_STRING,
"Text string to evaluate"},
{ "-mmap",
ARG_BOOLEAN,
"no",
"Use memory-mapped I/O for reading binary LM files"},
{ "-lw",
ARG_FLOAT32,
"1.0",
"Language model weight" },
{ "-wip",
ARG_FLOAT32,
"1.0",
"Word insertion probability" },
{ "-verbose",
ARG_BOOLEAN,
"no",
"Print details of perplexity calculation" },
/* FIXME: Support -lmstartsym, -lmendsym, -lmctlfn, -ctl_lm */
{ NULL, 0, NULL, NULL }
};
static int verbose;
static int
calc_entropy(ngram_model_t *lm, char **words, int32 n,
int32 *out_n_ccs, int32 *out_n_oovs, int32 *out_lm_score)
{
int32 *wids;
int32 startwid;
int32 i, ch, nccs, noovs, unk;
if (n == 0)
return 0;
unk = ngram_unknown_wid(lm);
/* Reverse this array into an array of word IDs. */
wids = ckd_calloc(n, sizeof(*wids));
for (i = 0; i < n; ++i)
wids[n-i-1] = ngram_wid(lm, words[i]);
/* Skip <s> as it's a context cue (HACK, this should be configurable). */
startwid = ngram_wid(lm, "<s>");
/* Now evaluate the list of words in reverse using the
* remainder of the array as the history. */
ch = noovs = nccs = 0;
for (i = 0; i < n; ++i) {
int32 n_used;
int32 prob;
/* Skip <s> as it's a context cue (HACK, this should be configurable). */
if (wids[i] == startwid) {
++nccs;
continue;
}
/* Skip and count OOVs. */
if (wids[i] == NGRAM_INVALID_WID || wids[i] == unk) {
++noovs;
continue;
}
/* Sum up information for each N-gram */
prob = ngram_ng_score(lm,
wids[i], wids + i + 1,
n - i - 1, &n_used);
if (verbose) {
int m;
printf("log P(%s|", ngram_word(lm, wids[i]));
m = i + ngram_model_get_size(lm) - 1;
if (m >= n)
m = n - 1;
while (m > i) {
printf("%s ", ngram_word(lm, wids[m--]));
}
printf(") = %d\n", prob);
}
ch -= prob;
}
if (out_n_ccs) *out_n_ccs = nccs;
if (out_n_oovs) *out_n_oovs = noovs;
/* Calculate cross-entropy CH = - 1/N sum log P(W|H) */
n -= (nccs + noovs);
if (n <= 0)
return 0;
if (out_lm_score)
*out_lm_score = -ch;
return ch / n;
}
static void
evaluate_file(ngram_model_t *lm, logmath_t *lmath, const char *lsnfn)
{
FILE *fh;
lineiter_t *litor;
int32 nccs, noovs, nwords, lscr;
float64 ch, log_to_log2;;
if ((fh = fopen(lsnfn, "r")) == NULL)
E_FATAL_SYSTEM("failed to open transcript file %s", lsnfn);
/* We have to keep ch in floating-point to avoid overflows, so
* we might as well use log2. */
log_to_log2 = log(logmath_get_base(lmath)) / log(2);
lscr = nccs = noovs = nwords = 0;
ch = 0.0;
for (litor = lineiter_start(fh); litor; litor = lineiter_next(litor)) {
char **words;
int32 n, tmp_ch, tmp_noovs, tmp_nccs, tmp_lscr;
n = str2words(litor->buf, NULL, 0);
if (n < 0)
E_FATAL("str2words(line, NULL, 0) = %d, should not happen\n", n);
if (n == 0) /* Do nothing! */
continue;
words = ckd_calloc(n, sizeof(*words));
str2words(litor->buf, words, n);
/* Remove any utterance ID (FIXME: has to be a single "word") */
if (words[n-1][0] == '('
&& words[n-1][strlen(words[n-1])-1] == ')')
n = n - 1;
tmp_ch = calc_entropy(lm, words, n, &tmp_nccs,
&tmp_noovs, &tmp_lscr);
ch += (float64) tmp_ch * (n - tmp_nccs - tmp_noovs) * log_to_log2;
nccs += tmp_nccs;
noovs += tmp_noovs;
lscr += tmp_lscr;
nwords += n;
ckd_free(words);
}
ch /= (nwords - nccs - noovs);
printf("cross-entropy: %f bits\n", ch);
/* Calculate perplexity pplx = exp CH */
printf("perplexity: %f\n", pow(2.0, ch));
printf("lm score: %d\n", lscr);
/* Report OOVs and CCs */
printf("%d words evaluated\n", nwords);
printf("%d OOVs (%.2f%%), %d context cues removed\n",
noovs, (double)noovs / nwords * 100, nccs);
}
static void
evaluate_string(ngram_model_t *lm, logmath_t *lmath, const char *text)
{
char *textfoo;
char **words;
int32 n, ch, noovs, nccs, lscr;
/* Split it into an array of strings. */
textfoo = ckd_salloc(text);
n = str2words(textfoo, NULL, 0);
if (n < 0)
E_FATAL("str2words(textfoo, NULL, 0) = %d, should not happen\n", n);
if (n == 0) /* Do nothing! */
return;
words = ckd_calloc(n, sizeof(*words));
str2words(textfoo, words, n);
ch = calc_entropy(lm, words, n, &nccs, &noovs, &lscr);
printf("input: %s\n", text);
printf("cross-entropy: %f bits\n",
ch * log(logmath_get_base(lmath)) / log(2));
/* Calculate perplexity pplx = exp CH */
printf("perplexity: %f\n", logmath_exp(lmath, ch));
printf("lm score: %d\n", lscr);
/* Report OOVs and CCs */
printf("%d words evaluated\n", n);
printf("%d OOVs, %d context cues removed\n",
noovs, nccs);
ckd_free(textfoo);
ckd_free(words);
}
int
main(int argc, char *argv[])
{
cmd_ln_t *config;
ngram_model_t *lm = NULL;
logmath_t *lmath;
const char *lmfn, *probdefn, *lsnfn, *text;
if ((config = cmd_ln_parse_r(NULL, defn, argc, argv, TRUE)) == NULL)
return 1;
verbose = cmd_ln_boolean_r(config, "-verbose");
/* Create log math object. */
if ((lmath = logmath_init
(cmd_ln_float64_r(config, "-logbase"), 0, 0)) == NULL) {
E_FATAL("Failed to initialize log math\n");
}
/* Load the language model. */
lmfn = cmd_ln_str_r(config, "-lm");
if (lmfn == NULL
|| (lm = ngram_model_read(config, lmfn,
NGRAM_AUTO, lmath)) == NULL) {
E_FATAL("Failed to load language model from %s\n",
cmd_ln_str_r(config, "-lm"));
}
if ((probdefn = cmd_ln_str_r(config, "-probdef")) != NULL)
ngram_model_read_classdef(lm, probdefn);
ngram_model_apply_weights(lm,
cmd_ln_float32_r(config, "-lw"),
cmd_ln_float32_r(config, "-wip"));
/* Now evaluate some text. */
lsnfn = cmd_ln_str_r(config, "-lsn");
text = cmd_ln_str_r(config, "-text");
if (lsnfn) {
evaluate_file(lm, lmath, lsnfn);
}
else if (text) {
evaluate_string(lm, lmath, text);
}
return 0;
}
ckd_alloc.h
Sphinx's memory allocation/deallocation routines.
ckd_free
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
Definition ckd_alloc.c:244
ckd_calloc
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
Definition ckd_alloc.h:248
ckd_salloc
#define ckd_salloc(ptr)
Macro for ckd_salloc.
Definition ckd_alloc.h:264
cmd_ln.h
Command-line and other configurationparsing and handling.
cmd_ln_boolean_r
#define cmd_ln_boolean_r(c, n)
Retrieve a boolean value from a command-line object.
Definition cmd_ln.h:334
ARG_STRING
#define ARG_STRING
String argument (optional).
Definition cmd_ln.h:114
cmd_ln_str_r
SPHINXBASE_EXPORT char const * cmd_ln_str_r(cmd_ln_t *cmdln, char const *name)
Retrieve a string from a command-line object.
Definition cmd_ln.c:949
ARG_BOOLEAN
#define ARG_BOOLEAN
Boolean (true/false) argument (optional).
Definition cmd_ln.h:118
ARG_FLOAT64
#define ARG_FLOAT64
Definition cmd_ln.h:152
ARG_FLOAT32
#define ARG_FLOAT32
Definition cmd_ln.h:148
cmd_ln_parse_r
SPHINXBASE_EXPORT cmd_ln_t * cmd_ln_parse_r(cmd_ln_t *inout_cmdln, arg_t const *defn, int32 argc, char *argv[], int32 strict)
Parse a list of strings into argumetns.
Definition cmd_ln.c:556
err.h
Implementation of logging routines.
E_FATAL
#define E_FATAL(...)
Exit with non-zero status after error message.
Definition err.h:81
E_FATAL_SYSTEM
#define E_FATAL_SYSTEM(...)
Print error text; Call perror(""); exit(errno);.
Definition err.h:90
logmath.h
Fast integer logarithmic addition operations.
logmath_get_base
SPHINXBASE_EXPORT float64 logmath_get_base(logmath_t *lmath)
Get the log base.
Definition logmath.c:368
logmath_init
SPHINXBASE_EXPORT logmath_t * logmath_init(float64 base, int shift, int use_table)
Initialize a log math computation table.
Definition logmath.c:62
logmath_t
struct logmath_s logmath_t
Integer log math computation class.
Definition logmath.h:108
logmath_exp
SPHINXBASE_EXPORT float64 logmath_exp(logmath_t *lmath, int logb_p)
Convert integer log in base B to linear floating point.
Definition logmath.c:456
ngram_model.h
N-Gram language models.
ngram_unknown_wid
SPHINXBASE_EXPORT int32 ngram_unknown_wid(ngram_model_t *model)
Get the unknown word ID for a language model.
Definition ngram_model.c:550
ngram_word
SPHINXBASE_EXPORT const char * ngram_word(ngram_model_t *model, int32 wid)
Look up word string for numerical word ID.
Definition ngram_model.c:596
NGRAM_INVALID_WID
#define NGRAM_INVALID_WID
Impossible word ID.
Definition ngram_model.h:83
NGRAM_AUTO
@ NGRAM_AUTO
Determine file type automatically.
Definition ngram_model.h:78
ngram_model_get_size
SPHINXBASE_EXPORT int32 ngram_model_get_size(ngram_model_t *model)
Get the order of the N-gram model (i.e.
Definition ngram_model.c:569
ngram_model_apply_weights
SPHINXBASE_EXPORT int ngram_model_apply_weights(ngram_model_t *model, float32 lw, float32 wip)
Apply a language weight, insertion penalty, and unigram weight to a language model.
Definition ngram_model.c:360
ngram_ng_score
SPHINXBASE_EXPORT int32 ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist, int32 *n_used)
Quick general N-Gram score lookup.
Definition ngram_model.c:375
ngram_model_read
SPHINXBASE_EXPORT ngram_model_t * ngram_model_read(cmd_ln_t *config, const char *file_name, ngram_file_type_t file_type, logmath_t *lmath)
Read an N-Gram model from a file on disk.
Definition ngram_model.c:124
ngram_model_read_classdef
SPHINXBASE_EXPORT int32 ngram_model_read_classdef(ngram_model_t *model, const char *file_name)
Read a class definition file and add classes to a language model.
Definition ngram_model.c:1027
ngram_wid
SPHINXBASE_EXPORT int32 ngram_wid(ngram_model_t *model, const char *word)
Look up numerical word ID.
Definition ngram_model.c:585
ngram_model_t
struct ngram_model_s ngram_model_t
Abstract type representing an N-Gram based language model.
Definition ngram_model.h:66
pio.h
file IO related operations.
lineiter_start
SPHINXBASE_EXPORT lineiter_t * lineiter_start(FILE *fh)
Start reading lines from a file.
Definition pio.c:264
lineiter_next
SPHINXBASE_EXPORT lineiter_t * lineiter_next(lineiter_t *li)
Move to the next line in the file.
Definition pio.c:347
strfuncs.h
Miscellaneous useful string functions.
str2words
SPHINXBASE_EXPORT int32 str2words(char *line, char **wptr, int32 n_wptr)
Convert a line to an array of "words", based on whitespace separators.
Definition strfuncs.c:123
arg_t
Argument definition structure.
cmd_ln_t
Opaque structure used to hold the results of command-line parsing.
lineiter_t
Line iterator for files.
Definition pio.h:177