Loading...
Searching...
No Matches
ngrammar_utils.cc
/*************************************************************************/
/* */
/* Centre for Speech Technology Research */
/* University of Edinburgh, UK */
/* Copyright (c) 1999 */
/* All Rights Reserved. */
/* */
/* Permission is hereby granted, free of charge, to use and distribute */
/* this software and its documentation without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of this work, and to */
/* permit persons to whom this work is furnished to do so, subject to */
/* the following conditions: */
/* 1. The code must retain the above copyright notice, this list of */
/* conditions and the following disclaimer. */
/* 2. Any modifications must be clearly marked as such. */
/* 3. Original authors' names are not deleted. */
/* 4. The authors' names are not used to endorse or promote products */
/* derived from this software without specific prior written */
/* permission. */
/* */
/* THE UNIVERSITY OF EDINBURGH AND THE CONTRIBUTORS TO THIS WORK */
/* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */
/* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */
/* SHALL THE UNIVERSITY OF EDINBURGH NOR THE CONTRIBUTORS BE LIABLE */
/* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */
/* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */
/* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */
/* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */
/* THIS SOFTWARE. */
/* */
/*************************************************************************/
/* Author : Alan W Black */
/* Date : February 1999 */
/*-----------------------------------------------------------------------*/
/* */
/* A rationalization of some of the general functions */
/* */
/*=======================================================================*/
#include <iostream>
#include <cstring>
#include "EST_String.h"
#include "EST_Token.h"
#include "EST_error.h"
#include "EST_Ngrammar.h"
static int get_next_window(EST_TokenStream &ts,
EST_StrVector &window,
const EST_String &input_format,
EST_Ngrammar &ngram)
{
int i;
if ((input_format == "sentence_per_line") ||
(input_format == "sentence_per_file"))
{
EST_String t = ts.get().string();
slide(window,-1);
window[ngram.order()-1] = t;
if (ngram.wordlist_index(t) == -1)
cerr << "EST_Ngrammar test: skipping bad word \"" <<
t << "\"" << endl;
}
else if (input_format == "ngram_per_line")
{
for (i=0; i < ngram.order(); i++)
{
EST_String t = ts.get().string();
window[i] = t;
if (ngram.wordlist_index(t) == -1)
cerr << "EST_Ngrammar test: skipping bad word \"" <<
t << "\"" << endl;
}
}
else
EST_error("EST_Ngrammar test: unknown input format \"%s\"\n",
(const char *)input_format);
// Sigh, you pull a little thread and it all falls down
// For the time being can only deal in StrVectors rather than
// IVectors
for (i=0; i < ngram.order(); i++)
if (ngram.wordlist_index(window(i)) == -1)
return FALSE;
return TRUE;
}
bool test_stats(EST_Ngrammar &ngram,
const EST_String &filename,
double &raw_entropy,
double &count,
double &entropy,
double &perplexity,
const EST_String &input_format,
const EST_String &prev,
const EST_String &prev_prev,
const EST_String &last)
{
// Apply an ngram to some data and report on its performance
// Output entropy and test set perplexity
// H = -1/Q . log P(wi | wi-1, wi-2, ... wi-n)
// H_p = 2^H
// Rabiner and Juang p450
EST_TokenStream ts;
double H,prob;
int Q;
EST_StrVector window(ngram.order());
(void)last;
if (filename == "-")
ts.open(stdin,FALSE);
else if (ts.open(filename) == -1)
EST_error("EST_Ngrammar test: unable to open test file \"%s\"\n",
(const char *)filename);
Q=0;
H=0.0;
ngram.fill_window_start(window,prev,prev_prev);
while (!ts.eof() &&
(get_next_window(ts,window,input_format,ngram) == TRUE))
{
prob = ngram.probability(window);
H += log(prob);
Q++;
if ((input_format == "sentence_per_line") && (ts.eoln()))
ngram.fill_window_start(window,prev,prev_prev);
}
count = Q;
raw_entropy = -1 * H;
entropy = -1 * (H/Q);
perplexity = pow(2.0,entropy);
// printf("count %g entropy %g perplexity %g\n",
// count,entropy,perplexity);
return true;
}
EST_Ngrammar
Definition EST_Ngrammar.h:209
EST_String
Definition EST_String.h:70
EST_TokenStream
Definition EST_Token.h:235
EST_TokenStream::eof
int eof()
end of file
Definition EST_Token.h:356
EST_TokenStream::eoln
int eoln()
end of line
Definition EST_Token.cc:818
EST_TokenStream::open
int open(const EST_String &filename)
open a \Ref{EST_TokenStream} for a file.
Definition EST_Token.cc:200
EST_TokenStream::get
EST_TokenStream & get(EST_Token &t)
get next token in stream
Definition EST_Token.cc:486