Loading...
Searching...
No Matches
relation_io.cc
/*************************************************************************/
/* */
/* Centre for Speech Technology Research */
/* University of Edinburgh, UK */
/* Copyright (c) 1995,1996 */
/* All Rights Reserved. */
/* */
/* Permission is hereby granted, free of charge, to use and distribute */
/* this software and its documentation without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of this work, and to */
/* permit persons to whom this work is furnished to do so, subject to */
/* the following conditions: */
/* 1. The code must retain the above copyright notice, this list of */
/* conditions and the following disclaimer. */
/* 2. Any modifications must be clearly marked as such. */
/* 3. Original authors' names are not deleted. */
/* 4. The authors' names are not used to endorse or promote products */
/* derived from this software without specific prior written */
/* permission. */
/* */
/* THE UNIVERSITY OF EDINBURGH AND THE CONTRIBUTORS TO THIS WORK */
/* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */
/* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */
/* SHALL THE UNIVERSITY OF EDINBURGH NOR THE CONTRIBUTORS BE LIABLE */
/* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */
/* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */
/* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */
/* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */
/* THIS SOFTWARE. */
/* */
/*************************************************************************/
/* Author : Paul Taylor updated by awb */
/* Date : Feb 1999 */
/*-----------------------------------------------------------------------*/
/* Relation class file i/o, label files */
/* */
/*=======================================================================*/
#include <cstdlib>
#include <cstdio>
#include <fstream>
#include "EST_unix.h"
#include "EST_types.h"
#include "ling_class/EST_Relation.h"
#include "EST_string_aux.h"
#include "EST_cutils.h"
#include "EST_TList.h"
#include "EST_Option.h"
#include "relation_io.h"
#define DEF_SAMPLE_RATE 16000
#define HTK_UNITS_PER_SECOND 10000000
static EST_Regex RXleadingwhitespace("^[ \t\n\r][ \t\n\r]*.*$");
EST_read_status read_label_portion(EST_TokenStream &ts, EST_Relation &s,
int sample);
EST_read_status load_esps_label(EST_TokenStream &ts,EST_Relation &rel)
{
ts.set_SingleCharSymbols(";");
ts.set_quotes('"','\\');
EST_String key, val;
// Skip the header
while (!ts.eof())
{
key = ts.get().string();
if (key == "#")
break;
val = ts.get_upto_eoln().string();
// delete leading whitespace
if (val.matches(RXleadingwhitespace))
val = val.after(RXwhite);
rel.f.set(key, val);
}
if (ts.peek() == "") return format_ok;
while (!ts.eof())
{
EST_Item *si = rel.append();
EST_String name;
si->set("end",(float)atof(ts.get().string()));
ts.get(); // skip the color;
for (name = ""; (!ts.eoln()) && (ts.peek() != ";"); )
{
EST_Token &t = ts.get();
if (name.length() > 0) // preserve internal whitespace
name += t.whitespace();
name += t.string();
}
si->set_name(name);
if (ts.peek().string() == ";") // absorb separator
{
ts.get();
si->features().load(ts);
}
}
return format_ok;
}
EST_write_status save_esps_label(const EST_String &filename,
const EST_Relation &s,
bool evaluate_ff)
{
ostream *outf;
if (filename == "-")
outf = &cout;
else
outf = new ofstream(filename);
if (!(*outf))
{
cerr << "save_esps_label: can't open label output file \"" <<
filename << "\"" << endl;
return write_fail;
}
EST_write_status st=save_esps_label(outf, s, evaluate_ff);
if (outf != &cout)
delete outf;
return st;
}
EST_write_status save_esps_label(ostream *outf,
const EST_Relation &s,
bool evaluate_ff)
{
EST_Item *ptr;
*outf << "separator ;\n";
if (!s.f.present("nfields"))
*outf << "nfields 1\n";
EST_Features::Entries p;
for (p.begin(s.f); p; ++p)
*outf << p->k << " " << p->v << endl;
*outf << "#\n";
/* if (f("timing_style") == "event")
*outf << "timing_style event\n";
else if (f("timing_style") == "unit")
*outf << "timing_style unit\n";
*/
for (ptr = s.head(); ptr != 0; ptr = inext(ptr))
{
*outf << "\t";
outf->precision(5);
outf->setf(ios::scientific, ios::floatfield);
outf->width(8);
// outf->fill('0');
if (s.f("timing_style","0") == "event")
*outf << ptr->F("time",0);
else
*outf << ptr->F("end",0);
*outf << " 26 \t" << ptr->S("name","0");
EST_Features f2;
f2 = ptr->features();
f2.remove("name");
f2.remove("end");
if (evaluate_ff)
evaluate(ptr,f2);
if (f2.length() > 0)
{
*outf << " ; ";
f2.save(*outf);
}
*outf << endl;
}
return write_ok;
}
EST_read_status load_ogi_label(EST_TokenStream &ts, EST_Relation &s)
{
// This function reads OGI style label files. The start, end
// time and names of the labels are mandatory.
EST_String key, val;
float sr;
int isr;
// set up the character constant values for this stream
ts.set_SingleCharSymbols(";");
// Skip over header
while(!ts.eof())
{
if ((ts.peek().col() == 0) && (ts.peek() == "END"))
{
if (ts.peek() == "END")
{ // read rest of header
ts.get();
ts.get();
ts.get();
}
break;
}
key = ts.get().string();
val = ts.get().string();
}
sr = 1000.0 / atof(val);
isr = (int)sr;
if (ts.eof())
{
cerr << "Error: couldn't find header in label file "
<< ts.filename() << endl;
return wrong_format;
}
if (read_label_portion(ts, s, isr) == misc_read_error)
{
cerr << "error: in label file " << ts.filename() << " at line " <<
ts.linenum() << endl;
return misc_read_error;
}
return format_ok;
}
EST_read_status load_words_label(EST_TokenStream &ts, EST_Relation &s)
{
// This function reads label files in the form of simple word strings
// with no timing information.
EST_Item *item;
while (!ts.eof())
{
item = s.append();
item->set("name",(EST_String)ts.get());
item->set("end",0.0);
}
return format_ok;
}
static float convert_long_num_string_to_time(const char *s,int sample)
{
// For those label files that think 100 nanosecond times are cool
// we have to provide a special function to convert them as
// this quickly gets beyond the capabilities of ints.
if (strlen(s) < 15)
return atof(s)/sample;
else
{
double a = 0,d;
int i=0;
for (i=0;
(strchr(" \n\r\t",s[i]) != NULL) && (s[i] != '\0');
i++);
for ( ;
(s[i] != '\0') && (s[i] >= '0') && (s[i] <= '9');
i++)
{
a = a*10;
d = s[i]-'0';
a += (d/(double)sample);
}
return a;
}
}
EST_read_status read_label_portion(EST_TokenStream &ts, EST_Relation &s,
int sample)
{
EST_Item *item;
float hend;
EST_String str;
while(!ts.eof())
{
str = ts.get().string();
if (str == ".")
return format_ok;
item = s.append();
str = ts.get().string();
hend = convert_long_num_string_to_time(str,sample);
item->set("end",hend); // time
item->set("name",ts.get().string()); // name
if (!ts.eoln())
item->set("rest_lab",ts.get_upto_eoln().string());
}
return format_ok;
}
EST_read_status load_sample_label(EST_TokenStream &ts,
EST_Relation &s, int sample)
{
if (sample == 0) // maybe this should be an error
sample = DEF_SAMPLE_RATE;
// set up the character constant values for this stream
ts.set_SingleCharSymbols(";");
s.clear();
if (read_label_portion(ts, s, sample) == misc_read_error)
{
cerr << "error: in label file " << ts.filename() << " at line " <<
ts.linenum() << endl;
return misc_read_error;
}
return format_ok;
}
EST_write_status save_htk_label(const EST_String &filename,
const EST_Relation &a)
{
ostream *outf;
if (filename == "-")
outf = &cout;
else
outf = new ofstream(filename);
if (!(*outf))
{
cerr << "save_htk_label: can't open label output file \"" <<
filename << "\"" << endl;
return write_fail;
}
EST_write_status s = save_htk_label(outf, a);
if (outf != &cout)
delete outf;
return s;
}
EST_write_status save_htk_label(ostream *outf,
const EST_Relation &a)
{
EST_Item *ptr;
float end,start;
outf->precision(6);
start = end = 0;
for (ptr = a.head(); ptr != 0; ptr = inext(ptr))
{
outf->width(15);
cout.setf(ios::left,ios::adjustfield);
*outf << (int)(start * HTK_UNITS_PER_SECOND);
outf->width(15);
end = ptr->F("end",0.0);
*outf << (int)(end * HTK_UNITS_PER_SECOND);
*outf << " " << ptr->name() << endl;
start = end;
}
return write_ok;
}
#if 0
EST_write_status save_label_spn(const EST_String &filename,
const EST_Relation &a)
{
EST_Stream_Item *ptr;
ostream *outf;
if (filename == "-")
outf = &cout;
else
outf = new ofstream(filename);
if (!(*outf))
{
cerr << "save_label_spn: can't open label output file \""
<< filename << "\"" << endl;
return write_fail;
}
ptr = a.head();
outf->precision(3);
outf->setf(ios::left, ios::adjustfield);
outf->width(8);
*outf << ptr->name();
outf->setf(ios::scientific, ios::floatfield);
outf->width(8);
*outf << (ptr->dur() * 1000.0) << "\t (0,140)" << endl;
for (; inext(ptr) != 0; ptr = inext(ptr))
{
outf->precision(3);
outf->setf(ios::left, ios::adjustfield);
outf->width(8);
*outf << ptr->name();
outf->setf(ios::scientific, ios::floatfield);
outf->width(8);
*outf << (ptr->dur() * 1000.0) << endl;
}
// outf->precision(3);
// outf->setf(ios::left, ios::adjustfield);
outf->width(8);
*outf << ptr->name();
outf->setf(ios::scientific, ios::floatfield);
outf->width(8);
*outf << (ptr->dur() * 1000.0) << "\t (99,80)" << endl;
if (outf != &cout)
delete outf;
return write_ok;
}
EST_write_status save_label_names(const EST_String &filename,
const EST_Relation &a,
const EST_String &features)
{
EST_Stream_Item *ptr;
ostream *outf;
if (filename == "-")
outf = &cout;
else
outf = new ofstream(filename);
if (!(*outf))
{
cerr << "save_label_name: can't open label output file \""
<< filename << "\"" << endl;
return misc_write_error;
}
for (ptr = a.head(); inext(ptr) != 0; ptr = inext(ptr))
{
*outf << ptr->name();
if ((features != "") && (features != "OneLine"))
*outf << endl;
else
*outf << " ";
}
*outf << ptr->name() << endl;
if (outf != &cout)
delete outf;
return write_ok;
}
#endif
EST_write_status save_RelationList(const EST_String &filename,
const EST_RelationList &plist,
int time, int path)
{
EST_Litem *p;
EST_Item *ptr;
EST_String outname;
float start,end;
ostream *outf;
if (filename == "-")
outf = &cout;
else
outf = new ofstream(filename);
if (!(*outf))
{
cerr << "save_StreamList: can't open MLF output file \""
<< filename << "\"\n";
return write_fail;
}
*outf << "#!MLF!#\n"; // MLF header/identifier
outf->precision(6);
start = end = 0;
for (p = plist.head(); p != 0; p = p->next())
{
outname = path ? plist(p).name() : basename(plist(p).name());
*outf << "\"*/" << outname<<"\"\n";
for (ptr = plist(p).head(); ptr != 0; ptr = inext(ptr))
{
if (time)
{
outf->width(15);
cout.setf(ios::left,ios::adjustfield);
*outf << (int)(start * HTK_UNITS_PER_SECOND);
outf->width(15);
end = ptr->F("end",0.0);
*outf << (int)(end * HTK_UNITS_PER_SECOND) << " ";
start = end;
}
*outf << ptr->S("name","0") << endl;
}
*outf << ".\n";
}
if (outf != &cout)
delete outf;
return write_ok;
}
EST_write_status save_WordList(const EST_String &filename,
const EST_RelationList &plist,
int style)
{
EST_Litem *p;
EST_Item *ptr;
ostream *outf;
if (filename == "-")
outf = &cout;
else
outf = new ofstream(filename);
if (!(*outf))
{
cerr << "save:WordList: can't open WordList output file \""
<< filename << "\"\n";
return write_fail;
}
for (p = plist.head(); p != 0; p = p->next())
{
for (ptr = plist(p).head(); inext(ptr) != 0; ptr = inext(ptr))
{
*outf << ptr->name();
if (style == 0)
*outf << endl;
else
*outf << " ";
}
if (ptr != 0)
*outf << ptr->name() << endl;
}
if (outf != &cout)
delete outf;
return write_ok;
}
EST_write_status save_ind_RelationList(const EST_String &filename,
const EST_RelationList &plist,
const EST_String &features,
int path)
{
EST_Litem *p;
EST_String outname;
(void) filename;
(void) features;
for (p = plist.head(); p != 0; p = p->next())
{
outname = path ? plist(p).name() : basename(plist(p).name());
if (plist(p).save(outname,false) != write_ok)
return misc_write_error;
}
return write_ok;
}
EST_read_status load_RelationList(const EST_String &filename,
EST_RelationList &plist)
{
EST_TokenStream ts;
EST_String fns, name;
if (((filename == "-") ? ts.open(cin) : ts.open(filename)) != 0)
{
cerr << "Can't open label input file " << filename << endl;
return misc_read_error;
}
// set up the character constant values for this stream
ts.set_SingleCharSymbols(";");
// Skip over header
if (ts.get().string() != "#!MLF!#")
{
cerr << "Not MLF file\n";
return wrong_format;
}
while(!ts.eof())
{
// put filename in as stream name. The filename is usually surrounded
// by quotes, so remove these.
fns = ts.get().string();
strip_quotes(fns);
EST_Relation s(fns);
s.f.set("name", fns); // simonk
plist.append(s);
if (read_label_portion(ts, plist.last(), 10000000) == misc_read_error)
{
cerr << "error: in reading MLF file\n";
cerr << "section for file " << fns <<
" at line " << ts.linenum() << " is badly formatted\n";
return misc_read_error;
}
}
return format_ok;
}
static void pad_ends(EST_Relation &s, float length)
{
// add evenly spaced dummy end values to Relation
EST_Item *p;
int i;
for (i = 0, p = s.head(); p; p = inext(p), ++i)
p->set("end",(length * float(i)/float(s.length())));
}
EST_read_status read_RelationList(EST_RelationList &plist,
EST_StrList &files, EST_Option &al)
{
EST_Litem *p, *plp;
if (al.val("-itype", 0) == "mlf")
{
if (load_RelationList(files.first(), plist) != format_ok)
exit (-1);
}
else
for (p = files.head(); p; p = p->next())
{
EST_Relation s(files(p));
plist.append(s);
plp = plist.tail();
if (al.present("-itype"))
{
if (plist(plp).load(files(p), al.val("-itype")) != format_ok)
exit (-1);
}
else if (plist(plp).load(files(p)) != format_ok)
exit (-1);
if ((al.val("-itype", 0) == "words") && (al.present("-length")))
pad_ends(s, al.fval("-length"));
}
return format_ok;
}
EST_Features
Definition EST_Features.h:62
EST_Features::set
void set(const EST_String &name, int ival)
Definition EST_Features.h:185
EST_Features::remove
void remove(const EST_String &name)
Definition EST_Features.h:246
EST_Features::load
EST_read_status load(EST_TokenStream &ts)
load features from already opened EST_TokenStream
Definition EST_features_io.cc:248
EST_Features::save
EST_write_status save(ostream &outf) const
save features in already opened ostream
Definition EST_features_io.cc:111
EST_Features::present
int present(const EST_String &name) const
Definition EST_Features.cc:147
EST_Features::length
int length() const
Definition EST_Features.h:250
EST_Item
Definition EST_Item.h:83
EST_Item::set
void set(const EST_String &name, int ival)
Definition EST_Item.h:179
EST_Item::S
const EST_String S(const EST_String &name) const
Definition EST_Item.h:143
EST_Item::F
const float F(const EST_String &name) const
Definition EST_Item.h:134
EST_Option
Definition EST_Option.h:50
EST_Option::fval
float fval(const EST_String &rkey, int m=1) const
Definition EST_Option.cc:98
EST_Regex
Definition EST_Regex.h:55
EST_Relation
Definition EST_Relation.h:68
EST_Relation::clear
void clear()
Definition EST_Relation.cc:153
EST_Relation::f
EST_Features f
Definition EST_Relation.h:103
EST_Relation::head
EST_Item * head() const
Definition EST_Relation.h:125
EST_Relation::length
int length() const
Definition EST_Relation.cc:137
EST_String
Definition EST_String.h:70
EST_String::length
int length(void) const
Length of string ({not} length of underlying chunk).
Definition EST_String.h:241
EST_String::after
EST_String after(int pos, int len=1) const
Part after pos+len.
Definition EST_String.h:318
EST_TIterator::begin
void begin(const Container &over)
Set the iterator ready to run over this container.
Definition EST_TIterator.h:115
EST_TKVL::val
const V & val(const K &rkey, bool m=0) const
return value according to key (const)
Definition EST_TKVL.cc:145
EST_TKVL::present
const int present(const K &rkey) const
Returns true if key is present.
Definition EST_TKVL.cc:222
EST_TList::last
const T & last() const
return const reference to last item in list
Definition EST_TList.h:149
EST_TList::append
void append(const T &item)
add item onto end of list
Definition EST_TList.h:191
EST_TList::first
const T & first() const
return const reference to first item in list
Definition EST_TList.h:146
EST_TokenStream
Definition EST_Token.h:235
EST_TokenStream::eof
int eof()
end of file
Definition EST_Token.h:356
EST_TokenStream::set_SingleCharSymbols
void set_SingleCharSymbols(const EST_String &sc)
set which characters are to be treated as single character symbols
Definition EST_Token.h:338
EST_TokenStream::filename
const EST_String filename() const
The originating filename (if there is one).
Definition EST_Token.h:372
EST_TokenStream::linenum
int linenum(void) const
returns line number of \Ref{EST_TokenStream}
Definition EST_Token.h:354
EST_TokenStream::set_quotes
void set_quotes(char q, char e)
set characters to be used as quotes and escape, and set quote mode
Definition EST_Token.h:347
EST_TokenStream::get_upto_eoln
EST_Token get_upto_eoln(void)
get up to {\tt s} in end of line as a single token.
Definition EST_Token.cc:516
EST_TokenStream::eoln
int eoln()
end of line
Definition EST_Token.cc:818
EST_TokenStream::peek
EST_Token & peek(void)
peek at next token
Definition EST_Token.cc:830
EST_TokenStream::open
int open(const EST_String &filename)
open a \Ref{EST_TokenStream} for a file.
Definition EST_Token.cc:200
EST_TokenStream::get
EST_TokenStream & get(EST_Token &t)
get next token in stream
Definition EST_Token.cc:486
EST_Token
Definition EST_Token.h:73
EST_Token::col
int col(void) const
Line position in original \Ref{EST_TokenStream}.
Definition EST_Token.h:184