forwardTokenizer.cpp
Go to the documentation of this file.
/******************************************************
* Presage, an extensible predictive text entry system
* ---------------------------------------------------
*
* Copyright (C) 2008 Matteo Vescovi <matteo.vescovi@yahoo.co.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
**********(*)*/
#include "forwardTokenizer.h"
ForwardTokenizer::ForwardTokenizer(std::istream& stream,
const std::string blankspaces,
const std::string separators)
: Tokenizer(stream, blankspaces, separators)
{
//std::cerr << "ForwardTokenizer::ForwardTokenizer()" << std::endl;
offset = offbeg;
}
ForwardTokenizer::~ForwardTokenizer()
{}
int ForwardTokenizer::countTokens()
{
StreamGuard guard(stream, offset);
// store current seek pointer position
std::streamoff curroff = offset;
// position get pointer at beginning of stream
offset = offbeg;
int count = 0;
while (hasMoreTokens()) {
count++;
nextToken();
}
// reposition seek get pointer to original position
offset = curroff;
return count;
}
bool ForwardTokenizer::hasMoreTokens() const
{
//StreamGuard guard(stream, offset);
if (offset >= offend) {
return false;
} else {
return true;
}
}
std::string ForwardTokenizer::nextToken()
{
StreamGuard guard(stream, offset);
int current;
std::string str;
if (stream.good()) { // good() if bad,fail and eof bit are not set
current = stream.peek();
if (offset < offend) {
while (isBlankspace(current)
|| isSeparator(current)) {
offset++;
stream.seekg(offset);
current = stream.peek();
}
while (!isBlankspace(current)
&& !isSeparator(current)
&& offset < offend) {
//std::cerr << "[DEBUG] read: "
// << static_cast<char>(current)
// << std::endl;
if( lowercaseMode() ) {
current = tolower( current );
}
str.push_back(current);
//std::cerr << "[DEBUG] pushed: "
// << static_cast<char>(current)
// << std::endl;
offset++;
stream.seekg(offset);
current = stream.peek();
}
}
// do {
// do {
// current = stream.peek();
// offset++;
// stream.seekg(offset);
//
// //std::cerr << "[DEBUG] read: "
// // << static_cast<char>(current)
// // << std::endl;
//
// if ( !isBlankspace(current)
// && !isSeparator(current)
// && offset <= offend) {
//
// if( lowercaseMode() ) {
// current = tolower( current );
// }
//
// str.push_back(current);
//
// //std::cerr << "[DEBUG] pushed: "
// // << static_cast<char>(current)
// // << std::endl;
// }
// } while ( !isBlankspace(current)
// && !isSeparator(current)
// && offset < offend);
// } while (str.empty() && (offset < offend));
} else {
std::cerr << "stream is NOT good!" << std::endl;
}
//std::cerr << "[DEBUG] token: " << str << std::endl;
return str;
}
double ForwardTokenizer::progress() const
{
return static_cast<double>(offset) / offend;
}
ForwardTokenizer::hasMoreTokens
virtual bool hasMoreTokens() const
Definition forwardTokenizer.cpp:61
ForwardTokenizer::countTokens
virtual int countTokens()
Definition forwardTokenizer.cpp:39
ForwardTokenizer::nextToken
virtual std::string nextToken()
Definition forwardTokenizer.cpp:72
ForwardTokenizer::~ForwardTokenizer
virtual ~ForwardTokenizer()
Definition forwardTokenizer.cpp:36
ForwardTokenizer::ForwardTokenizer
ForwardTokenizer(std::istream &stream, const std::string blankspaces, const std::string separators)
Definition forwardTokenizer.cpp:27
ForwardTokenizer::progress
virtual double progress() const
Definition forwardTokenizer.cpp:151
Tokenizer::StreamGuard
Definition tokenizer.h:125
Tokenizer::stream
std::istream & stream
Definition tokenizer.h:144
Tokenizer::offend
std::streamoff offend
Definition tokenizer.h:147
Tokenizer::offbeg
std::streamoff offbeg
Definition tokenizer.h:146
Tokenizer::isSeparator
bool isSeparator(const int character) const
Definition tokenizer.cpp:101
Tokenizer::blankspaces
std::string blankspaces
Definition tokenizer.h:154
Tokenizer::offset
std::streamoff offset
Definition tokenizer.h:148
Tokenizer::lowercaseMode
bool lowercaseMode() const
Definition tokenizer.cpp:86
Tokenizer::separators
std::string separators
Definition tokenizer.h:155
Tokenizer::isBlankspace
bool isBlankspace(const int character) const
Definition tokenizer.cpp:91
Tokenizer::Tokenizer
Tokenizer(std::istream &stream, const std::string blankspaces, const std::string separators)
Definition tokenizer.cpp:27
forwardTokenizer.h