tokenizer.h
Go to the documentation of this file.
/******************************************************
* Presage, an extensible predictive text entry system
* ---------------------------------------------------
*
* Copyright (C) 2008 Matteo Vescovi <matteo.vescovi@yahoo.co.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
**********(*)*/
#ifndef PRESAGE_TOKENIZER
#define PRESAGE_TOKENIZER
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <iostream>
#include <istream>
#include <string>
#include <assert.h>
class Tokenizer {
public:
Tokenizer(std::istream& stream,
const std::string blankspaces,
const std::string separators );
virtual ~Tokenizer();
virtual int countTokens() = 0;
virtual bool hasMoreTokens() const = 0;
virtual std::string nextToken() = 0;
virtual double progress() const = 0;
void blankspaceChars(const std::string);
std::string blankspaceChars() const;
void separatorChars(const std::string);
std::string separatorChars() const;
void lowercaseMode(const bool);
bool lowercaseMode() const;
std::string streamToString() const {
std::streamoff offbackup = stream.tellg();
std::string str;
std::streamoff curroff = offbeg;
stream.seekg(curroff);
while (curroff < offend) {
stream.clear();
str.push_back(stream.peek());
curroff++;
stream.seekg(curroff);
}
stream.seekg(offbackup);
return str;
}
protected:
class StreamGuard {
public:
StreamGuard(std::istream& so, std::streamoff& of)
: guardedStream(so) {
currstate = guardedStream.rdstate();
curroff = guardedStream.tellg();
guardedStream.seekg (of );
}
~StreamGuard() {
guardedStream.seekg (curroff );
guardedStream.setstate(currstate);
}
private:
std::istream& guardedStream;
std::ios::iostate currstate;
std::streamoff curroff;
};
std::istream& stream;
std::ios::iostate sstate;
std::streamoff offbeg;
std::streamoff offend;
std::streamoff offset;
bool isBlankspace(const int character) const;
bool isSeparator (const int character) const;
private:
std::string blankspaces;
std::string separators;
bool lowercase;
};
#endif // PRESAGE_TOKENIZER
Tokenizer::StreamGuard::currstate
std::ios::iostate currstate
Definition: tokenizer.h:160
Tokenizer::isBlankspace
bool isBlankspace(const int character) const
Definition: tokenizer.cpp:91
Tokenizer::countTokens
virtual int countTokens()=0
Tokenizer
Definition: tokenizer.h:64
Tokenizer::~Tokenizer
virtual ~Tokenizer()
Definition: tokenizer.cpp:53
Tokenizer::sstate
std::ios::iostate sstate
Definition: tokenizer.h:165
Tokenizer::Tokenizer
Tokenizer(std::istream &stream, const std::string blankspaces, const std::string separators)
Definition: tokenizer.cpp:27
Tokenizer::lowercaseMode
bool lowercaseMode() const
Definition: tokenizer.cpp:86
Tokenizer::StreamGuard::guardedStream
std::istream & guardedStream
Definition: tokenizer.h:159
Tokenizer::offbeg
std::streamoff offbeg
Definition: tokenizer.h:166
Tokenizer::lowercase
bool lowercase
Definition: tokenizer.h:177
Tokenizer::progress
virtual double progress() const =0
Tokenizer::isSeparator
bool isSeparator(const int character) const
Definition: tokenizer.cpp:101
Tokenizer::blankspaces
std::string blankspaces
Definition: tokenizer.h:174
Tokenizer::StreamGuard::~StreamGuard
~StreamGuard()
Definition: tokenizer.h:153
Tokenizer::StreamGuard::StreamGuard
StreamGuard(std::istream &so, std::streamoff &of)
Definition: tokenizer.h:147
Tokenizer::separators
std::string separators
Definition: tokenizer.h:175
Tokenizer::offset
std::streamoff offset
Definition: tokenizer.h:168
Tokenizer::streamToString
std::string streamToString() const
Definition: tokenizer.h:129
Tokenizer::StreamGuard
Definition: tokenizer.h:145
Tokenizer::separatorChars
std::string separatorChars() const
Definition: tokenizer.cpp:76
Tokenizer::nextToken
virtual std::string nextToken()=0
Tokenizer::stream
std::istream & stream
Definition: tokenizer.h:164
Tokenizer::hasMoreTokens
virtual bool hasMoreTokens() const =0
Tokenizer::offend
std::streamoff offend
Definition: tokenizer.h:167
Tokenizer::StreamGuard::curroff
std::streamoff curroff
Definition: tokenizer.h:161
Tokenizer::blankspaceChars
std::string blankspaceChars() const
Definition: tokenizer.cpp:66