contextTracker.cpp
Go to the documentation of this file.
/******************************************************
* Presage, an extensible predictive text entry system
* ---------------------------------------------------
*
* Copyright (C) 2008 Matteo Vescovi <matteo.vescovi@yahoo.co.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
**********(*)*/
#include "contextTracker.h"
#include "../utility.h"
#include "../predictorRegistry.h"
#include "../tokenizer/forwardTokenizer.h"
#include <stdlib.h> // for atoi()
const char* ContextTracker::LOGGER = "Presage.ContextTracker.LOGGER";
const char* ContextTracker::SLIDING_WINDOW_SIZE = "Presage.ContextTracker.SLIDING_WINDOW_SIZE";
const char* ContextTracker::LOWERCASE_MODE = "Presage.ContextTracker.LOWERCASE_MODE";
const char* ContextTracker::ONLINE_LEARNING = "Presage.ContextTracker.ONLINE_LEARNING";
ContextTracker::ContextTracker(Configuration* config,
PredictorRegistry* registry,
PresageCallback* callback,
const unsigned char wChars[],
const char tChars[],
const char bChars[],
const char cChars[])
: wordChars (reinterpret_cast<const char *>(wChars)),
separatorChars (tChars),
blankspaceChars(bChars),
controlChars (cChars),
predictorRegistry (registry),
logger ("ContextTracker", std::cerr),
//tokenizer (pastStream, blankspaceChars, separatorChars),
lowercase_mode (true),
dispatcher (this)
{
if (callback) {
context_tracker_callback = callback;
} else {
throw new PresageException(PRESAGE_INVALID_CALLBACK_ERROR, "Invalid callback object");
}
contextChangeDetector = new ContextChangeDetector(wordChars,
separatorChars,
blankspaceChars,
controlChars,
lowercase_mode);
// set pointer to this context tracker in predictor registry so that
// predictors can be constructed when next iterator is requested
//
if (predictorRegistry) {
predictorRegistry->setContextTracker(this);
}
// build dispatch map
dispatcher.map (config->find (LOGGER), & ContextTracker::set_logger);
dispatcher.map (config->find (SLIDING_WINDOW_SIZE), & ContextTracker::set_sliding_window_size);
dispatcher.map (config->find (LOWERCASE_MODE), & ContextTracker::set_lowercase_mode);
dispatcher.map (config->find (ONLINE_LEARNING), & ContextTracker::set_online_learning);
}
ContextTracker::~ContextTracker()
{
delete contextChangeDetector;
}
void ContextTracker::set_logger (const std::string& value)
{
logger << setlevel (value);
logger << INFO << "LOGGER: " << value << endl;
}
void ContextTracker::set_sliding_window_size (const std::string& value)
{
contextChangeDetector->set_sliding_window_size (value);
logger << INFO << "SLIDING_WINDOWS_SIZE: " << value << endl;
}
void ContextTracker::set_lowercase_mode (const std::string& value)
{
lowercase_mode = Utility::isYes(value);
logger << INFO << "LOWERCASE_MODE: " << value << endl;
}
void ContextTracker::set_online_learning(const std::string& value)
{
online_learning = Utility::isYes(value);
logger << INFO << "ONLINE_LEARNING: " << value << endl;
}
const PresageCallback* ContextTracker::callback(const PresageCallback* new_callback)
{
const PresageCallback* result = context_tracker_callback;
if (new_callback) {
context_tracker_callback = new_callback;
}
return result;
}
bool ContextTracker::contextChange()
{
return contextChangeDetector->context_change(getPastStream());
}
void ContextTracker::update()
{
// detect change that needs to be learned
std::string change = contextChangeDetector->change(getPastStream());
if (online_learning)
{
learn (change);
}
// update sliding window
contextChangeDetector->update_sliding_window(getPastStream());
}
void ContextTracker::learn(const std::string& text) const
{
logger << INFO << "learn(): text: " << text << endl;
std::stringstream stream_to_learn(text);
// split stream up into tokens
std::vector<std::string> tokens;
ForwardTokenizer tok(stream_to_learn,
blankspaceChars,
separatorChars);
tok.lowercaseMode(lowercase_mode);
logger << INFO << "learn(): tokenized change: ";
while (tok.hasMoreTokens()) {
std::string token = tok.nextToken();
tokens.push_back(token);
logger << INFO << token << '|';
}
logger << INFO << endl;
if (! tokens.empty()) {
// remove prefix (partially entered token or empty token)
tokens.pop_back();
}
if ((logger << INFO).shouldLog())
{
logger << "learn(): sanitized change: ";
for (std::vector<std::string>::const_iterator it = tokens.begin();
it != tokens.end();
it++) {
logger << *it << '|';
}
logger << endl;
}
// time to learn
PredictorRegistry::Iterator it = predictorRegistry->iterator();
Predictor* predictor = 0;
while (it.hasNext()) {
predictor = it.next();
predictor->learn(tokens);
}
}
std::string ContextTracker::getPrefix() const
{
return getToken(0);
}
std::string ContextTracker::getToken(const int index) const
{
std::stringstream pastStringStream(context_tracker_callback->get_past_stream());
ReverseTokenizer tokenizer(pastStringStream, blankspaceChars, separatorChars);
tokenizer.lowercaseMode(lowercase_mode);
std::string token;
int i = 0;
while (tokenizer.hasMoreTokens() && i <= index) {
token = tokenizer.nextToken();
i++;
}
if (i <= index) {
// in case the index points too far back
token = "";
}
return token;
// "a b c"
// 2 1 0
// 0 1 2
// 1 2 3
//
// ForwardTokenizer tokenizer(pastStream, blankspaceChars, separatorChars);
// tokenizer.lowercaseMode(lowercase_mode);
// std::string result;
// int tokens = tokenizer.countTokens();
// // why oh why is this clear() required to get it to work???
// pastStream.clear();
// int j = 0;
// while (tokenizer.hasMoreTokens() && j < tokens - index) {
// result = tokenizer.nextToken();
// j++;
//
// std::cerr << "ContextTracker::getToken() current token: " << result << std::endl;
// }
// return result;
}
std::string ContextTracker::getExtraTokenToLearn(const int index, const std::vector<std::string>& change) const
{
//logger << DEBUG
// << "past_stream : " << getPastStream() << endl
// << "change : " << contextChangeDetector->change(getPastStream()) << endl
// << "sliding_window: " << contextChangeDetector->get_sliding_window() + "\n" << endl;
// Extra tokens to learn are to be found in (past_stream - change)
//
// The change tokens are tokens that have not been seen or learnt
// before.
//
// The extra tokens to learn are tokens that have been seen and
// learn before, but that we need to reuse to fill out the n-gram
// of required cardinality that we are about to learn.
//
// To find the extra tokens to learn, we use the size of tokenized
// change vector to offset the index and extract the extra tokens
// to learn from the past stream.
//
// For example:
// past_stream : "The quick brown fox jumped over the "
// change : |over|the|
// extra_tokens: |The|quick|brown|fox|jumped|
//
return getToken(index + change.size());
}
std::string ContextTracker::getFutureStream() const
{
return context_tracker_callback->get_future_stream();
}
std::string ContextTracker::getPastStream() const
{
std::string result = context_tracker_callback->get_past_stream();
return result;
}
bool ContextTracker::isCompletionValid(const std::string& completion) const
{
bool result = false;
std::string prefix = getPrefix();
prefix = Utility::strtolower(prefix); // no need to be case sensitive
if (completion.find(prefix) == 0) {
result = true;
}
return result;
}
bool ContextTracker::isWordChar(const char c) const
{
if(wordChars.find(c, 0) != std::string::npos)
return true;
else
return false;
}
bool ContextTracker::isSeparatorChar(const char c) const
{
if(separatorChars.find(c, 0) != std::string::npos)
return true;
else
return false;
}
bool ContextTracker::isBlankspaceChar(const char c) const
{
if(blankspaceChars.find(c, 0) != std::string::npos)
return true;
else
return false;
}
bool ContextTracker::isControlChar(const char c) const
{
if(controlChars.find(c, 0) != std::string::npos)
return true;
else
return false;
}
std::string ContextTracker::getWordChars() const
{
return wordChars;
}
std::string ContextTracker::getSeparatorChars() const
{
return separatorChars;
}
std::string ContextTracker::getBlankspaceChars() const
{
return blankspaceChars;
}
std::string ContextTracker::getControlChars() const
{
return controlChars;
}
std::string ContextTracker::toString() const
{
return context_tracker_callback->get_past_stream() + "<|>" + context_tracker_callback->get_future_stream() + "\n";
}
void ContextTracker::update (const Observable* variable)
{
logger << DEBUG << "Notification received: "
<< variable->get_name () << " - " << variable->get_value () << endl;
dispatcher.dispatch (variable);
}
ContextTracker::context_tracker_callback
const PresageCallback * context_tracker_callback
Definition: contextTracker.h:231
ContextTracker::contextChange
bool contextChange()
Definition: contextTracker.cpp:121
ContextTracker::blankspaceChars
std::string blankspaceChars
Definition: contextTracker.h:215
ContextTracker::LOGGER
static const char * LOGGER
Definition: contextTracker.h:207
ContextTracker::contextChangeDetector
ContextChangeDetector * contextChangeDetector
Definition: contextTracker.h:233
ContextChangeDetector::change
std::string change(const std::string &past_stream) const
Definition: contextChangeDetector.cpp:203
ContextTracker::learn
void learn(const std::string &text) const
Learn from text.
Definition: contextTracker.cpp:140
ContextTracker::set_online_learning
void set_online_learning(const std::string &value)
Definition: contextTracker.cpp:103
PresageCallback
Definition: presageCallback.h:66
ReverseTokenizer
Definition: reverseTokenizer.h:38
ContextTracker::predictorRegistry
PredictorRegistry * predictorRegistry
Definition: contextTracker.h:232
ContextTracker::isBlankspaceChar
bool isBlankspaceChar(const char) const
Definition: contextTracker.cpp:300
Dispatcher::dispatch
void dispatch(const Observable *var)
Definition: dispatcher.h:73
ContextTracker::ONLINE_LEARNING
static const char * ONLINE_LEARNING
Definition: contextTracker.h:210
ContextTracker::toString
std::string toString() const
Definition: contextTracker.cpp:336
ContextTracker::getControlChars
std::string getControlChars() const
Definition: contextTracker.cpp:331
Predictor
Definition: predictor.h:46
ContextTracker::ContextTracker
ContextTracker(Configuration *config, PredictorRegistry *predictorRegistry, PresageCallback *callback, const unsigned char[]=DEFAULT_WORD_CHARS, const char[]=DEFAULT_SEPARATOR_CHARS, const char[]=DEFAULT_BLANKSPACE_CHARS, const char[]=DEFAULT_CONTROL_CHARS)
Definition: contextTracker.cpp:37
ContextTracker::getBlankspaceChars
std::string getBlankspaceChars() const
Definition: contextTracker.cpp:326
ContextTracker::getPastStream
std::string getPastStream() const
Definition: contextTracker.cpp:265
ContextTracker::set_lowercase_mode
void set_lowercase_mode(const std::string &value)
Definition: contextTracker.cpp:97
ContextTracker::callback
const PresageCallback * callback(const PresageCallback *callback)
Definition: contextTracker.cpp:109
contextTracker.h
ContextTracker::online_learning
bool online_learning
Definition: contextTracker.h:219
ContextTracker::controlChars
std::string controlChars
Definition: contextTracker.h:216
ContextTracker::isControlChar
bool isControlChar(const char) const
Definition: contextTracker.cpp:308
ContextTracker::~ContextTracker
~ContextTracker()
Definition: contextTracker.cpp:80
PredictorRegistry
Definition: predictorRegistry.h:46
setlevel
_SetLevel setlevel(std::string __l)
Manipulator for level.
Definition: logger.h:46
ContextChangeDetector::context_change
bool context_change(const std::string &past_stream) const
Definition: contextChangeDetector.cpp:75
ContextTracker::isCompletionValid
bool isCompletionValid(const std::string &) const
Definition: contextTracker.cpp:271
ContextTracker::getSeparatorChars
std::string getSeparatorChars() const
Definition: contextTracker.cpp:321
ForwardTokenizer::hasMoreTokens
virtual bool hasMoreTokens() const
Definition: forwardTokenizer.cpp:61
PredictorRegistry::iterator
Iterator iterator()
Definition: predictorRegistry.cpp:232
Configuration::find
Variable * find(const std::string &variable) const
Definition: configuration.cpp:46
Predictor::learn
virtual void learn(const std::vector< std::string > &change)=0
ContextTracker::set_sliding_window_size
void set_sliding_window_size(const std::string &value)
Definition: contextTracker.cpp:91
ContextTracker::update
void update()
Definition: contextTracker.cpp:126
PresageException
Definition: presageException.h:67
config
std::string config
Definition: presageDemo.cpp:70
ContextTracker::getFutureStream
std::string getFutureStream() const
Definition: contextTracker.cpp:260
ForwardTokenizer
Definition: forwardTokenizer.h:38
PredictorRegistry::Iterator::hasNext
bool hasNext() const
Definition: predictorRegistry.cpp:248
ContextTracker::wordChars
std::string wordChars
Definition: contextTracker.h:213
PresageCallback::get_past_stream
virtual std::string get_past_stream() const =0
PredictorRegistry::setContextTracker
void setContextTracker(ContextTracker *ct)
Definition: predictorRegistry.cpp:67
PredictorRegistry::Iterator::next
Predictor * next()
Definition: predictorRegistry.cpp:255
Tokenizer::lowercaseMode
void lowercaseMode(const bool)
Definition: tokenizer.cpp:81
ContextTracker::getExtraTokenToLearn
std::string getExtraTokenToLearn(const int index, const std::vector< std::string > &change) const
Definition: contextTracker.cpp:231
Configuration
Definition: configuration.h:36
Observable::get_name
virtual std::string get_name() const =0
PredictorRegistry::Iterator
Definition: predictorRegistry.h:53
ForwardTokenizer::nextToken
virtual std::string nextToken()
Definition: forwardTokenizer.cpp:72
Dispatcher::map
void map(Observable *var, const mbr_func_ptr_t &ptr)
Definition: dispatcher.h:62
Utility::strtolower
static char * strtolower(char *)
Definition: utility.cpp:42
ContextTracker::SLIDING_WINDOW_SIZE
static const char * SLIDING_WINDOW_SIZE
Definition: contextTracker.h:208
ContextTracker::LOWERCASE_MODE
static const char * LOWERCASE_MODE
Definition: contextTracker.h:209
ContextTracker::dispatcher
Dispatcher< ContextTracker > dispatcher
Definition: contextTracker.h:236
ContextChangeDetector
Definition: contextChangeDetector.h:36
ContextTracker::lowercase_mode
bool lowercase_mode
Definition: contextTracker.h:218
Utility::isYes
static bool isYes(const char *)
Definition: utility.cpp:185
ContextChangeDetector::set_sliding_window_size
void set_sliding_window_size(const std::string &str)
Definition: contextChangeDetector.cpp:54
ContextTracker::isWordChar
bool isWordChar(const char) const
Definition: contextTracker.cpp:284
ContextTracker::set_logger
void set_logger(const std::string &value)
Definition: contextTracker.cpp:85
ContextTracker::separatorChars
std::string separatorChars
Definition: contextTracker.h:214
Observable::get_value
virtual std::string get_value() const =0
PRESAGE_INVALID_CALLBACK_ERROR
Definition: presageException.h:39
ContextTracker::getPrefix
std::string getPrefix() const
Definition: contextTracker.cpp:186
ContextTracker::isSeparatorChar
bool isSeparatorChar(const char) const
Definition: contextTracker.cpp:292
Observable
Definition: observable.h:37
ContextTracker::logger
Logger< char > logger
Definition: contextTracker.h:234
ContextTracker::getWordChars
std::string getWordChars() const
Definition: contextTracker.cpp:316
PresageCallback::get_future_stream
virtual std::string get_future_stream() const =0
ContextChangeDetector::update_sliding_window
void update_sliding_window(const std::string &str)
Definition: contextChangeDetector.cpp:63
endl
const Logger< _charT, _Traits > & endl(const Logger< _charT, _Traits > &lgr)
Definition: logger.h:278
ContextTracker::getToken
std::string getToken(const int) const
Definition: contextTracker.cpp:191