contextChangeDetector.cpp
Go to the documentation of this file.
/******************************************************
* Presage, an extensible predictive text entry system
* ---------------------------------------------------
*
* Copyright (C) 2008 Matteo Vescovi <matteo.vescovi@yahoo.co.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
**********(*)*/
#include "contextChangeDetector.h"
#include "../tokenizer/reverseTokenizer.h"
#include <iostream>
#include <sstream>
#include <stdlib.h> // for atoi()
#include <assert.h>
const std::string::size_type ContextChangeDetector::DEFAULT_SLIDING_WINDOW_SIZE = 80;
ContextChangeDetector::ContextChangeDetector(const std::string wChars,
const std::string tChars,
const std::string bChars,
const std::string cChars,
bool lowercase)
: wordChars (wChars),
separatorChars (tChars),
blankspaceChars(bChars),
controlChars (cChars),
lowercase_mode (lowercase)
{
// intentionally empty
}
ContextChangeDetector::~ContextChangeDetector()
{
// intentionally empty
}
void ContextChangeDetector::set_sliding_window_size(const std::string& str)
{
if(!str.empty()) {
SLIDING_WINDOW_SIZE = atoi(str.c_str());
} else {
SLIDING_WINDOW_SIZE = DEFAULT_SLIDING_WINDOW_SIZE;
}
}
void ContextChangeDetector::update_sliding_window(const std::string& str)
{
if (str.size() <= SLIDING_WINDOW_SIZE) {
// past stream fits in sliding window
sliding_window = str;
} else {
// trim past stream down to sliding window
sliding_window = str.substr(str.size() - SLIDING_WINDOW_SIZE);
assert(sliding_window.size() == SLIDING_WINDOW_SIZE);
}
}
bool ContextChangeDetector::context_change(const std::string& past_stream) const
{
// Here's how this is going to be implemented... We'll keep a
// sliding window on the last few chars seen by presage; the
// buffer holding them is the sliding window. We'll search for the
// last occurence of sliding_window in past_stream, if any.
// If sliding_window is not found in past_stream, then it is not
// possible to relate the current context to the previously seen
// context, hence we assume a context change has occured.
// If sliding_window is found, then we need to examine the chars
// following the sliding window in the past stream. We call this
// the remainder. If there are any non-word chars in the
// remainder, then a context change has occurred. Else, no context
// change occured.
// The sliding window is never implicitly updated as part of
// invoking this method.
return context_change_helper(sliding_window, past_stream);
}
bool ContextChangeDetector::context_change_helper(const std::string& prev_context, const std::string& curr_context) const
{
bool result = false;
if (prev_context.empty()) {
if (curr_context.empty()) {
// both contexts are empty, nothing has happened, no
// change happened
result = false;
} else {
// current context changed, previous context is empty,
// first change happened
result = true;
}
} else {
// find position of previous context in current context
// i.e. find index pointing to last char of last occurence of
// prev_context in curr_context
std::string::size_type ctx_idx = curr_context.rfind(prev_context);
if (ctx_idx == std::string::npos) {
// prev_context could not be found in curr_context, a lot
// changed
result = true;
} else {
// found prev_context, examine remainder string.
// remainder string is substr(ctx_idx +
// prev_context.size()); i.e. substring given by index
// returned by rfind (which points at beginning of
// prev_context string found in curr_context) plus size of
// prev_context: this index points at end of prev_context
// substring found in curr_context
std::string remainder = curr_context.substr(ctx_idx + prev_context.size());
std::string::size_type idx = remainder.find_last_of(wordChars);
if (idx == std::string::npos) {
if (remainder.empty()) {
result = false;
} else {
char last_char = curr_context[ctx_idx + prev_context.size() - 1];
idx = wordChars.find(last_char);
if (idx == std::string::npos) {
result = false;
} else {
result = true;
}
}
} else {
if (idx == remainder.size() - 1) {
result = false;
} else {
result = true;
}
}
/*
* alternate implementation of the logic in the enclosing else
* block. This uses tokenizers, which is not desirable as it makes
* tokenizer a dependency of context change detector.
std::string remainder = curr_context.substr(loc + prev_context.size());
std::stringstream curr_strstream(curr_context);
std::stringstream prev_strstream(prev_context);
ReverseTokenizer curr_tokenizer(curr_strstream, blankspaceChars, separatorChars);
ReverseTokenizer prev_tokenizer(prev_strstream, blankspaceChars, separatorChars);
std::string prev_token = prev_tokenizer.nextToken();
std::string curr_token = curr_tokenizer.nextToken();
if (curr_token.empty()) {
if (prev_token.empty()) {
result = false;
loc = curr_context.find_first_of(wordChars, loc);
if (loc == std::string::npos) {
result = false;
} else {
result = true;
}
} else {
result = true;
}
} else {
loc = curr_token.find(prev_token);
if (loc == std::string::npos) {
result = true;
} else {
result = false;
}
}
*/
}
}
return result;
}
std::string ContextChangeDetector::change(const std::string& past_stream) const
{
const std::string& prev_context = sliding_window; // let's rename these
const std::string& curr_context = past_stream; // for clarity's sake
std::string result;
if (sliding_window.empty()) {
result = past_stream;
} else {
// find position of previous context in current context
// i.e. find index pointing to last char of last occurence of
// prev_context in curr_context
std::string::size_type ctx_idx = curr_context.rfind(prev_context);
if (ctx_idx == std::string::npos) {
// prev_context could not be found in curr_context, a lot
// changed
result = past_stream;
} else {
// found prev_context, examine remainder string.
// remainder string is substr(ctx_idx +
// prev_context.size()); i.e. substring given by index
// returned by rfind (which points at beginning of
// prev_context string found in curr_context) plus size of
// prev_context: this index points at end of prev_context
// substring found in curr_context
result = curr_context.substr(ctx_idx + prev_context.size());
// handle case where a context change has occured and
// remainder string only contains part of the last token,
// i.e.:
//
// sliding_window = "The quick bro";
// past_stream = "The quick brown ";
//
// In this case, the remainder will only contain "wn", and
// the last token in the sliding window must be prepended
// to the change to be learnt
//
if (context_change(past_stream)) {
// prepend partially entered token to change if it
// exists, need to look into sliding_window to get
// previously partially entered token if it exists
std::stringstream sliding_window_stream;
sliding_window_stream << get_sliding_window();
ReverseTokenizer rTok(sliding_window_stream,
blankspaceChars,
separatorChars);
rTok.lowercaseMode(lowercase_mode);
std::string first_token = rTok.nextToken();
if (!first_token.empty()) {
result = first_token + result;
}
}
}
}
return result;
}
std::string ContextChangeDetector::get_sliding_window() const
{
return sliding_window;
}
Tokenizer::lowercaseMode
void lowercaseMode(const bool)
Definition: tokenizer.cpp:81
ContextChangeDetector::lowercase_mode
bool lowercase_mode
Definition: contextChangeDetector.h:84
ContextChangeDetector::separatorChars
const std::string separatorChars
Definition: contextChangeDetector.h:81
ContextChangeDetector::~ContextChangeDetector
~ContextChangeDetector()
Definition: contextChangeDetector.cpp:49
ContextChangeDetector::ContextChangeDetector
ContextChangeDetector(const std::string, const std::string, const std::string, const std::string, bool)
Definition: contextChangeDetector.cpp:35
ContextChangeDetector::context_change_helper
bool context_change_helper(const std::string &str1, const std::string &str2) const
Definition: contextChangeDetector.cpp:99
ReverseTokenizer::nextToken
virtual std::string nextToken()
Definition: reverseTokenizer.cpp:80
ContextChangeDetector::DEFAULT_SLIDING_WINDOW_SIZE
static const std::string::size_type DEFAULT_SLIDING_WINDOW_SIZE
Definition: contextChangeDetector.h:76
ContextChangeDetector::blankspaceChars
const std::string blankspaceChars
Definition: contextChangeDetector.h:82
contextChangeDetector.h
ContextChangeDetector::sliding_window
std::string sliding_window
Definition: contextChangeDetector.h:78
ContextChangeDetector::update_sliding_window
void update_sliding_window(const std::string &str)
Definition: contextChangeDetector.cpp:63
ContextChangeDetector::set_sliding_window_size
void set_sliding_window_size(const std::string &str)
Definition: contextChangeDetector.cpp:54
ContextChangeDetector::get_sliding_window
std::string get_sliding_window() const
Definition: contextChangeDetector.cpp:265
ContextChangeDetector::SLIDING_WINDOW_SIZE
std::string::size_type SLIDING_WINDOW_SIZE
Definition: contextChangeDetector.h:77
ReverseTokenizer
Definition: reverseTokenizer.h:38
ContextChangeDetector::wordChars
const std::string wordChars
Definition: contextChangeDetector.h:80
ContextChangeDetector::context_change
bool context_change(const std::string &past_stream) const
Definition: contextChangeDetector.cpp:75
ContextChangeDetector::change
std::string change(const std::string &past_stream) const
Definition: contextChangeDetector.cpp:203