Lucene++ - a full-featured, c++ search engine
API Documentation

Loading...
Searching...
No Matches
KeywordTokenizer.h
Go to the documentation of this file.
1
2// Copyright (c) 2009-2014 Alan Wright. All rights reserved.
3// Distributable under the terms of either the Apache License (Version 2.0)
4// or the GNU Lesser General Public License.
6
7#ifndef KEYWORDTOKENIZER_H
8#define KEYWORDTOKENIZER_H
9
10#include "Tokenizer.h"
11
12 namespace Lucene {
13
15 class LPPAPI KeywordTokenizer : public Tokenizer {
16 public:
17 KeywordTokenizer(const ReaderPtr& input);
18 KeywordTokenizer(const ReaderPtr& input, int32_t bufferSize);
19 KeywordTokenizer(const AttributeSourcePtr& source, const ReaderPtr& input, int32_t bufferSize);
20 KeywordTokenizer(const AttributeFactoryPtr& factory, const ReaderPtr& input, int32_t bufferSize);
21
22 virtual ~KeywordTokenizer();
23
24 LUCENE_CLASS(KeywordTokenizer);
25
26 protected:
27 static const int32_t DEFAULT_BUFFER_SIZE;
28
29 bool done;
30 int32_t finalOffset;
31 TermAttributePtr termAtt;
32 OffsetAttributePtr offsetAtt;
33
34 protected:
35 void init(int32_t bufferSize);
36
37 public:
38 virtual bool incrementToken();
39 virtual void end();
40 virtual void reset();
41};
42
43}
44
45#endif
LUCENE_CLASS
#define LUCENE_CLASS(Name)
Definition LuceneObject.h:24
Tokenizer.h
Lucene::AttributeSource::factory
AttributeFactoryPtr factory
Definition AttributeSource.h:60
Lucene::KeywordTokenizer::offsetAtt
OffsetAttributePtr offsetAtt
Definition KeywordTokenizer.h:32
Lucene::KeywordTokenizer::init
void init(int32_t bufferSize)
Lucene::KeywordTokenizer::incrementToken
virtual bool incrementToken()
Consumers (ie., IndexWriter) use this method to advance the stream to the next token....
Lucene::KeywordTokenizer::~KeywordTokenizer
virtual ~KeywordTokenizer()
Lucene::KeywordTokenizer::done
bool done
Definition KeywordTokenizer.h:29
Lucene::KeywordTokenizer::KeywordTokenizer
KeywordTokenizer(const ReaderPtr &input)
Lucene::KeywordTokenizer::KeywordTokenizer
KeywordTokenizer(const AttributeFactoryPtr &factory, const ReaderPtr &input, int32_t bufferSize)
Lucene::KeywordTokenizer::reset
virtual void reset()
Resets this stream to the beginning. This is an optional operation, so subclasses may or may not impl...
Lucene::KeywordTokenizer::termAtt
TermAttributePtr termAtt
Definition KeywordTokenizer.h:31
Lucene::KeywordTokenizer::DEFAULT_BUFFER_SIZE
static const int32_t DEFAULT_BUFFER_SIZE
Definition KeywordTokenizer.h:27
Lucene::KeywordTokenizer::KeywordTokenizer
KeywordTokenizer(const ReaderPtr &input, int32_t bufferSize)
Lucene::KeywordTokenizer::finalOffset
int32_t finalOffset
Definition KeywordTokenizer.h:30
Lucene::KeywordTokenizer::KeywordTokenizer
KeywordTokenizer(const AttributeSourcePtr &source, const ReaderPtr &input, int32_t bufferSize)
Lucene::KeywordTokenizer::end
virtual void end()
This method is called by the consumer after the last token has been consumed, after incrementToken() ...
Lucene::Tokenizer::Tokenizer
Tokenizer()
Construct a tokenizer with null input.
Lucene::Tokenizer::input
ReaderPtr input
The text source for this Tokenizer.
Definition Tokenizer.h:47
Lucene
Definition AbstractAllTermDocs.h:12
Lucene::ReaderPtr
boost::shared_ptr< Reader > ReaderPtr
Definition LuceneTypes.h:547
Lucene::TermAttributePtr
boost::shared_ptr< TermAttribute > TermAttributePtr
Definition LuceneTypes.h:58
Lucene::AttributeFactoryPtr
boost::shared_ptr< AttributeFactory > AttributeFactoryPtr
Definition LuceneTypes.h:519
Lucene::AttributeSourcePtr
boost::shared_ptr< AttributeSource > AttributeSourcePtr
Definition LuceneTypes.h:520
Lucene::OffsetAttributePtr
boost::shared_ptr< OffsetAttribute > OffsetAttributePtr
Definition LuceneTypes.h:40

clucene.sourceforge.net