Lucene++ - a full-featured, c++ search engine
API Documentation

Token.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2014 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
6
7 #ifndef TOKEN_H
8 #define TOKEN_H
9
10 #include "Attribute.h"
11 #include "AttributeSource.h"
12
13 namespace Lucene {
14
76 class LPPAPI Token : public Attribute {
77 public:
79 Token();
80
84 Token(int32_t start, int32_t end);
85
90 Token(int32_t start, int32_t end, const String& type);
91
96 Token(int32_t start, int32_t end, int32_t flags);
97
103 Token(const String& text, int32_t start, int32_t end);
104
111 Token(const String& text, int32_t start, int32_t end, const String& type);
112
119 Token(const String& text, int32_t start, int32_t end, int32_t flags);
120
122 Token(CharArray startTermBuffer, int32_t termBufferOffset, int32_t termBufferLength, int32_t start, int32_t end);
123
124 virtual ~Token();
125
126 LUCENE_CLASS(Token);
127
128 public:
129 static const String& DEFAULT_TYPE();
130
131 protected:
132 static const int32_t MIN_BUFFER_SIZE;
133
134 CharArray _termBuffer;
135 int32_t _termLength;
136 int32_t _startOffset;
137 int32_t _endOffset;
138 String _type;
139 int32_t flags;
140 PayloadPtr payload;
141 int32_t positionIncrement;
142
143 public:
163 virtual void setPositionIncrement(int32_t positionIncrement);
164
167 virtual int32_t getPositionIncrement();
168
174 virtual String term();
175
180 virtual void setTermBuffer(const wchar_t* buffer, int32_t offset, int32_t length);
181
184 virtual void setTermBuffer(const String& buffer);
185
190 virtual void setTermBuffer(const String& buffer, int32_t offset, int32_t length);
191
195 virtual CharArray termBuffer();
196
198 virtual wchar_t* termBufferArray();
199
206 virtual CharArray resizeTermBuffer(int32_t newSize);
207
209 virtual int32_t termLength();
210
215 virtual void setTermLength(int32_t length);
216
222 virtual int32_t startOffset();
223
226 virtual void setStartOffset(int32_t offset);
227
230 virtual int32_t endOffset();
231
234 virtual void setEndOffset(int32_t offset);
235
238 virtual void setOffset(int32_t startOffset, int32_t endOffset);
239
241 virtual String type();
242
245 virtual void setType(const String& type);
246
252 virtual int32_t getFlags();
253
255 virtual void setFlags(int32_t flags);
256
258 virtual PayloadPtr getPayload();
259
261 virtual void setPayload(const PayloadPtr& payload);
262
263 virtual String toString();
264
266 virtual void clear();
267
268 virtual LuceneObjectPtr clone(const LuceneObjectPtr& other = LuceneObjectPtr());
269
272 TokenPtr clone(CharArray newTermBuffer, int32_t newTermOffset, int32_t newTermLength, int32_t newStartOffset, int32_t newEndOffset);
273
274 virtual bool equals(const LuceneObjectPtr& other);
275 virtual int32_t hashCode();
276
280 TokenPtr reinit(CharArray newTermBuffer, int32_t newTermOffset, int32_t newTermLength, int32_t newStartOffset, int32_t newEndOffset, const String& newType);
281
285 TokenPtr reinit(CharArray newTermBuffer, int32_t newTermOffset, int32_t newTermLength, int32_t newStartOffset, int32_t newEndOffset);
286
290 TokenPtr reinit(const String& newTerm, int32_t newStartOffset, int32_t newEndOffset, const String& newType);
291
295 TokenPtr reinit(const String& newTerm, int32_t newTermOffset, int32_t newTermLength, int32_t newStartOffset, int32_t newEndOffset, const String& newType);
296
300 TokenPtr reinit(const String& newTerm, int32_t newStartOffset, int32_t newEndOffset);
301
305 TokenPtr reinit(const String& newTerm, int32_t newTermOffset, int32_t newTermLength, int32_t newStartOffset, int32_t newEndOffset);
306
308 void reinit(const TokenPtr& prototype);
309
311 void reinit(const TokenPtr& prototype, const String& newTerm);
312
314 void reinit(const TokenPtr& prototype, CharArray newTermBuffer, int32_t offset, int32_t length);
315
316 virtual void copyTo(const AttributePtr& target);
317
319 static AttributeFactoryPtr TOKEN_ATTRIBUTE_FACTORY();
320
321 protected:
323 void ConstructToken(int32_t start, int32_t end, const String& type, int32_t flags);
324
328 void growTermBuffer(int32_t newSize);
329
330 void initTermBuffer();
331
333 void clearNoTermBuffer();
334 };
335
338 class LPPAPI TokenAttributeFactory : public AttributeFactory {
339 public:
340 TokenAttributeFactory(const AttributeFactoryPtr& delegate);
341 virtual ~TokenAttributeFactory();
342
343 LUCENE_CLASS(TokenAttributeFactory);
344
345 protected:
346 AttributeFactoryPtr delegate;
347
348 public:
349 virtual AttributePtr createAttributeInstance(const String& className);
350 virtual bool equals(const LuceneObjectPtr& other);
351 virtual int32_t hashCode();
352 };
353
354 }
355
356 #endif
LUCENE_CLASS
#define LUCENE_CLASS(Name)
Definition: LuceneObject.h:24
Lucene::LuceneObjectPtr
boost::shared_ptr< LuceneObject > LuceneObjectPtr
Definition: LuceneTypes.h:539
Lucene::Attribute
Base class for Attributes that can be added to a AttributeSource.
Definition: Attribute.h:18
Lucene::TokenAttributeFactory
Creates a TokenAttributeFactory returning Token as instance for the basic attributes and for all othe...
Definition: Token.h:338
Lucene::AttributeFactory
Definition: AttributeSource.h:14
Lucene::TokenPtr
boost::shared_ptr< Token > TokenPtr
Definition: LuceneTypes.h:59
Lucene::Token::_type
String _type
Definition: Token.h:138
Lucene
Definition: AbstractAllTermDocs.h:12
Lucene::Token::_termLength
int32_t _termLength
Definition: Token.h:135
AttributeSource.h
Lucene::Token::MIN_BUFFER_SIZE
static const int32_t MIN_BUFFER_SIZE
Definition: Token.h:132
Lucene::AttributePtr
boost::shared_ptr< Attribute > AttributePtr
Definition: LuceneTypes.h:518
Lucene::Token::flags
int32_t flags
Definition: Token.h:139
Lucene::Token::positionIncrement
int32_t positionIncrement
Definition: Token.h:141
Lucene::PayloadPtr
boost::shared_ptr< Payload > PayloadPtr
Definition: LuceneTypes.h:198
Lucene::Token::_startOffset
int32_t _startOffset
Definition: Token.h:136
Lucene::Token::_termBuffer
CharArray _termBuffer
Definition: Token.h:134
Lucene::Token::payload
PayloadPtr payload
Definition: Token.h:140
Attribute.h
Lucene::AttributeFactoryPtr
boost::shared_ptr< AttributeFactory > AttributeFactoryPtr
Definition: LuceneTypes.h:519
Lucene::Token
A Token is an occurrence of a term from the text of a field. It consists of a term's text,...
Definition: Token.h:76
Lucene::Token::_endOffset
int32_t _endOffset
Definition: Token.h:137

clucene.sourceforge.net