Lucene++ - a full-featured, c++ search engine
API Documentation

- Main Page
- Related Pages
- Namespaces
- Data Structures
- Files

- File List
- Globals

All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Tokenizer.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2011 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
6
7 #ifndef TOKENIZER_H
8 #define TOKENIZER_H
9
10 #include "TokenStream.h"
11
12 namespace Lucene
13 {
20 class LPPAPI Tokenizer : public TokenStream
21 {
22 protected:
24 Tokenizer();
25
27 Tokenizer(ReaderPtr input);
28
30 Tokenizer(AttributeFactoryPtr factory);
31
33 Tokenizer(AttributeFactoryPtr factory, ReaderPtr input);
34
36 Tokenizer(AttributeSourcePtr source);
37
39 Tokenizer(AttributeSourcePtr source, ReaderPtr input);
40
41 public:
42 virtual ~Tokenizer();
43
44 LUCENE_CLASS(Tokenizer);
45
46 protected:
48 ReaderPtr input;
49 CharStreamPtr charStream;
50
51 public:
53 virtual void close();
54
60 virtual int32_t correctOffset(int32_t currentOff);
61
62 using TokenStream::reset;
63
66 virtual void reset(ReaderPtr input);
67 };
68 }
69
70 #endif

clucene.sourceforge.net