Lucene++ - a full-featured, c++ search engine
API Documentation

- Main Page
- Related Pages
- Namespaces
- Data Structures
- Files

- File List
- Globals

All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
StandardTokenizerImpl.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2011 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
6
7 #ifndef STANDARDTOKENIZERIMPL_H
8 #define STANDARDTOKENIZERIMPL_H
9
10 #include "LuceneObject.h"
11
12 namespace Lucene
13 {
14 class StandardTokenizerImpl : public LuceneObject
15 {
16 public:
19 StandardTokenizerImpl(ReaderPtr in);
20
21 virtual ~StandardTokenizerImpl();
22
23 LUCENE_CLASS(StandardTokenizerImpl);
24
25 protected:
27 static const int32_t ZZ_BUFFERSIZE;
28
30 static const wchar_t ZZ_CMAP_PACKED[];
31 static const int32_t ZZ_CMAP_LENGTH;
32 static const int32_t ZZ_CMAP_PACKED_LENGTH;
33
35 static const wchar_t* ZZ_CMAP();
36
38 static const wchar_t ZZ_ACTION_PACKED_0[];
39 static const int32_t ZZ_ACTION_LENGTH;
40 static const int32_t ZZ_ACTION_PACKED_LENGTH;
41
43 static const int32_t* ZZ_ACTION();
44
46 static const wchar_t ZZ_ROWMAP_PACKED_0[];
47 static const int32_t ZZ_ROWMAP_LENGTH;
48 static const int32_t ZZ_ROWMAP_PACKED_LENGTH;
49
51 static const int32_t* ZZ_ROWMAP();
52
54 static const wchar_t ZZ_TRANS_PACKED_0[];
55 static const int32_t ZZ_TRANS_LENGTH;
56 static const int32_t ZZ_TRANS_PACKED_LENGTH;
57
59 static const int32_t* ZZ_TRANS();
60
61 // error codes
62 static const int32_t ZZ_UNKNOWN_ERROR;
63 static const int32_t ZZ_NO_MATCH;
64 static const int32_t ZZ_PUSHBACK_2BIG;
65
66 static const wchar_t* ZZ_ERROR_MSG[];
67
69 static const wchar_t ZZ_ATTRIBUTE_PACKED_0[];
70 static const int32_t ZZ_ATTRIBUTE_LENGTH;
71 static const int32_t ZZ_ATTRIBUTE_PACKED_LENGTH;
72
74 static const int32_t* ZZ_ATTRIBUTE();
75
77 ReaderPtr zzReader;
78
80 int32_t zzState;
81
83 int32_t zzLexicalState;
84
86 CharArray zzBuffer;
87
89 int32_t zzMarkedPos;
90
92 int32_t zzPushbackPos;
93
95 int32_t zzCurrentPos;
96
98 int32_t zzStartRead;
99
101 int32_t zzEndRead;
102
104 int32_t yyline;
105
107 int32_t _yychar;
108
110 int32_t yycolumn;
111
113 bool zzAtBOL;
114
116 bool zzAtEOF;
117
118 public:
120 static const int32_t YYEOF;
121
123 static const int32_t YYINITIAL;
124
125 public:
126 int32_t yychar();
127
129 void reset(ReaderPtr r);
130
132 void getText(TokenPtr t);
133
135 void getText(TermAttributePtr t);
136
138 void yyclose();
139
146 void yyreset(ReaderPtr reader);
147
149 int32_t yystate();
150
153 void yybegin(int32_t newState);
154
156 String yytext();
157
163 wchar_t yycharat(int32_t pos);
164
166 int32_t yylength();
167
172 void yypushback(int32_t number);
173
176 int32_t getNextToken();
177
178 protected:
180 bool zzRefill();
181
191 void zzScanError(int32_t errorCode);
192 };
193 }
194
195 #endif

clucene.sourceforge.net