Lucene++ - a full-featured, c++ search engine
API Documentation

- Main Page
- Related Pages
- Namespaces
- Data Structures
- Files

- File List
- Globals

All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
IndexWriter.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2011 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
6
7 #ifndef INDEXWRITER_H
8 #define INDEXWRITER_H
9
10 #include "MergePolicy.h"
11
12 namespace Lucene
13 {
90 class LPPAPI IndexWriter : public LuceneObject
91 {
92 protected:
93 IndexWriter(DirectoryPtr d, AnalyzerPtr a, bool create, IndexDeletionPolicyPtr deletionPolicy, int32_t mfl, IndexingChainPtr indexingChain, IndexCommitPtr commit);
94
95 public:
96 IndexWriter(DirectoryPtr d, AnalyzerPtr a, bool create, int32_t mfl);
97 IndexWriter(DirectoryPtr d, AnalyzerPtr a, int32_t mfl);
98 IndexWriter(DirectoryPtr d, AnalyzerPtr a, IndexDeletionPolicyPtr deletionPolicy, int32_t mfl);
99 IndexWriter(DirectoryPtr d, AnalyzerPtr a, bool create, IndexDeletionPolicyPtr deletionPolicy, int32_t mfl);
100 IndexWriter(DirectoryPtr d, AnalyzerPtr a, IndexDeletionPolicyPtr deletionPolicy, int32_t mfl, IndexCommitPtr commit);
101 virtual ~IndexWriter();
102
103 LUCENE_CLASS(IndexWriter);
104
105 protected:
106 int64_t writeLockTimeout;
107
111 static const int32_t MERGE_READ_BUFFER_SIZE;
112
113 SynchronizePtr messageIDLock;
114 static int32_t MESSAGE_ID;
115 int32_t messageID;
116 bool hitOOM;
117
118 DirectoryPtr directory; // where this index resides
119 AnalyzerPtr analyzer; // how to analyze text
120
121 bool create;
122 IndexDeletionPolicyPtr deletionPolicy;
123 IndexingChainPtr indexingChain;
124 IndexCommitPtr indexCommit;
125
126 SimilarityPtr similarity; // how to normalize
127
128 int64_t changeCount; // increments every time a change is completed
129 int64_t lastCommitChangeCount; // last changeCount that was committed
130
131 SegmentInfosPtr rollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
132 MapSegmentInfoInt rollbackSegments;
133
134 SegmentInfosPtr localRollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
135 int32_t localFlushedDocCount;
136
137 SegmentInfosPtr segmentInfos; // the segments
138
139 DocumentsWriterPtr docWriter;
140 IndexFileDeleterPtr deleter;
141
142 SetSegmentInfo segmentsToOptimize; // used by optimize to note those needing optimization
143 int32_t optimizeMaxNumSegments;
144
145 LockPtr writeLock;
146
147 int32_t termIndexInterval;
148
149 bool closed;
150 bool closing;
151
152 SetSegmentInfo mergingSegments;
153 MergePolicyPtr mergePolicy;
154 MergeSchedulerPtr mergeScheduler;
155 Collection<OneMergePtr> pendingMerges;
156 SetOneMerge runningMerges;
157 Collection<OneMergePtr> mergeExceptions;
158 int64_t mergeGen;
159 bool stopMerges;
160
161 int32_t flushCount;
162 int32_t flushDeletesCount;
163
165 int32_t readCount; // count of how many threads are holding read lock
166 int64_t writeThread; // non-null if any thread holds write lock
167 int32_t upgradeCount;
168
169 int32_t readerTermsIndexDivisor;
170
171 // This is a "write once" variable (like the organic dye on a DVD-R that may or may not
172 // be heated by a laser and then cooled to permanently record the event): it's false,
173 // until getReader() is called for the first time, at which point it's switched to true
174 // and never changes back to false. Once this is true, we hold open and reuse SegmentReader
175 // instances internally for applying deletes, doing merges, and reopening near real-time readers.
176 bool poolReaders;
177
188 int32_t maxFieldLength;
189
190 InfoStreamPtr infoStream;
191 static InfoStreamPtr defaultInfoStream;
192
193 HashSet<String> synced; // files that have been sync'd already
194 HashSet<String> syncing; // files that are now being sync'd
195
196 IndexReaderWarmerPtr mergedSegmentWarmer;
197
199 SynchronizePtr commitLock;
200
201 INTERNAL:
202 SegmentInfosPtr pendingCommit; // set when a commit is pending (after prepareCommit() & before commit())
203 int64_t pendingCommitChangeCount;
204
205 ReaderPoolPtr readerPool;
206
207 public:
210 static int64_t WRITE_LOCK_TIMEOUT;
211
212 static const String WRITE_LOCK_NAME;
213
215 static const int32_t DISABLE_AUTO_FLUSH;
216
219 static const int32_t DEFAULT_MAX_BUFFERED_DOCS;
220
223 static const double DEFAULT_RAM_BUFFER_SIZE_MB;
224
227 static const int32_t DEFAULT_MAX_BUFFERED_DELETE_TERMS;
228
230 static const int32_t DEFAULT_MAX_FIELD_LENGTH;
231
233 static const int32_t DEFAULT_TERM_INDEX_INTERVAL;
234
238 static int32_t MAX_TERM_LENGTH();
239
241 static const int32_t MaxFieldLengthUNLIMITED;
242
244 static const int32_t MaxFieldLengthLIMITED;
245
246 public:
247 virtual void initialize();
248
286 virtual IndexReaderPtr getReader();
287
297 virtual IndexReaderPtr getReader(int32_t termInfosIndexDivisor);
298
301 virtual int32_t numDeletedDocs(SegmentInfoPtr info);
302
303 virtual void acquireWrite();
304 virtual void releaseWrite();
305 virtual void acquireRead();
306
309 virtual void upgradeReadToWrite();
310
311 virtual void releaseRead();
312 virtual bool isOpen(bool includePendingClose);
313 virtual void message(const String& message);
314
323 virtual bool getUseCompoundFile();
324
331 virtual void setUseCompoundFile(bool value);
332
334 virtual void setSimilarity(SimilarityPtr similarity);
335
338 virtual SimilarityPtr getSimilarity();
339
356 virtual void setTermIndexInterval(int32_t interval);
357
360 virtual int32_t getTermIndexInterval();
361
363 virtual void setMergePolicy(MergePolicyPtr mp);
364
367 virtual MergePolicyPtr getMergePolicy();
368
370 virtual void setMergeScheduler(MergeSchedulerPtr mergeScheduler);
371
374 virtual MergeSchedulerPtr getMergeScheduler();
375
389 virtual void setMaxMergeDocs(int32_t maxMergeDocs);
390
399 virtual int32_t getMaxMergeDocs();
400
410 virtual void setMaxFieldLength(int32_t maxFieldLength);
411
414 virtual int32_t getMaxFieldLength();
415
419 virtual void setReaderTermsIndexDivisor(int32_t divisor);
420
422 virtual int32_t getReaderTermsIndexDivisor();
423
435 virtual void setMaxBufferedDocs(int32_t maxBufferedDocs);
436
439 virtual int32_t getMaxBufferedDocs();
440
462 virtual void setRAMBufferSizeMB(double mb);
463
465 virtual double getRAMBufferSizeMB();
466
473 virtual void setMaxBufferedDeleteTerms(int32_t maxBufferedDeleteTerms);
474
477 virtual int32_t getMaxBufferedDeleteTerms();
478
488 virtual void setMergeFactor(int32_t mergeFactor);
489
497 virtual int32_t getMergeFactor();
498
501 static void setDefaultInfoStream(InfoStreamPtr infoStream);
502
505 static InfoStreamPtr getDefaultInfoStream();
506
509 virtual void setInfoStream(InfoStreamPtr infoStream);
510
513 virtual InfoStreamPtr getInfoStream();
514
516 virtual bool verbose();
517
521 virtual void setWriteLockTimeout(int64_t writeLockTimeout);
522
525 virtual int64_t getWriteLockTimeout();
526
529 static void setDefaultWriteLockTimeout(int64_t writeLockTimeout);
530
533 static int64_t getDefaultWriteLockTimeout();
534
548 virtual void close();
549
562 virtual void close(bool waitForMerges);
563
565 virtual DirectoryPtr getDirectory();
566
568 virtual AnalyzerPtr getAnalyzer();
569
573 virtual int32_t maxDoc();
574
579 virtual int32_t numDocs();
580
581 virtual bool hasDeletions();
582
604 virtual void addDocument(DocumentPtr doc);
605
614 virtual void addDocument(DocumentPtr doc, AnalyzerPtr analyzer);
615
621 virtual void deleteDocuments(TermPtr term);
622
628 virtual void deleteDocuments(Collection<TermPtr> terms);
629
635 virtual void deleteDocuments(QueryPtr query);
636
643 virtual void deleteDocuments(Collection<QueryPtr> queries);
644
653 virtual void updateDocument(TermPtr term, DocumentPtr doc);
654
664 virtual void updateDocument(TermPtr term, DocumentPtr doc, AnalyzerPtr analyzer);
665
666 virtual int32_t getSegmentCount();
667 virtual int32_t getNumBufferedDocuments();
668 virtual int32_t getDocCount(int32_t i);
669 virtual int32_t getFlushCount();
670 virtual int32_t getFlushDeletesCount();
671
672 virtual String newSegmentName();
673
708 virtual void optimize();
709
716 virtual void optimize(int32_t maxNumSegments);
717
723 virtual void optimize(bool doWait);
724
730 virtual void optimize(int32_t maxNumSegments, bool doWait);
731
737 virtual void expungeDeletes(bool doWait);
738
750 virtual void expungeDeletes();
751
760 virtual void maybeMerge();
761
764 virtual OneMergePtr getNextMerge();
765
771 virtual void rollback();
772
784 virtual void deleteAll();
785
790 virtual void waitForMerges();
791
822 virtual void addIndexesNoOptimize(Collection<DirectoryPtr> dirs);
823
834 virtual void addIndexes(Collection<IndexReaderPtr> readers);
835
840 virtual void prepareCommit();
841
857 // will only "stick" if there are actually changes in the index to commit.
858 virtual void prepareCommit(MapStringString commitUserData);
859
877 virtual void commit();
878
884 virtual void commit(MapStringString commitUserData);
885
888 virtual int64_t ramSizeInBytes();
889
891 virtual int32_t numRamDocs();
892
894 virtual void merge(OneMergePtr merge);
895
897 virtual void mergeSuccess(OneMergePtr merge);
898
902 virtual bool registerMerge(OneMergePtr merge);
903
906 virtual void mergeInit(OneMergePtr merge);
907
909 virtual void mergeFinish(OneMergePtr merge);
910
911 virtual void addMergeException(OneMergePtr merge);
912
914 virtual int32_t getBufferedDeleteTermsSize();
915
917 virtual int32_t getNumBufferedDeleteTerms();
918
920 virtual SegmentInfoPtr newestSegment();
921
922 virtual String segString();
923
926 static bool isLocked(DirectoryPtr directory);
927
931 static void unlock(DirectoryPtr directory);
932
934 virtual void setMergedSegmentWarmer(IndexReaderWarmerPtr warmer);
935
937 virtual IndexReaderWarmerPtr getMergedSegmentWarmer();
938
952 virtual bool testPoint(const String& name);
953
954 virtual bool nrtIsCurrent(SegmentInfosPtr infos);
955 virtual bool isClosed();
956
957 protected:
958 virtual void ensureOpen(bool includePendingClose);
959 virtual void ensureOpen();
960 virtual void setMessageID(InfoStreamPtr infoStream);
961
964 virtual LogMergePolicyPtr getLogMergePolicy();
965
966 virtual void setRollbackSegmentInfos(SegmentInfosPtr infos);
967
970 virtual void pushMaxBufferedDocs();
971
972 virtual void messageState();
973
976 virtual bool shouldClose();
977 virtual void closeInternal(bool waitForMerges);
978
981 virtual bool flushDocStores();
982
984 virtual bool optimizeMergesPending();
985
986 virtual void maybeMerge(bool optimize);
987 virtual void maybeMerge(int32_t maxNumSegmentsOptimize, bool optimize);
988 virtual void updatePendingMerges(int32_t maxNumSegmentsOptimize, bool optimize);
989
991 virtual OneMergePtr getNextExternalMerge();
992
1000 virtual void startTransaction(bool haveReadLock);
1001
1003 virtual void rollbackTransaction();
1004
1007 virtual void commitTransaction();
1008 virtual void rollbackInternal();
1009
1010 virtual void finishMerges(bool waitForMerges);
1011
1014 virtual void checkpoint();
1015
1016 virtual void finishAddIndexes();
1017 virtual void blockAddIndexes(bool includePendingClose);
1018 virtual void resumeAddIndexes();
1019 virtual void resetMergeExceptions();
1020 virtual void noDupDirs(Collection<DirectoryPtr> dirs);
1021
1022 virtual bool hasExternalSegments();
1023
1028 virtual void resolveExternalSegments();
1029
1032 virtual void doAfterFlush();
1033
1036 virtual void doBeforeFlush();
1037
1038 virtual void commit(int64_t sizeInBytes);
1039 virtual void finishCommit();
1040
1045 virtual void flush(bool triggerMerge, bool flushDocStores, bool flushDeletes);
1046 virtual bool doFlush(bool flushDocStores, bool flushDeletes);
1047 virtual bool doFlushInternal(bool flushDocStores, bool flushDeletes);
1048
1049 virtual int32_t ensureContiguousMerge(OneMergePtr merge);
1050
1056 virtual void commitMergedDeletes(OneMergePtr merge, SegmentReaderPtr mergeReader);
1057 virtual bool commitMerge(OneMergePtr merge, SegmentMergerPtr merger, int32_t mergedDocCount, SegmentReaderPtr mergedReader);
1058
1059 virtual LuceneException handleMergeException(const LuceneException& exc, OneMergePtr merge);
1060
1061 virtual void _mergeInit(OneMergePtr merge);
1062
1063 virtual void setDiagnostics(SegmentInfoPtr info, const String& source);
1064 virtual void setDiagnostics(SegmentInfoPtr info, const String& source, MapStringString details);
1065
1066 virtual void setMergeDocStoreIsCompoundFile(OneMergePtr merge);
1067 virtual void closeMergeReaders(OneMergePtr merge, bool suppressExceptions);
1068
1071 virtual int32_t mergeMiddle(OneMergePtr merge);
1072
1074 virtual bool applyDeletes();
1075
1076 virtual String segString(SegmentInfosPtr infos);
1077
1078 virtual bool startSync(const String& fileName, HashSet<String> pending);
1079 virtual void finishSync(const String& fileName, bool success);
1080
1082 bool waitForAllSynced(HashSet<String> syncing);
1083 void doWait();
1084
1088 virtual void startCommit(int64_t sizeInBytes, MapStringString commitUserData);
1089
1090 virtual LuceneException handleOOM(const std::bad_alloc& oom, const String& location);
1091
1092 friend class ReaderPool;
1093 };
1094
1101 class LPPAPI IndexReaderWarmer : public LuceneObject
1102 {
1103 public:
1104 virtual ~IndexReaderWarmer();
1105
1106 LUCENE_CLASS(IndexReaderWarmer);
1107
1108 public:
1109 virtual void warm(IndexReaderPtr reader) = 0;
1110 };
1111 }
1112
1113 #endif

clucene.sourceforge.net