You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2024/06/06 13:42:39 UTC
(jackrabbit-oak) branch 1.22 updated: OAK-10786: oak-lucene: use copy of lucene-core 4.7.2 source code (#1439)
This is an automated email from the ASF dual-hosted git repository.
reschke pushed a commit to branch 1.22
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/1.22 by this push:
new d8a8999106 OAK-10786: oak-lucene: use copy of lucene-core 4.7.2 source code (#1439)
d8a8999106 is described below
commit d8a8999106dbc2b1024bd48b1592c76b020ce6b9
Author: Julian Reschke <re...@apache.org>
AuthorDate: Tue May 14 07:30:57 2024 +0200
OAK-10786: oak-lucene: use copy of lucene-core 4.7.2 source code (#1439)
* OAK-10786: oak-lucene: use copy of lucene-core 4.7.2 source code - include copy of lucene-core 4.7.2 (git@github.com:apache/lucene.git, tag releases/lucene-solr/4.7.2, path lucene/core/src/java)
* OAK-10786: oak-lucene: use copy of lucene-core 4.7.2 source code - fix two issues JDK 11 incompatibility
* OAK-10786: oak-lucene: use copy of lucene-core 4.7.2 source code - remove lucene-core dependency
* OAK-10786: add a comment about where the code originates from
* OAK-10786: bump exported lucene package version to clarify that this is Oak's fork
* OAK-10786: add copies of META-INF/services/org.apache.lucene.codecs.DocValuesFormat and META-INF/services/org.apache.lucene.codecs.PostingsFormat
* OAK-10786: tried to explain the 'oak1' modifier in the version number
---
oak-lucene/pom.xml | 7 +-
.../main/java/org/apache/lucene/LucenePackage.java | 37 +
.../java/org/apache/lucene/analysis/Analyzer.java | 466 ++
.../apache/lucene/analysis/AnalyzerWrapper.java | 127 +
.../apache/lucene/analysis/CachingTokenFilter.java | 106 +
.../org/apache/lucene/analysis/CharFilter.java | 92 +
.../apache/lucene/analysis/NumericTokenStream.java | 331 ++
.../lucene/analysis/ReusableStringReader.java | 69 +
.../java/org/apache/lucene/analysis/Token.java | 670 +++
.../org/apache/lucene/analysis/TokenFilter.java | 80 +
.../org/apache/lucene/analysis/TokenStream.java | 209 +
.../lucene/analysis/TokenStreamToAutomaton.java | 274 ++
.../java/org/apache/lucene/analysis/Tokenizer.java | 127 +
.../java/org/apache/lucene/analysis/package.html | 972 ++++
.../tokenattributes/CharTermAttribute.java | 106 +
.../tokenattributes/CharTermAttributeImpl.java | 291 ++
.../analysis/tokenattributes/FlagsAttribute.java | 53 +
.../tokenattributes/FlagsAttributeImpl.java | 75 +
.../analysis/tokenattributes/KeywordAttribute.java | 59 +
.../tokenattributes/KeywordAttributeImpl.java | 74 +
.../analysis/tokenattributes/OffsetAttribute.java | 64 +
.../tokenattributes/OffsetAttributeImpl.java | 101 +
.../analysis/tokenattributes/PayloadAttribute.java | 58 +
.../tokenattributes/PayloadAttributeImpl.java | 101 +
.../PositionIncrementAttribute.java | 70 +
.../PositionIncrementAttributeImpl.java | 80 +
.../tokenattributes/PositionLengthAttribute.java | 58 +
.../PositionLengthAttributeImpl.java | 80 +
.../tokenattributes/TermToBytesRefAttribute.java | 85 +
.../analysis/tokenattributes/TypeAttribute.java | 49 +
.../tokenattributes/TypeAttributeImpl.java | 83 +
.../lucene/analysis/tokenattributes/package.html | 25 +
.../org/apache/lucene/codecs/BlockTermState.java | 68 +
.../apache/lucene/codecs/BlockTreeTermsReader.java | 3010 ++++++++++++
.../apache/lucene/codecs/BlockTreeTermsWriter.java | 1151 +++++
.../main/java/org/apache/lucene/codecs/Codec.java | 155 +
.../java/org/apache/lucene/codecs/CodecUtil.java | 161 +
.../apache/lucene/codecs/DocValuesConsumer.java | 653 +++
.../org/apache/lucene/codecs/DocValuesFormat.java | 131 +
.../apache/lucene/codecs/DocValuesProducer.java | 134 +
.../org/apache/lucene/codecs/FieldInfosFormat.java | 49 +
.../org/apache/lucene/codecs/FieldInfosReader.java | 47 +
.../org/apache/lucene/codecs/FieldInfosWriter.java | 47 +
.../org/apache/lucene/codecs/FieldsConsumer.java | 84 +
.../org/apache/lucene/codecs/FieldsProducer.java | 50 +
.../java/org/apache/lucene/codecs/FilterCodec.java | 110 +
.../org/apache/lucene/codecs/LiveDocsFormat.java | 62 +
.../codecs/MappingMultiDocsAndPositionsEnum.java | 155 +
.../apache/lucene/codecs/MappingMultiDocsEnum.java | 135 +
.../lucene/codecs/MultiLevelSkipListReader.java | 324 ++
.../lucene/codecs/MultiLevelSkipListWriter.java | 185 +
.../java/org/apache/lucene/codecs/NormsFormat.java | 57 +
.../apache/lucene/codecs/PostingsBaseFormat.java | 63 +
.../org/apache/lucene/codecs/PostingsConsumer.java | 156 +
.../org/apache/lucene/codecs/PostingsFormat.java | 134 +
.../apache/lucene/codecs/PostingsReaderBase.java | 85 +
.../apache/lucene/codecs/PostingsWriterBase.java | 99 +
.../apache/lucene/codecs/SegmentInfoFormat.java | 51 +
.../apache/lucene/codecs/SegmentInfoReader.java | 54 +
.../apache/lucene/codecs/SegmentInfoWriter.java | 51 +
.../apache/lucene/codecs/StoredFieldsFormat.java | 51 +
.../apache/lucene/codecs/StoredFieldsReader.java | 54 +
.../apache/lucene/codecs/StoredFieldsWriter.java | 140 +
.../java/org/apache/lucene/codecs/TermStats.java | 50 +
.../apache/lucene/codecs/TermVectorsFormat.java | 51 +
.../apache/lucene/codecs/TermVectorsReader.java | 60 +
.../apache/lucene/codecs/TermVectorsWriter.java | 311 ++
.../org/apache/lucene/codecs/TermsConsumer.java | 214 +
.../compressing/CompressingStoredFieldsFormat.java | 138 +
.../CompressingStoredFieldsIndexReader.java | 191 +
.../CompressingStoredFieldsIndexWriter.java | 216 +
.../compressing/CompressingStoredFieldsReader.java | 517 +++
.../compressing/CompressingStoredFieldsWriter.java | 433 ++
.../compressing/CompressingTermVectorsFormat.java | 110 +
.../compressing/CompressingTermVectorsReader.java | 1063 +++++
.../compressing/CompressingTermVectorsWriter.java | 831 ++++
.../lucene/codecs/compressing/CompressionMode.java | 287 ++
.../lucene/codecs/compressing/Compressor.java | 47 +
.../lucene/codecs/compressing/Decompressor.java | 60 +
.../org/apache/lucene/codecs/compressing/LZ4.java | 555 +++
.../apache/lucene/codecs/compressing/package.html | 25 +
.../lucene/codecs/lucene3x/Lucene3xCodec.java | 150 +
.../codecs/lucene3x/Lucene3xFieldInfosFormat.java | 53 +
.../codecs/lucene3x/Lucene3xFieldInfosReader.java | 133 +
.../lucene/codecs/lucene3x/Lucene3xFields.java | 1095 +++++
.../codecs/lucene3x/Lucene3xNormsFormat.java | 54 +
.../codecs/lucene3x/Lucene3xNormsProducer.java | 240 +
.../codecs/lucene3x/Lucene3xPostingsFormat.java | 72 +
.../codecs/lucene3x/Lucene3xSegmentInfoFormat.java | 97 +
.../codecs/lucene3x/Lucene3xSegmentInfoReader.java | 280 ++
.../codecs/lucene3x/Lucene3xSkipListReader.java | 125 +
.../lucene3x/Lucene3xStoredFieldsFormat.java | 53 +
.../lucene3x/Lucene3xStoredFieldsReader.java | 315 ++
.../codecs/lucene3x/Lucene3xTermVectorsFormat.java | 88 +
.../codecs/lucene3x/Lucene3xTermVectorsReader.java | 755 +++
.../lucene/codecs/lucene3x/SegmentTermDocs.java | 236 +
.../lucene/codecs/lucene3x/SegmentTermEnum.java | 234 +
.../codecs/lucene3x/SegmentTermPositions.java | 229 +
.../apache/lucene/codecs/lucene3x/TermBuffer.java | 139 +
.../apache/lucene/codecs/lucene3x/TermInfo.java | 71 +
.../lucene/codecs/lucene3x/TermInfosReader.java | 367 ++
.../codecs/lucene3x/TermInfosReaderIndex.java | 274 ++
.../org/apache/lucene/codecs/lucene3x/package.html | 25 +
.../apache/lucene/codecs/lucene40/BitVector.java | 415 ++
.../lucene/codecs/lucene40/Lucene40Codec.java | 126 +
.../codecs/lucene40/Lucene40DocValuesFormat.java | 214 +
.../codecs/lucene40/Lucene40DocValuesReader.java | 665 +++
.../codecs/lucene40/Lucene40FieldInfosFormat.java | 137 +
.../codecs/lucene40/Lucene40FieldInfosReader.java | 163 +
.../codecs/lucene40/Lucene40LiveDocsFormat.java | 120 +
.../codecs/lucene40/Lucene40NormsFormat.java | 71 +
.../lucene40/Lucene40PostingsBaseFormat.java | 60 +
.../codecs/lucene40/Lucene40PostingsFormat.java | 290 ++
.../codecs/lucene40/Lucene40PostingsReader.java | 1179 +++++
.../codecs/lucene40/Lucene40SegmentInfoFormat.java | 106 +
.../codecs/lucene40/Lucene40SegmentInfoReader.java | 94 +
.../codecs/lucene40/Lucene40SegmentInfoWriter.java | 84 +
.../codecs/lucene40/Lucene40SkipListReader.java | 151 +
.../lucene40/Lucene40StoredFieldsFormat.java | 107 +
.../lucene40/Lucene40StoredFieldsReader.java | 260 ++
.../lucene40/Lucene40StoredFieldsWriter.java | 344 ++
.../codecs/lucene40/Lucene40TermVectorsFormat.java | 139 +
.../codecs/lucene40/Lucene40TermVectorsReader.java | 785 ++++
.../codecs/lucene40/Lucene40TermVectorsWriter.java | 459 ++
.../org/apache/lucene/codecs/lucene40/package.html | 383 ++
.../org/apache/lucene/codecs/lucene41/ForUtil.java | 254 ++
.../lucene/codecs/lucene41/Lucene41Codec.java | 143 +
.../lucene41/Lucene41PostingsBaseFormat.java | 59 +
.../codecs/lucene41/Lucene41PostingsFormat.java | 460 ++
.../codecs/lucene41/Lucene41PostingsReader.java | 1558 +++++++
.../codecs/lucene41/Lucene41PostingsWriter.java | 597 +++
.../lucene/codecs/lucene41/Lucene41SkipReader.java | 231 +
.../lucene/codecs/lucene41/Lucene41SkipWriter.java | 159 +
.../lucene41/Lucene41StoredFieldsFormat.java | 135 +
.../org/apache/lucene/codecs/lucene41/package.html | 392 ++
.../lucene/codecs/lucene42/Lucene42Codec.java | 157 +
.../codecs/lucene42/Lucene42DocValuesFormat.java | 179 +
.../codecs/lucene42/Lucene42DocValuesProducer.java | 599 +++
.../codecs/lucene42/Lucene42FieldInfosFormat.java | 130 +
.../codecs/lucene42/Lucene42FieldInfosReader.java | 133 +
.../codecs/lucene42/Lucene42NormsConsumer.java | 217 +
.../codecs/lucene42/Lucene42NormsFormat.java | 90 +
.../codecs/lucene42/Lucene42TermVectorsFormat.java | 138 +
.../org/apache/lucene/codecs/lucene42/package.html | 395 ++
.../lucene/codecs/lucene45/Lucene45Codec.java | 151 +
.../codecs/lucene45/Lucene45DocValuesConsumer.java | 459 ++
.../codecs/lucene45/Lucene45DocValuesFormat.java | 195 +
.../codecs/lucene45/Lucene45DocValuesProducer.java | 894 ++++
.../org/apache/lucene/codecs/lucene45/package.html | 396 ++
.../lucene/codecs/lucene46/Lucene46Codec.java | 146 +
.../codecs/lucene46/Lucene46FieldInfosFormat.java | 134 +
.../codecs/lucene46/Lucene46FieldInfosReader.java | 133 +
.../codecs/lucene46/Lucene46FieldInfosWriter.java | 117 +
.../codecs/lucene46/Lucene46SegmentInfoFormat.java | 101 +
.../codecs/lucene46/Lucene46SegmentInfoReader.java | 91 +
.../codecs/lucene46/Lucene46SegmentInfoWriter.java | 81 +
.../org/apache/lucene/codecs/lucene46/package.html | 398 ++
.../java/org/apache/lucene/codecs/package.html | 73 +
.../codecs/perfield/PerFieldDocValuesFormat.java | 335 ++
.../codecs/perfield/PerFieldPostingsFormat.java | 262 ++
.../org/apache/lucene/codecs/perfield/package.html | 25 +
.../lucene/document/BinaryDocValuesField.java | 71 +
.../apache/lucene/document/ByteDocValuesField.java | 64 +
.../apache/lucene/document/CompressionTools.java | 159 +
.../java/org/apache/lucene/document/DateTools.java | 233 +
.../lucene/document/DerefBytesDocValuesField.java | 81 +
.../java/org/apache/lucene/document/Document.java | 261 ++
.../document/DocumentStoredFieldVisitor.java | 120 +
.../lucene/document/DoubleDocValuesField.java | 63 +
.../org/apache/lucene/document/DoubleField.java | 183 +
.../java/org/apache/lucene/document/Field.java | 1045 +++++
.../java/org/apache/lucene/document/FieldType.java | 443 ++
.../lucene/document/FloatDocValuesField.java | 63 +
.../org/apache/lucene/document/FloatField.java | 183 +
.../apache/lucene/document/IntDocValuesField.java | 62 +
.../java/org/apache/lucene/document/IntField.java | 183 +
.../apache/lucene/document/LongDocValuesField.java | 57 +
.../java/org/apache/lucene/document/LongField.java | 193 +
.../lucene/document/NumericDocValuesField.java | 65 +
.../lucene/document/PackedLongDocValuesField.java | 58 +
.../lucene/document/ShortDocValuesField.java | 63 +
.../lucene/document/SortedBytesDocValuesField.java | 81 +
.../lucene/document/SortedDocValuesField.java | 68 +
.../lucene/document/SortedSetDocValuesField.java | 69 +
.../org/apache/lucene/document/StoredField.java | 139 +
.../document/StraightBytesDocValuesField.java | 81 +
.../org/apache/lucene/document/StringField.java | 70 +
.../java/org/apache/lucene/document/TextField.java | 86 +
.../java/org/apache/lucene/document/package.html | 47 +
.../java/org/apache/lucene/index/AtomicReader.java | 262 ++
.../apache/lucene/index/AtomicReaderContext.java | 77 +
.../apache/lucene/index/AutomatonTermsEnum.java | 329 ++
.../apache/lucene/index/BaseCompositeReader.java | 207 +
.../org/apache/lucene/index/BinaryDocValues.java | 51 +
.../apache/lucene/index/BinaryDocValuesWriter.java | 187 +
.../java/org/apache/lucene/index/BitsSlice.java | 61 +
.../org/apache/lucene/index/BufferedUpdates.java | 237 +
.../apache/lucene/index/BufferedUpdatesStream.java | 600 +++
.../org/apache/lucene/index/ByteSliceReader.java | 149 +
.../org/apache/lucene/index/ByteSliceWriter.java | 94 +
.../java/org/apache/lucene/index/CheckIndex.java | 1979 ++++++++
.../org/apache/lucene/index/CoalescedUpdates.java | 105 +
.../org/apache/lucene/index/CompositeReader.java | 118 +
.../lucene/index/CompositeReaderContext.java | 131 +
.../lucene/index/ConcurrentMergeScheduler.java | 585 +++
.../apache/lucene/index/CorruptIndexException.java | 39 +
.../org/apache/lucene/index/DirectoryReader.java | 459 ++
.../java/org/apache/lucene/index/DocConsumer.java | 35 +
.../org/apache/lucene/index/DocFieldConsumer.java | 44 +
.../lucene/index/DocFieldConsumerPerField.java | 35 +
.../org/apache/lucene/index/DocFieldProcessor.java | 280 ++
.../lucene/index/DocFieldProcessorPerField.java | 65 +
.../java/org/apache/lucene/index/DocInverter.java | 92 +
.../apache/lucene/index/DocInverterPerField.java | 229 +
.../java/org/apache/lucene/index/DocTermOrds.java | 919 ++++
.../apache/lucene/index/DocValuesProcessor.java | 185 +
.../org/apache/lucene/index/DocValuesWriter.java | 36 +
.../apache/lucene/index/DocsAndPositionsEnum.java | 70 +
.../java/org/apache/lucene/index/DocsEnum.java | 76 +
.../org/apache/lucene/index/DocumentsWriter.java | 744 +++
.../lucene/index/DocumentsWriterDeleteQueue.java | 447 ++
.../lucene/index/DocumentsWriterFlushControl.java | 711 +++
.../lucene/index/DocumentsWriterFlushQueue.java | 275 ++
.../lucene/index/DocumentsWriterPerThread.java | 642 +++
.../lucene/index/DocumentsWriterPerThreadPool.java | 339 ++
.../lucene/index/DocumentsWriterStallControl.java | 125 +
.../java/org/apache/lucene/index/FieldInfo.java | 346 ++
.../java/org/apache/lucene/index/FieldInfos.java | 349 ++
.../org/apache/lucene/index/FieldInvertState.java | 166 +
.../main/java/org/apache/lucene/index/Fields.java | 83 +
.../apache/lucene/index/FilterAtomicReader.java | 436 ++
.../apache/lucene/index/FilterDirectoryReader.java | 159 +
.../org/apache/lucene/index/FilteredTermsEnum.java | 267 ++
.../lucene/index/FlushByRamOrCountsPolicy.java | 143 +
.../java/org/apache/lucene/index/FlushPolicy.java | 158 +
.../apache/lucene/index/FreqProxTermsWriter.java | 131 +
.../lucene/index/FreqProxTermsWriterPerField.java | 563 +++
.../apache/lucene/index/FrozenBufferedUpdates.java | 174 +
.../java/org/apache/lucene/index/IndexCommit.java | 138 +
.../apache/lucene/index/IndexDeletionPolicy.java | 127 +
.../org/apache/lucene/index/IndexFileDeleter.java | 733 +++
.../org/apache/lucene/index/IndexFileNames.java | 216 +
.../lucene/index/IndexFormatTooNewException.java | 62 +
.../lucene/index/IndexFormatTooOldException.java | 84 +
.../lucene/index/IndexNotFoundException.java | 42 +
.../java/org/apache/lucene/index/IndexReader.java | 586 +++
.../apache/lucene/index/IndexReaderContext.java | 72 +
.../org/apache/lucene/index/IndexUpgrader.java | 184 +
.../java/org/apache/lucene/index/IndexWriter.java | 4801 ++++++++++++++++++++
.../org/apache/lucene/index/IndexWriterConfig.java | 592 +++
.../org/apache/lucene/index/IndexableField.java | 99 +
.../apache/lucene/index/IndexableFieldType.java | 108 +
.../apache/lucene/index/InvertedDocConsumer.java | 44 +
.../lucene/index/InvertedDocConsumerPerField.java | 50 +
.../lucene/index/InvertedDocEndConsumer.java | 37 +
.../index/InvertedDocEndConsumerPerField.java | 33 +
.../index/KeepOnlyLastCommitDeletionPolicy.java | 64 +
.../apache/lucene/index/LiveIndexWriterConfig.java | 618 +++
.../lucene/index/LogByteSizeMergePolicy.java | 117 +
.../org/apache/lucene/index/LogDocMergePolicy.java | 74 +
.../org/apache/lucene/index/LogMergePolicy.java | 647 +++
.../java/org/apache/lucene/index/MergePolicy.java | 603 +++
.../org/apache/lucene/index/MergeScheduler.java | 62 +
.../java/org/apache/lucene/index/MergeState.java | 220 +
.../java/org/apache/lucene/index/MultiBits.java | 125 +
.../org/apache/lucene/index/MultiDocValues.java | 527 +++
.../lucene/index/MultiDocsAndPositionsEnum.java | 201 +
.../org/apache/lucene/index/MultiDocsEnum.java | 182 +
.../java/org/apache/lucene/index/MultiFields.java | 289 ++
.../java/org/apache/lucene/index/MultiReader.java | 93 +
.../java/org/apache/lucene/index/MultiTerms.java | 197 +
.../org/apache/lucene/index/MultiTermsEnum.java | 550 +++
.../org/apache/lucene/index/NoDeletionPolicy.java | 55 +
.../org/apache/lucene/index/NoMergePolicy.java | 91 +
.../org/apache/lucene/index/NoMergeScheduler.java | 58 +
.../org/apache/lucene/index/NormsConsumer.java | 92 +
.../apache/lucene/index/NormsConsumerPerField.java | 80 +
.../org/apache/lucene/index/NumericDocValues.java | 51 +
.../lucene/index/NumericDocValuesWriter.java | 153 +
.../apache/lucene/index/NumericFieldUpdates.java | 257 ++
.../org/apache/lucene/index/NumericUpdate.java | 76 +
.../java/org/apache/lucene/index/OrdTermState.java | 52 +
.../apache/lucene/index/ParallelAtomicReader.java | 310 ++
.../lucene/index/ParallelCompositeReader.java | 192 +
.../apache/lucene/index/ParallelPostingsArray.java | 66 +
.../index/PersistentSnapshotDeletionPolicy.java | 303 ++
.../org/apache/lucene/index/PrefixCodedTerms.java | 157 +
.../org/apache/lucene/index/ReaderManager.java | 98 +
.../java/org/apache/lucene/index/ReaderSlice.java | 58 +
.../java/org/apache/lucene/index/ReaderUtil.java | 102 +
.../org/apache/lucene/index/ReadersAndUpdates.java | 584 +++
.../org/apache/lucene/index/SegmentCommitInfo.java | 275 ++
.../apache/lucene/index/SegmentCoreReaders.java | 228 +
.../org/apache/lucene/index/SegmentDocValues.java | 116 +
.../java/org/apache/lucene/index/SegmentInfo.java | 336 ++
.../java/org/apache/lucene/index/SegmentInfos.java | 1213 +++++
.../org/apache/lucene/index/SegmentMerger.java | 403 ++
.../org/apache/lucene/index/SegmentReadState.java | 102 +
.../org/apache/lucene/index/SegmentReader.java | 573 +++
.../org/apache/lucene/index/SegmentWriteState.java | 126 +
.../apache/lucene/index/SerialMergeScheduler.java | 54 +
.../lucene/index/SimpleMergedSegmentWarmer.java | 98 +
.../org/apache/lucene/index/SingleTermsEnum.java | 59 +
.../lucene/index/SingletonSortedSetDocValues.java | 84 +
.../lucene/index/SlowCompositeReaderWrapper.java | 250 +
.../lucene/index/SnapshotDeletionPolicy.java | 282 ++
.../org/apache/lucene/index/SortedDocValues.java | 134 +
.../lucene/index/SortedDocValuesTermsEnum.java | 153 +
.../apache/lucene/index/SortedDocValuesWriter.java | 220 +
.../apache/lucene/index/SortedSetDocValues.java | 136 +
.../lucene/index/SortedSetDocValuesTermsEnum.java | 153 +
.../lucene/index/SortedSetDocValuesWriter.java | 315 ++
.../lucene/index/StandardDirectoryReader.java | 464 ++
.../apache/lucene/index/StoredFieldVisitor.java | 104 +
.../apache/lucene/index/StoredFieldsConsumer.java | 36 +
.../apache/lucene/index/StoredFieldsProcessor.java | 165 +
.../main/java/org/apache/lucene/index/Term.java | 166 +
.../java/org/apache/lucene/index/TermContext.java | 179 +
.../java/org/apache/lucene/index/TermState.java | 65 +
.../apache/lucene/index/TermVectorsConsumer.java | 188 +
.../lucene/index/TermVectorsConsumerPerField.java | 316 ++
.../main/java/org/apache/lucene/index/Terms.java | 136 +
.../java/org/apache/lucene/index/TermsEnum.java | 290 ++
.../java/org/apache/lucene/index/TermsHash.java | 151 +
.../org/apache/lucene/index/TermsHashConsumer.java | 37 +
.../lucene/index/TermsHashConsumerPerField.java | 45 +
.../org/apache/lucene/index/TermsHashPerField.java | 336 ++
.../ThreadAffinityDocumentsWriterThreadPool.java | 92 +
.../org/apache/lucene/index/TieredMergePolicy.java | 651 +++
.../apache/lucene/index/TrackingIndexWriter.java | 213 +
.../org/apache/lucene/index/TwoPhaseCommit.java | 63 +
.../apache/lucene/index/TwoPhaseCommitTool.java | 136 +
.../lucene/index/TwoStoredFieldsConsumers.java | 76 +
.../lucene/index/UpgradeIndexMergePolicy.java | 169 +
.../main/java/org/apache/lucene/index/package.html | 264 ++
.../src/main/java/org/apache/lucene/package.html | 17 +
.../org/apache/lucene/search/AutomatonQuery.java | 131 +
.../apache/lucene/search/BitsFilteredDocIdSet.java | 71 +
.../org/apache/lucene/search/BooleanClause.java | 115 +
.../org/apache/lucene/search/BooleanQuery.java | 532 +++
.../org/apache/lucene/search/BooleanScorer.java | 366 ++
.../org/apache/lucene/search/BooleanScorer2.java | 358 ++
.../org/apache/lucene/search/BoostAttribute.java | 47 +
.../apache/lucene/search/BoostAttributeImpl.java | 55 +
.../org/apache/lucene/search/CachingCollector.java | 476 ++
.../apache/lucene/search/CachingWrapperFilter.java | 176 +
.../apache/lucene/search/CollectionStatistics.java | 84 +
.../search/CollectionTerminatedException.java | 42 +
.../java/org/apache/lucene/search/Collector.java | 187 +
.../apache/lucene/search/ComplexExplanation.java | 79 +
.../apache/lucene/search/ConjunctionScorer.java | 150 +
.../lucene/search/ConstantScoreAutoRewrite.java | 228 +
.../apache/lucene/search/ConstantScoreQuery.java | 321 ++
.../search/ControlledRealTimeReopenThread.java | 259 ++
.../apache/lucene/search/DisjunctionMaxQuery.java | 311 ++
.../apache/lucene/search/DisjunctionMaxScorer.java | 96 +
.../apache/lucene/search/DisjunctionScorer.java | 183 +
.../apache/lucene/search/DisjunctionSumScorer.java | 92 +
.../java/org/apache/lucene/search/DocIdSet.java | 76 +
.../org/apache/lucene/search/DocIdSetIterator.java | 155 +
.../lucene/search/DocTermOrdsRangeFilter.java | 175 +
.../lucene/search/DocTermOrdsRewriteMethod.java | 194 +
.../apache/lucene/search/ExactPhraseScorer.java | 335 ++
.../java/org/apache/lucene/search/Explanation.java | 136 +
.../java/org/apache/lucene/search/FieldCache.java | 822 ++++
.../apache/lucene/search/FieldCacheDocIdSet.java | 175 +
.../org/apache/lucene/search/FieldCacheImpl.java | 1459 ++++++
.../lucene/search/FieldCacheRangeFilter.java | 580 +++
.../lucene/search/FieldCacheRewriteMethod.java | 193 +
.../lucene/search/FieldCacheTermsFilter.java | 151 +
.../org/apache/lucene/search/FieldComparator.java | 1313 ++++++
.../lucene/search/FieldComparatorSource.java | 49 +
.../java/org/apache/lucene/search/FieldDoc.java | 88 +
.../org/apache/lucene/search/FieldValueFilter.java | 150 +
.../apache/lucene/search/FieldValueHitQueue.java | 232 +
.../main/java/org/apache/lucene/search/Filter.java | 68 +
.../org/apache/lucene/search/FilteredDocIdSet.java | 108 +
.../lucene/search/FilteredDocIdSetIterator.java | 97 +
.../org/apache/lucene/search/FilteredQuery.java | 619 +++
.../java/org/apache/lucene/search/FuzzyQuery.java | 250 +
.../org/apache/lucene/search/FuzzyTermsEnum.java | 455 ++
.../java/org/apache/lucene/search/HitQueue.java | 90 +
.../org/apache/lucene/search/IndexSearcher.java | 964 ++++
.../org/apache/lucene/search/LiveFieldValues.java | 141 +
.../apache/lucene/search/MatchAllDocsQuery.java | 173 +
.../search/MaxNonCompetitiveBoostAttribute.java | 54 +
.../MaxNonCompetitiveBoostAttributeImpl.java | 70 +
.../lucene/search/MinShouldMatchSumScorer.java | 413 ++
.../org/apache/lucene/search/MultiCollector.java | 132 +
.../org/apache/lucene/search/MultiPhraseQuery.java | 593 +++
.../org/apache/lucene/search/MultiTermQuery.java | 341 ++
.../lucene/search/MultiTermQueryWrapperFilter.java | 130 +
.../org/apache/lucene/search/NGramPhraseQuery.java | 107 +
.../apache/lucene/search/NumericRangeFilter.java | 205 +
.../apache/lucene/search/NumericRangeQuery.java | 541 +++
.../org/apache/lucene/search/PhrasePositions.java | 97 +
.../java/org/apache/lucene/search/PhraseQuery.java | 414 ++
.../java/org/apache/lucene/search/PhraseQueue.java | 53 +
.../lucene/search/PositiveScoresOnlyCollector.java | 71 +
.../org/apache/lucene/search/PrefixFilter.java | 55 +
.../java/org/apache/lucene/search/PrefixQuery.java | 104 +
.../org/apache/lucene/search/PrefixTermsEnum.java | 57 +
.../main/java/org/apache/lucene/search/Query.java | 141 +
.../apache/lucene/search/QueryWrapperFilter.java | 90 +
.../org/apache/lucene/search/ReferenceManager.java | 330 ++
.../java/org/apache/lucene/search/RegexpQuery.java | 115 +
.../org/apache/lucene/search/ReqExclScorer.java | 144 +
.../org/apache/lucene/search/ReqOptSumScorer.java | 116 +
.../lucene/search/ScoreCachingWrappingScorer.java | 105 +
.../java/org/apache/lucene/search/ScoreDoc.java | 59 +
.../main/java/org/apache/lucene/search/Scorer.java | 146 +
.../org/apache/lucene/search/ScoringRewrite.java | 202 +
.../org/apache/lucene/search/SearcherFactory.java | 65 +
.../lucene/search/SearcherLifetimeManager.java | 311 ++
.../org/apache/lucene/search/SearcherManager.java | 180 +
.../apache/lucene/search/SloppyPhraseScorer.java | 613 +++
.../main/java/org/apache/lucene/search/Sort.java | 223 +
.../java/org/apache/lucene/search/SortField.java | 473 ++
.../lucene/search/TermCollectingRewrite.java | 111 +
.../java/org/apache/lucene/search/TermQuery.java | 221 +
.../org/apache/lucene/search/TermRangeFilter.java | 96 +
.../org/apache/lucene/search/TermRangeQuery.java | 179 +
.../apache/lucene/search/TermRangeTermsEnum.java | 116 +
.../java/org/apache/lucene/search/TermScorer.java | 104 +
.../org/apache/lucene/search/TermStatistics.java | 63 +
.../lucene/search/TimeLimitingCollector.java | 323 ++
.../java/org/apache/lucene/search/TopDocs.java | 272 ++
.../org/apache/lucene/search/TopDocsCollector.java | 169 +
.../apache/lucene/search/TopFieldCollector.java | 1237 +++++
.../org/apache/lucene/search/TopFieldDocs.java | 47 +
.../apache/lucene/search/TopScoreDocCollector.java | 319 ++
.../org/apache/lucene/search/TopTermsRewrite.java | 220 +
.../lucene/search/TotalHitCountCollector.java | 59 +
.../main/java/org/apache/lucene/search/Weight.java | 136 +
.../org/apache/lucene/search/WildcardQuery.java | 124 +
.../lucene/search/doc-files/nrq-formula-1.png | Bin 0 -> 3171 bytes
.../lucene/search/doc-files/nrq-formula-2.png | Bin 0 -> 3694 bytes
.../java/org/apache/lucene/search/package.html | 561 +++
.../search/payloads/AveragePayloadFunction.java | 65 +
.../lucene/search/payloads/MaxPayloadFunction.java | 68 +
.../lucene/search/payloads/MinPayloadFunction.java | 67 +
.../lucene/search/payloads/PayloadFunction.java | 79 +
.../lucene/search/payloads/PayloadNearQuery.java | 279 ++
.../lucene/search/payloads/PayloadSpanUtil.java | 207 +
.../lucene/search/payloads/PayloadTermQuery.java | 256 ++
.../org/apache/lucene/search/payloads/package.html | 32 +
.../lucene/search/similarities/AfterEffect.java | 82 +
.../lucene/search/similarities/AfterEffectB.java | 61 +
.../lucene/search/similarities/AfterEffectL.java | 57 +
.../lucene/search/similarities/BM25Similarity.java | 353 ++
.../lucene/search/similarities/BasicModel.java | 75 +
.../lucene/search/similarities/BasicModelBE.java | 63 +
.../lucene/search/similarities/BasicModelD.java | 64 +
.../lucene/search/similarities/BasicModelG.java | 55 +
.../lucene/search/similarities/BasicModelIF.java | 50 +
.../lucene/search/similarities/BasicModelIn.java | 64 +
.../lucene/search/similarities/BasicModelIne.java | 52 +
.../lucene/search/similarities/BasicModelP.java | 57 +
.../lucene/search/similarities/BasicStats.java | 154 +
.../lucene/search/similarities/DFRSimilarity.java | 162 +
.../search/similarities/DefaultSimilarity.java | 181 +
.../lucene/search/similarities/Distribution.java | 60 +
.../lucene/search/similarities/DistributionLL.java | 49 +
.../search/similarities/DistributionSPL.java | 54 +
.../lucene/search/similarities/IBSimilarity.java | 160 +
.../search/similarities/LMDirichletSimilarity.java | 107 +
.../similarities/LMJelinekMercerSimilarity.java | 87 +
.../lucene/search/similarities/LMSimilarity.java | 170 +
.../apache/lucene/search/similarities/Lambda.java | 57 +
.../lucene/search/similarities/LambdaDF.java | 60 +
.../lucene/search/similarities/LambdaTTF.java | 60 +
.../search/similarities/MultiSimilarity.java | 135 +
.../lucene/search/similarities/Normalization.java | 94 +
.../search/similarities/NormalizationH1.java | 74 +
.../search/similarities/NormalizationH2.java | 76 +
.../search/similarities/NormalizationH3.java | 67 +
.../lucene/search/similarities/NormalizationZ.java | 68 +
.../similarities/PerFieldSimilarityWrapper.java | 89 +
.../lucene/search/similarities/Similarity.java | 258 ++
.../lucene/search/similarities/SimilarityBase.java | 307 ++
.../search/similarities/TFIDFSimilarity.java | 830 ++++
.../apache/lucene/search/similarities/package.html | 151 +
.../lucene/search/spans/FieldMaskingSpanQuery.java | 167 +
.../lucene/search/spans/NearSpansOrdered.java | 370 ++
.../lucene/search/spans/NearSpansUnordered.java | 344 ++
.../apache/lucene/search/spans/SpanFirstQuery.java | 98 +
.../search/spans/SpanMultiTermQueryWrapper.java | 264 ++
.../search/spans/SpanNearPayloadCheckQuery.java | 125 +
.../apache/lucene/search/spans/SpanNearQuery.java | 203 +
.../apache/lucene/search/spans/SpanNotQuery.java | 259 ++
.../apache/lucene/search/spans/SpanOrQuery.java | 279 ++
.../lucene/search/spans/SpanPayloadCheckQuery.java | 127 +
.../search/spans/SpanPositionCheckQuery.java | 202 +
.../search/spans/SpanPositionRangeQuery.java | 113 +
.../org/apache/lucene/search/spans/SpanQuery.java | 57 +
.../org/apache/lucene/search/spans/SpanScorer.java | 119 +
.../apache/lucene/search/spans/SpanTermQuery.java | 140 +
.../org/apache/lucene/search/spans/SpanWeight.java | 121 +
.../java/org/apache/lucene/search/spans/Spans.java | 107 +
.../org/apache/lucene/search/spans/TermSpans.java | 191 +
.../org/apache/lucene/search/spans/package.html | 93 +
.../lucene/store/AlreadyClosedException.java | 36 +
.../org/apache/lucene/store/BaseDirectory.java | 77 +
.../apache/lucene/store/BufferedIndexInput.java | 360 ++
.../apache/lucene/store/BufferedIndexOutput.java | 158 +
.../apache/lucene/store/ByteArrayDataInput.java | 175 +
.../apache/lucene/store/ByteArrayDataOutput.java | 80 +
.../apache/lucene/store/ByteBufferIndexInput.java | 323 ++
.../apache/lucene/store/ChecksumIndexInput.java | 85 +
.../apache/lucene/store/ChecksumIndexOutput.java | 91 +
.../apache/lucene/store/CompoundFileDirectory.java | 386 ++
.../apache/lucene/store/CompoundFileWriter.java | 370 ++
.../java/org/apache/lucene/store/DataInput.java | 245 +
.../java/org/apache/lucene/store/DataOutput.java | 308 ++
.../java/org/apache/lucene/store/Directory.java | 335 ++
.../java/org/apache/lucene/store/FSDirectory.java | 540 +++
.../org/apache/lucene/store/FSLockFactory.java | 66 +
.../apache/lucene/store/FileSwitchDirectory.java | 188 +
.../org/apache/lucene/store/FilterDirectory.java | 126 +
.../java/org/apache/lucene/store/FlushInfo.java | 85 +
.../java/org/apache/lucene/store/IOContext.java | 152 +
.../java/org/apache/lucene/store/IndexInput.java | 93 +
.../java/org/apache/lucene/store/IndexOutput.java | 77 +
.../apache/lucene/store/InputStreamDataInput.java | 66 +
.../main/java/org/apache/lucene/store/Lock.java | 157 +
.../java/org/apache/lucene/store/LockFactory.java | 82 +
.../lucene/store/LockObtainFailedException.java | 41 +
.../lucene/store/LockReleaseFailedException.java | 39 +
.../org/apache/lucene/store/LockStressTest.java | 118 +
.../org/apache/lucene/store/LockVerifyServer.java | 104 +
.../org/apache/lucene/store/MMapDirectory.java | 297 ++
.../java/org/apache/lucene/store/MergeInfo.java | 97 +
.../org/apache/lucene/store/NIOFSDirectory.java | 195 +
.../apache/lucene/store/NRTCachingDirectory.java | 326 ++
.../apache/lucene/store/NativeFSLockFactory.java | 339 ++
.../org/apache/lucene/store/NoLockFactory.java | 78 +
.../lucene/store/NoSuchDirectoryException.java | 39 +
.../lucene/store/OutputStreamDataOutput.java | 54 +
.../java/org/apache/lucene/store/RAMDirectory.java | 218 +
.../main/java/org/apache/lucene/store/RAMFile.java | 90 +
.../org/apache/lucene/store/RAMInputStream.java | 129 +
.../org/apache/lucene/store/RAMOutputStream.java | 190 +
.../lucene/store/RateLimitedDirectoryWrapper.java | 160 +
.../lucene/store/RateLimitedIndexOutput.java | 91 +
.../java/org/apache/lucene/store/RateLimiter.java | 132 +
.../org/apache/lucene/store/SimpleFSDirectory.java | 152 +
.../apache/lucene/store/SimpleFSLockFactory.java | 154 +
.../lucene/store/SingleInstanceLockFactory.java | 99 +
.../lucene/store/TrackingDirectoryWrapper.java | 72 +
.../apache/lucene/store/VerifyingLockFactory.java | 131 +
.../main/java/org/apache/lucene/store/package.html | 25 +
.../lucene/util/ArrayInPlaceMergeSorter.java | 55 +
.../org/apache/lucene/util/ArrayIntroSorter.java | 67 +
.../org/apache/lucene/util/ArrayTimSorter.java | 84 +
.../java/org/apache/lucene/util/ArrayUtil.java | 728 +++
.../java/org/apache/lucene/util/Attribute.java | 32 +
.../java/org/apache/lucene/util/AttributeImpl.java | 144 +
.../org/apache/lucene/util/AttributeReflector.java | 42 +
.../org/apache/lucene/util/AttributeSource.java | 521 +++
.../main/java/org/apache/lucene/util/BitUtil.java | 198 +
.../src/main/java/org/apache/lucene/util/Bits.java | 89 +
.../java/org/apache/lucene/util/BroadWord.java | 160 +
.../java/org/apache/lucene/util/ByteBlockPool.java | 367 ++
.../main/java/org/apache/lucene/util/BytesRef.java | 383 ++
.../java/org/apache/lucene/util/BytesRefHash.java | 631 +++
.../org/apache/lucene/util/BytesRefIterator.java | 70 +
.../main/java/org/apache/lucene/util/CharsRef.java | 336 ++
.../apache/lucene/util/CloseableThreadLocal.java | 152 +
.../org/apache/lucene/util/CollectionUtil.java | 186 +
.../org/apache/lucene/util/CommandLineUtil.java | 120 +
.../java/org/apache/lucene/util/Constants.java | 164 +
.../main/java/org/apache/lucene/util/Counter.java | 101 +
.../java/org/apache/lucene/util/DocIdBitSet.java | 113 +
.../apache/lucene/util/DoubleBarrelLRUCache.java | 130 +
.../lucene/util/FieldCacheSanityChecker.java | 450 ++
.../org/apache/lucene/util/FilterIterator.java | 83 +
.../java/org/apache/lucene/util/FixedBitSet.java | 617 +++
.../lucene/util/GrowableByteArrayDataOutput.java | 63 +
.../main/java/org/apache/lucene/util/IOUtils.java | 388 ++
.../org/apache/lucene/util/InPlaceMergeSorter.java | 54 +
.../lucene/util/IndexableBinaryStringTools.java | 249 +
.../java/org/apache/lucene/util/InfoStream.java | 95 +
.../java/org/apache/lucene/util/IntBlockPool.java | 388 ++
.../java/org/apache/lucene/util/IntroSorter.java | 106 +
.../main/java/org/apache/lucene/util/IntsRef.java | 220 +
.../java/org/apache/lucene/util/LongBitSet.java | 382 ++
.../java/org/apache/lucene/util/LongValues.java | 46 +
.../main/java/org/apache/lucene/util/LongsRef.java | 219 +
.../java/org/apache/lucene/util/MapOfSets.java | 90 +
.../main/java/org/apache/lucene/util/MathUtil.java | 161 +
.../org/apache/lucene/util/MergedIterator.java | 168 +
.../java/org/apache/lucene/util/MutableBits.java | 39 +
.../org/apache/lucene/util/NamedSPILoader.java | 140 +
.../org/apache/lucene/util/NamedThreadFactory.java | 77 +
.../java/org/apache/lucene/util/NumericUtils.java | 506 +++
.../java/org/apache/lucene/util/OpenBitSet.java | 908 ++++
.../org/apache/lucene/util/OpenBitSetDISI.java | 112 +
.../org/apache/lucene/util/OpenBitSetIterator.java | 156 +
.../org/apache/lucene/util/PForDeltaDocIdSet.java | 527 +++
.../java/org/apache/lucene/util/PagedBytes.java | 422 ++
.../apache/lucene/util/PrintStreamInfoStream.java | 75 +
.../java/org/apache/lucene/util/PriorityQueue.java | 275 ++
.../java/org/apache/lucene/util/QueryBuilder.java | 425 ++
.../org/apache/lucene/util/RamUsageEstimator.java | 852 ++++
.../lucene/util/RecyclingByteBlockAllocator.java | 165 +
.../lucene/util/RecyclingIntBlockAllocator.java | 165 +
.../main/java/org/apache/lucene/util/RefCount.java | 92 +
.../java/org/apache/lucene/util/RollingBuffer.java | 144 +
.../org/apache/lucene/util/SPIClassIterator.java | 157 +
.../org/apache/lucene/util/SentinelIntSet.java | 156 +
.../main/java/org/apache/lucene/util/SetOnce.java | 91 +
.../java/org/apache/lucene/util/SloppyMath.java | 275 ++
.../java/org/apache/lucene/util/SmallFloat.java | 135 +
.../main/java/org/apache/lucene/util/Sorter.java | 267 ++
.../java/org/apache/lucene/util/StringHelper.java | 155 +
.../lucene/util/ThreadInterruptedException.java | 38 +
.../java/org/apache/lucene/util/TimSorter.java | 381 ++
.../java/org/apache/lucene/util/ToStringUtils.java | 64 +
.../java/org/apache/lucene/util/UnicodeUtil.java | 656 +++
.../main/java/org/apache/lucene/util/Version.java | 172 +
.../java/org/apache/lucene/util/VirtualMethod.java | 157 +
.../java/org/apache/lucene/util/WAH8DocIdSet.java | 755 +++
.../org/apache/lucene/util/WeakIdentityMap.java | 281 ++
.../apache/lucene/util/automaton/Automaton.java | 785 ++++
.../lucene/util/automaton/AutomatonProvider.java | 58 +
.../lucene/util/automaton/BasicAutomata.java | 286 ++
.../lucene/util/automaton/BasicOperations.java | 861 ++++
.../lucene/util/automaton/ByteRunAutomaton.java | 54 +
.../util/automaton/CharacterRunAutomaton.java | 62 +
.../lucene/util/automaton/CompiledAutomaton.java | 403 ++
.../automaton/DaciukMihovAutomatonBuilder.java | 340 ++
.../util/automaton/Lev1ParametricDescription.java | 125 +
.../util/automaton/Lev1TParametricDescription.java | 127 +
.../util/automaton/Lev2ParametricDescription.java | 225 +
.../util/automaton/Lev2TParametricDescription.java | 272 ++
.../lucene/util/automaton/LevenshteinAutomata.java | 292 ++
.../util/automaton/MinimizationOperations.java | 268 ++
.../org/apache/lucene/util/automaton/RegExp.java | 1012 +++++
.../apache/lucene/util/automaton/RunAutomaton.java | 204 +
.../apache/lucene/util/automaton/SortedIntSet.java | 286 ++
.../lucene/util/automaton/SpecialOperations.java | 276 ++
.../org/apache/lucene/util/automaton/State.java | 293 ++
.../apache/lucene/util/automaton/StatePair.java | 109 +
.../apache/lucene/util/automaton/Transition.java | 222 +
.../apache/lucene/util/automaton/UTF32ToUTF8.java | 324 ++
.../apache/lucene/util/automaton/UTF32ToUTF8.py | 366 ++
.../lucene/util/automaton/createLevAutomata.py | 500 ++
.../org/apache/lucene/util/automaton/package.html | 47 +
.../java/org/apache/lucene/util/fst/Builder.java | 649 +++
.../lucene/util/fst/ByteSequenceOutputs.java | 148 +
.../apache/lucene/util/fst/BytesRefFSTEnum.java | 134 +
.../org/apache/lucene/util/fst/BytesStore.java | 476 ++
.../lucene/util/fst/CharSequenceOutputs.java | 153 +
.../main/java/org/apache/lucene/util/fst/FST.java | 1872 ++++++++
.../java/org/apache/lucene/util/fst/FSTEnum.java | 537 +++
.../apache/lucene/util/fst/ForwardBytesReader.java | 70 +
.../apache/lucene/util/fst/IntSequenceOutputs.java | 152 +
.../org/apache/lucene/util/fst/IntsRefFSTEnum.java | 134 +
.../java/org/apache/lucene/util/fst/NoOutputs.java | 112 +
.../java/org/apache/lucene/util/fst/NodeHash.java | 181 +
.../java/org/apache/lucene/util/fst/Outputs.java | 93 +
.../org/apache/lucene/util/fst/PairOutputs.java | 175 +
.../apache/lucene/util/fst/PositiveIntOutputs.java | 130 +
.../apache/lucene/util/fst/ReverseBytesReader.java | 69 +
.../main/java/org/apache/lucene/util/fst/Util.java | 960 ++++
.../java/org/apache/lucene/util/fst/package.html | 92 +
.../apache/lucene/util/mutable/MutableValue.java | 73 +
.../lucene/util/mutable/MutableValueBool.java | 72 +
.../lucene/util/mutable/MutableValueDate.java | 46 +
.../lucene/util/mutable/MutableValueDouble.java | 75 +
.../lucene/util/mutable/MutableValueFloat.java | 73 +
.../lucene/util/mutable/MutableValueInt.java | 78 +
.../lucene/util/mutable/MutableValueLong.java | 75 +
.../lucene/util/mutable/MutableValueStr.java | 76 +
.../org/apache/lucene/util/mutable/package.html | 25 +
.../main/java/org/apache/lucene/util/package.html | 25 +
.../util/packed/AbstractAppendingLongBuffer.java | 223 +
.../util/packed/AbstractBlockPackedWriter.java | 145 +
.../lucene/util/packed/AbstractPagedMutable.java | 172 +
.../packed/AppendingDeltaPackedLongBuffer.java | 144 +
.../util/packed/AppendingPackedLongBuffer.java | 104 +
.../lucene/util/packed/BlockPackedReader.java | 105 +
.../util/packed/BlockPackedReaderIterator.java | 251 +
.../lucene/util/packed/BlockPackedWriter.java | 113 +
.../apache/lucene/util/packed/BulkOperation.java | 189 +
.../lucene/util/packed/BulkOperationPacked.java | 281 ++
.../lucene/util/packed/BulkOperationPacked1.java | 89 +
.../lucene/util/packed/BulkOperationPacked10.java | 155 +
.../lucene/util/packed/BulkOperationPacked11.java | 251 +
.../lucene/util/packed/BulkOperationPacked12.java | 111 +
.../lucene/util/packed/BulkOperationPacked13.java | 259 ++
.../lucene/util/packed/BulkOperationPacked14.java | 163 +
.../lucene/util/packed/BulkOperationPacked15.java | 267 ++
.../lucene/util/packed/BulkOperationPacked16.java | 73 +
.../lucene/util/packed/BulkOperationPacked17.java | 275 ++
.../lucene/util/packed/BulkOperationPacked18.java | 171 +
.../lucene/util/packed/BulkOperationPacked19.java | 283 ++
.../lucene/util/packed/BulkOperationPacked2.java | 81 +
.../lucene/util/packed/BulkOperationPacked20.java | 119 +
.../lucene/util/packed/BulkOperationPacked21.java | 291 ++
.../lucene/util/packed/BulkOperationPacked22.java | 179 +
.../lucene/util/packed/BulkOperationPacked23.java | 299 ++
.../lucene/util/packed/BulkOperationPacked24.java | 93 +
.../lucene/util/packed/BulkOperationPacked3.java | 219 +
.../lucene/util/packed/BulkOperationPacked4.java | 77 +
.../lucene/util/packed/BulkOperationPacked5.java | 227 +
.../lucene/util/packed/BulkOperationPacked6.java | 147 +
.../lucene/util/packed/BulkOperationPacked7.java | 235 +
.../lucene/util/packed/BulkOperationPacked8.java | 73 +
.../lucene/util/packed/BulkOperationPacked9.java | 243 +
.../packed/BulkOperationPackedSingleBlock.java | 191 +
.../org/apache/lucene/util/packed/Direct16.java | 125 +
.../org/apache/lucene/util/packed/Direct32.java | 125 +
.../org/apache/lucene/util/packed/Direct64.java | 115 +
.../org/apache/lucene/util/packed/Direct8.java | 123 +
.../packed/DirectPacked64SingleBlockReader.java | 67 +
.../lucene/util/packed/DirectPackedReader.java | 109 +
.../lucene/util/packed/EliasFanoDecoder.java | 489 ++
.../lucene/util/packed/EliasFanoDocIdSet.java | 138 +
.../lucene/util/packed/EliasFanoEncoder.java | 362 ++
.../apache/lucene/util/packed/GrowableWriter.java | 162 +
.../util/packed/MonotonicAppendingLongBuffer.java | 168 +
.../util/packed/MonotonicBlockPackedReader.java | 108 +
.../util/packed/MonotonicBlockPackedWriter.java | 104 +
.../lucene/util/packed/Packed16ThreeBlocks.java | 139 +
.../org/apache/lucene/util/packed/Packed64.java | 327 ++
.../lucene/util/packed/Packed64SingleBlock.java | 597 +++
.../lucene/util/packed/Packed8ThreeBlocks.java | 137 +
.../apache/lucene/util/packed/PackedDataInput.java | 80 +
.../lucene/util/packed/PackedDataOutput.java | 82 +
.../org/apache/lucene/util/packed/PackedInts.java | 1308 ++++++
.../lucene/util/packed/PackedReaderIterator.java | 103 +
.../apache/lucene/util/packed/PackedWriter.java | 103 +
.../lucene/util/packed/PagedGrowableWriter.java | 79 +
.../apache/lucene/util/packed/PagedMutable.java | 79 +
.../apache/lucene/util/packed/gen_BulkOperation.py | 331 ++
.../org/apache/lucene/util/packed/gen_Direct.py | 186 +
.../lucene/util/packed/gen_Packed64SingleBlock.py | 296 ++
.../lucene/util/packed/gen_PackedThreeBlocks.py | 168 +
.../org/apache/lucene/util/packed/package.html | 92 +
.../org.apache.lucene.codecs.DocValuesFormat | 17 +
.../org.apache.lucene.codecs.PostingsFormat | 17 +
742 files changed, 169136 insertions(+), 6 deletions(-)
diff --git a/oak-lucene/pom.xml b/oak-lucene/pom.xml
index c5fc124cc3..c15a394bf1 100644
--- a/oak-lucene/pom.xml
+++ b/oak-lucene/pom.xml
@@ -107,7 +107,7 @@
<_exportcontents>
!org.apache.lucene.queryparser.xml,
!org.apache.lucene.queryparser.xml.builders,
- org.apache.lucene.*;version=${lucene.version}
+ org.apache.lucene.*;version=4.7.2-oak1<!-- first Oak modification of original lucence-core 4.7.2 source code, see OAK-10786 for further information -->
</_exportcontents>
<Export-Package>
org.apache.jackrabbit.oak.plugins.index.lucene,
@@ -221,11 +221,6 @@
</dependency>
<!-- Lucene dependencies -->
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${lucene.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
diff --git a/oak-lucene/src/main/java/org/apache/lucene/LucenePackage.java b/oak-lucene/src/main/java/org/apache/lucene/LucenePackage.java
new file mode 100644
index 0000000000..01039d0425
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/LucenePackage.java
@@ -0,0 +1,37 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Lucene's package information, including version. **/
+public final class LucenePackage {
+
+ private LucenePackage() {} // can't construct
+
+ /** Return Lucene's package, including version information. */
+ public static Package get() {
+ return LucenePackage.class.getPackage();
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/Analyzer.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/Analyzer.java
new file mode 100644
index 0000000000..db05f3ed9f
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/Analyzer.java
@@ -0,0 +1,466 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.CloseableThreadLocal;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An Analyzer builds TokenStreams, which analyze text. It thus represents a
+ * policy for extracting index terms from text.
+ * <p>
+ * In order to define what analysis is done, subclasses must define their
+ * {@link TokenStreamComponents TokenStreamComponents} in {@link #createComponents(String, Reader)}.
+ * The components are then reused in each call to {@link #tokenStream(String, Reader)}.
+ * <p>
+ * Simple example:
+ * <pre class="prettyprint">
+ * Analyzer analyzer = new Analyzer() {
+ * {@literal @Override}
+ * protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ * Tokenizer source = new FooTokenizer(reader);
+ * TokenStream filter = new FooFilter(source);
+ * filter = new BarFilter(filter);
+ * return new TokenStreamComponents(source, filter);
+ * }
+ * };
+ * </pre>
+ * For more examples, see the {@link org.apache.lucene.analysis Analysis package documentation}.
+ * <p>
+ * For some concrete implementations bundled with Lucene, look in the analysis modules:
+ * <ul>
+ * <li><a href="{@docRoot}/../analyzers-common/overview-summary.html">Common</a>:
+ * Analyzers for indexing content in different languages and domains.
+ * <li><a href="{@docRoot}/../analyzers-icu/overview-summary.html">ICU</a>:
+ * Exposes functionality from ICU to Apache Lucene.
+ * <li><a href="{@docRoot}/../analyzers-kuromoji/overview-summary.html">Kuromoji</a>:
+ * Morphological analyzer for Japanese text.
+ * <li><a href="{@docRoot}/../analyzers-morfologik/overview-summary.html">Morfologik</a>:
+ * Dictionary-driven lemmatization for the Polish language.
+ * <li><a href="{@docRoot}/../analyzers-phonetic/overview-summary.html">Phonetic</a>:
+ * Analysis for indexing phonetic signatures (for sounds-alike search).
+ * <li><a href="{@docRoot}/../analyzers-smartcn/overview-summary.html">Smart Chinese</a>:
+ * Analyzer for Simplified Chinese, which indexes words.
+ * <li><a href="{@docRoot}/../analyzers-stempel/overview-summary.html">Stempel</a>:
+ * Algorithmic Stemmer for the Polish Language.
+ * <li><a href="{@docRoot}/../analyzers-uima/overview-summary.html">UIMA</a>:
+ * Analysis integration with Apache UIMA.
+ * </ul>
+ */
+public abstract class Analyzer implements Closeable {
+
+ private final ReuseStrategy reuseStrategy;
+
+ // non final as it gets nulled if closed; pkg private for access by ReuseStrategy's final helper methods:
+ CloseableThreadLocal<Object> storedValue = new CloseableThreadLocal<Object>();
+
+ /**
+ * Create a new Analyzer, reusing the same set of components per-thread
+ * across calls to {@link #tokenStream(String, Reader)}.
+ */
+ public Analyzer() {
+ this(GLOBAL_REUSE_STRATEGY);
+ }
+
+ /**
+ * Expert: create a new Analyzer with a custom {@link ReuseStrategy}.
+ * <p>
+ * NOTE: if you just want to reuse on a per-field basis, its easier to
+ * use a subclass of {@link AnalyzerWrapper} such as
+ * <a href="{@docRoot}/../analyzers-common/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.html">
+ * PerFieldAnalyerWrapper</a> instead.
+ */
+ public Analyzer(ReuseStrategy reuseStrategy) {
+ this.reuseStrategy = reuseStrategy;
+ }
+
+ /**
+ * Creates a new {@link TokenStreamComponents} instance for this analyzer.
+ *
+ * @param fieldName
+ * the name of the fields content passed to the
+ * {@link TokenStreamComponents} sink as a reader
+ * @param reader
+ * the reader passed to the {@link Tokenizer} constructor
+ * @return the {@link TokenStreamComponents} for this analyzer.
+ */
+ protected abstract TokenStreamComponents createComponents(String fieldName,
+ Reader reader);
+
+ /**
+ * Returns a TokenStream suitable for <code>fieldName</code>, tokenizing
+ * the contents of <code>reader</code>.
+ * <p>
+ * This method uses {@link #createComponents(String, Reader)} to obtain an
+ * instance of {@link TokenStreamComponents}. It returns the sink of the
+ * components and stores the components internally. Subsequent calls to this
+ * method will reuse the previously stored components after resetting them
+ * through {@link TokenStreamComponents#setReader(Reader)}.
+ * <p>
+ * <b>NOTE:</b> After calling this method, the consumer must follow the
+ * workflow described in {@link TokenStream} to properly consume its contents.
+ * See the {@link org.apache.lucene.analysis Analysis package documentation} for
+ * some examples demonstrating this.
+ *
+ * <b>NOTE:</b> If your data is available as a {@code String}, use
+ * {@link #tokenStream(String, String)} which reuses a {@code StringReader}-like
+ * instance internally.
+ *
+ * @param fieldName the name of the field the created TokenStream is used for
+ * @param reader the reader the streams source reads from
+ * @return TokenStream for iterating the analyzed content of <code>reader</code>
+ * @throws AlreadyClosedException if the Analyzer is closed.
+ * @throws IOException if an i/o error occurs.
+ * @see #tokenStream(String, String)
+ */
+ public final TokenStream tokenStream(final String fieldName,
+ final Reader reader) throws IOException {
+ TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
+ final Reader r = initReader(fieldName, reader);
+ if (components == null) {
+ components = createComponents(fieldName, r);
+ reuseStrategy.setReusableComponents(this, fieldName, components);
+ } else {
+ components.setReader(r);
+ }
+ return components.getTokenStream();
+ }
+
+ /**
+ * Returns a TokenStream suitable for <code>fieldName</code>, tokenizing
+ * the contents of <code>text</code>.
+ * <p>
+ * This method uses {@link #createComponents(String, Reader)} to obtain an
+ * instance of {@link TokenStreamComponents}. It returns the sink of the
+ * components and stores the components internally. Subsequent calls to this
+ * method will reuse the previously stored components after resetting them
+ * through {@link TokenStreamComponents#setReader(Reader)}.
+ * <p>
+ * <b>NOTE:</b> After calling this method, the consumer must follow the
+ * workflow described in {@link TokenStream} to properly consume its contents.
+ * See the {@link org.apache.lucene.analysis Analysis package documentation} for
+ * some examples demonstrating this.
+ *
+ * @param fieldName the name of the field the created TokenStream is used for
+ * @param text the String the streams source reads from
+ * @return TokenStream for iterating the analyzed content of <code>reader</code>
+ * @throws AlreadyClosedException if the Analyzer is closed.
+ * @throws IOException if an i/o error occurs (may rarely happen for strings).
+ * @see #tokenStream(String, Reader)
+ */
+ public final TokenStream tokenStream(final String fieldName, final String text) throws IOException {
+ TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
+ @SuppressWarnings("resource") final ReusableStringReader strReader =
+ (components == null || components.reusableStringReader == null) ?
+ new ReusableStringReader() : components.reusableStringReader;
+ strReader.setValue(text);
+ final Reader r = initReader(fieldName, strReader);
+ if (components == null) {
+ components = createComponents(fieldName, r);
+ reuseStrategy.setReusableComponents(this, fieldName, components);
+ } else {
+ components.setReader(r);
+ }
+ components.reusableStringReader = strReader;
+ return components.getTokenStream();
+ }
+
+ /**
+ * Override this if you want to add a CharFilter chain.
+ * <p>
+ * The default implementation returns <code>reader</code>
+ * unchanged.
+ *
+ * @param fieldName IndexableField name being indexed
+ * @param reader original Reader
+ * @return reader, optionally decorated with CharFilter(s)
+ */
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
+ }
+
+ /**
+ * Invoked before indexing a IndexableField instance if
+ * terms have already been added to that field. This allows custom
+ * analyzers to place an automatic position increment gap between
+ * IndexbleField instances using the same field name. The default value
+ * position increment gap is 0. With a 0 position increment gap and
+ * the typical default token position increment of 1, all terms in a field,
+ * including across IndexableField instances, are in successive positions, allowing
+ * exact PhraseQuery matches, for instance, across IndexableField instance boundaries.
+ *
+ * @param fieldName IndexableField name being indexed.
+ * @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}.
+ * This value must be {@code >= 0}.
+ */
+ public int getPositionIncrementGap(String fieldName) {
+ return 0;
+ }
+
+ /**
+ * Just like {@link #getPositionIncrementGap}, except for
+ * Token offsets instead. By default this returns 1.
+ * This method is only called if the field
+ * produced at least one token for indexing.
+ *
+ * @param fieldName the field just indexed
+ * @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}.
+ * This value must be {@code >= 0}.
+ */
+ public int getOffsetGap(String fieldName) {
+ return 1;
+ }
+
+ /**
+ * Returns the used {@link ReuseStrategy}.
+ */
+ public final ReuseStrategy getReuseStrategy() {
+ return reuseStrategy;
+ }
+
+ /** Frees persistent resources used by this Analyzer */
+ @Override
+ public void close() {
+ if (storedValue != null) {
+ storedValue.close();
+ storedValue = null;
+ }
+ }
+
+ /**
+ * This class encapsulates the outer components of a token stream. It provides
+ * access to the source ({@link Tokenizer}) and the outer end (sink), an
+ * instance of {@link TokenFilter} which also serves as the
+ * {@link TokenStream} returned by
+ * {@link Analyzer#tokenStream(String, Reader)}.
+ */
+ public static class TokenStreamComponents {
+ /**
+ * Original source of the tokens.
+ */
+ protected final Tokenizer source;
+ /**
+ * Sink tokenstream, such as the outer tokenfilter decorating
+ * the chain. This can be the source if there are no filters.
+ */
+ protected final TokenStream sink;
+
+ /** Internal cache only used by {@link Analyzer#tokenStream(String, String)}. */
+ transient ReusableStringReader reusableStringReader;
+
+ /**
+ * Creates a new {@link TokenStreamComponents} instance.
+ *
+ * @param source
+ * the analyzer's tokenizer
+ * @param result
+ * the analyzer's resulting token stream
+ */
+ public TokenStreamComponents(final Tokenizer source,
+ final TokenStream result) {
+ this.source = source;
+ this.sink = result;
+ }
+
+ /**
+ * Creates a new {@link TokenStreamComponents} instance.
+ *
+ * @param source
+ * the analyzer's tokenizer
+ */
+ public TokenStreamComponents(final Tokenizer source) {
+ this.source = source;
+ this.sink = source;
+ }
+
+ /**
+ * Resets the encapsulated components with the given reader. If the components
+ * cannot be reset, an Exception should be thrown.
+ *
+ * @param reader
+ * a reader to reset the source component
+ * @throws IOException
+ * if the component's reset method throws an {@link IOException}
+ */
+ protected void setReader(final Reader reader) throws IOException {
+ source.setReader(reader);
+ }
+
+ /**
+ * Returns the sink {@link TokenStream}
+ *
+ * @return the sink {@link TokenStream}
+ */
+ public TokenStream getTokenStream() {
+ return sink;
+ }
+
+ /**
+ * Returns the component's {@link Tokenizer}
+ *
+ * @return Component's {@link Tokenizer}
+ */
+ public Tokenizer getTokenizer() {
+ return source;
+ }
+ }
+
+ /**
+ * Strategy defining how TokenStreamComponents are reused per call to
+ * {@link Analyzer#tokenStream(String, java.io.Reader)}.
+ */
+ public static abstract class ReuseStrategy {
+
+ /** Sole constructor. (For invocation by subclass constructors, typically implicit.) */
+ public ReuseStrategy() {}
+
+ /**
+ * Gets the reusable TokenStreamComponents for the field with the given name.
+ *
+ * @param analyzer Analyzer from which to get the reused components. Use
+ * {@link #getStoredValue(Analyzer)} and {@link #setStoredValue(Analyzer, Object)}
+ * to access the data on the Analyzer.
+ * @param fieldName Name of the field whose reusable TokenStreamComponents
+ * are to be retrieved
+ * @return Reusable TokenStreamComponents for the field, or {@code null}
+ * if there was no previous components for the field
+ */
+ public abstract TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName);
+
+ /**
+ * Stores the given TokenStreamComponents as the reusable components for the
+ * field with the give name.
+ *
+ * @param fieldName Name of the field whose TokenStreamComponents are being set
+ * @param components TokenStreamComponents which are to be reused for the field
+ */
+ public abstract void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components);
+
+ /**
+ * Returns the currently stored value.
+ *
+ * @return Currently stored value or {@code null} if no value is stored
+ * @throws AlreadyClosedException if the Analyzer is closed.
+ */
+ protected final Object getStoredValue(Analyzer analyzer) {
+ if (analyzer.storedValue == null) {
+ throw new AlreadyClosedException("this Analyzer is closed");
+ }
+ return analyzer.storedValue.get();
+ }
+
+ /**
+ * Sets the stored value.
+ *
+ * @param storedValue Value to store
+ * @throws AlreadyClosedException if the Analyzer is closed.
+ */
+ protected final void setStoredValue(Analyzer analyzer, Object storedValue) {
+ if (analyzer.storedValue == null) {
+ throw new AlreadyClosedException("this Analyzer is closed");
+ }
+ analyzer.storedValue.set(storedValue);
+ }
+
+ }
+
+ /**
+ * A predefined {@link ReuseStrategy} that reuses the same components for
+ * every field.
+ */
+ public static final ReuseStrategy GLOBAL_REUSE_STRATEGY = new GlobalReuseStrategy();
+
+ /**
+ * Implementation of {@link ReuseStrategy} that reuses the same components for
+ * every field.
+ * @deprecated This implementation class will be hidden in Lucene 5.0.
+ * Use {@link Analyzer#GLOBAL_REUSE_STRATEGY} instead!
+ */
+ @Deprecated
+ public final static class GlobalReuseStrategy extends ReuseStrategy {
+
+ /** Sole constructor. (For invocation by subclass constructors, typically implicit.)
+ * @deprecated Don't create instances of this class, use {@link Analyzer#GLOBAL_REUSE_STRATEGY} */
+ @Deprecated
+ public GlobalReuseStrategy() {}
+
+ @Override
+ public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
+ return (TokenStreamComponents) getStoredValue(analyzer);
+ }
+
+ @Override
+ public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
+ setStoredValue(analyzer, components);
+ }
+ }
+
+ /**
+ * A predefined {@link ReuseStrategy} that reuses components per-field by
+ * maintaining a Map of TokenStreamComponent per field name.
+ */
+ public static final ReuseStrategy PER_FIELD_REUSE_STRATEGY = new PerFieldReuseStrategy();
+
+ /**
+ * Implementation of {@link ReuseStrategy} that reuses components per-field by
+ * maintaining a Map of TokenStreamComponent per field name.
+ * @deprecated This implementation class will be hidden in Lucene 5.0.
+ * Use {@link Analyzer#PER_FIELD_REUSE_STRATEGY} instead!
+ */
+ @Deprecated
+ public static class PerFieldReuseStrategy extends ReuseStrategy {
+
+ /** Sole constructor. (For invocation by subclass constructors, typically implicit.)
+ * @deprecated Don't create instances of this class, use {@link Analyzer#PER_FIELD_REUSE_STRATEGY} */
+ @Deprecated
+ public PerFieldReuseStrategy() {}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
+ Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue(analyzer);
+ return componentsPerField != null ? componentsPerField.get(fieldName) : null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
+ Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue(analyzer);
+ if (componentsPerField == null) {
+ componentsPerField = new HashMap<String, TokenStreamComponents>();
+ setStoredValue(analyzer, componentsPerField);
+ }
+ componentsPerField.put(fieldName, components);
+ }
+ }
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/AnalyzerWrapper.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/AnalyzerWrapper.java
new file mode 100644
index 0000000000..73832765c6
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/AnalyzerWrapper.java
@@ -0,0 +1,127 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+/**
+ * Extension to {@link Analyzer} suitable for Analyzers which wrap
+ * other Analyzers.
+ * <p/>
+ * {@link #getWrappedAnalyzer(String)} allows the Analyzer
+ * to wrap multiple Analyzers which are selected on a per field basis.
+ * <p/>
+ * {@link #wrapComponents(String, Analyzer.TokenStreamComponents)} allows the
+ * TokenStreamComponents of the wrapped Analyzer to then be wrapped
+ * (such as adding a new {@link TokenFilter} to form new TokenStreamComponents.
+ */
+public abstract class AnalyzerWrapper extends Analyzer {
+
+ /**
+ * Creates a new AnalyzerWrapper. Since the {@link Analyzer.ReuseStrategy} of
+ * the wrapped Analyzers are unknown, {@link #PER_FIELD_REUSE_STRATEGY} is assumed.
+ * @deprecated Use {@link #AnalyzerWrapper(Analyzer.ReuseStrategy)}
+ * and specify a valid {@link Analyzer.ReuseStrategy}, probably retrieved from the
+ * wrapped analyzer using {@link #getReuseStrategy()}.
+ */
+ @Deprecated
+ protected AnalyzerWrapper() {
+ this(PER_FIELD_REUSE_STRATEGY);
+ }
+
+ /**
+ * Creates a new AnalyzerWrapper with the given reuse strategy.
+ * <p>If you want to wrap a single delegate Analyzer you can probably
+ * reuse its strategy when instantiating this subclass:
+ * {@code super(delegate.getReuseStrategy());}.
+ * <p>If you choose different analyzers per field, use
+ * {@link #PER_FIELD_REUSE_STRATEGY}.
+ * @see #getReuseStrategy()
+ */
+ protected AnalyzerWrapper(ReuseStrategy reuseStrategy) {
+ super(reuseStrategy);
+ }
+
+ /**
+ * Retrieves the wrapped Analyzer appropriate for analyzing the field with
+ * the given name
+ *
+ * @param fieldName Name of the field which is to be analyzed
+ * @return Analyzer for the field with the given name. Assumed to be non-null
+ */
+ protected abstract Analyzer getWrappedAnalyzer(String fieldName);
+
+ /**
+ * Wraps / alters the given TokenStreamComponents, taken from the wrapped
+ * Analyzer, to form new components. It is through this method that new
+ * TokenFilters can be added by AnalyzerWrappers. By default, the given
+ * components are returned.
+ *
+ * @param fieldName
+ * Name of the field which is to be analyzed
+ * @param components
+ * TokenStreamComponents taken from the wrapped Analyzer
+ * @return Wrapped / altered TokenStreamComponents.
+ */
+ protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+ return components;
+ }
+
+ /**
+ * Wraps / alters the given Reader. Through this method AnalyzerWrappers can
+ * implement {@link #initReader(String, Reader)}. By default, the given reader
+ * is returned.
+ *
+ * @param fieldName
+ * name of the field which is to be analyzed
+ * @param reader
+ * the reader to wrap
+ * @return the wrapped reader
+ */
+ protected Reader wrapReader(String fieldName, Reader reader) {
+ return reader;
+ }
+
+ @Override
+ protected final TokenStreamComponents createComponents(String fieldName, Reader aReader) {
+ return wrapComponents(fieldName, getWrappedAnalyzer(fieldName).createComponents(fieldName, aReader));
+ }
+
+ @Override
+ public int getPositionIncrementGap(String fieldName) {
+ return getWrappedAnalyzer(fieldName).getPositionIncrementGap(fieldName);
+ }
+
+ @Override
+ public int getOffsetGap(String fieldName) {
+ return getWrappedAnalyzer(fieldName).getOffsetGap(fieldName);
+ }
+
+ @Override
+ public final Reader initReader(String fieldName, Reader reader) {
+ return getWrappedAnalyzer(fieldName).initReader(fieldName, wrapReader(fieldName, reader));
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/CachingTokenFilter.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/CachingTokenFilter.java
new file mode 100644
index 0000000000..8750101dff
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/CachingTokenFilter.java
@@ -0,0 +1,106 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.lucene.util.AttributeSource;
+
+/**
+ * This class can be used if the token attributes of a TokenStream
+ * are intended to be consumed more than once. It caches
+ * all token attribute states locally in a List.
+ *
+ * <P>CachingTokenFilter implements the optional method
+ * {@link TokenStream#reset()}, which repositions the
+ * stream to the first Token.
+ */
+public final class CachingTokenFilter extends TokenFilter {
+ private List<AttributeSource.State> cache = null;
+ private Iterator<AttributeSource.State> iterator = null;
+ private AttributeSource.State finalState;
+
+ /**
+ * Create a new CachingTokenFilter around <code>input</code>,
+ * caching its token attributes, which can be replayed again
+ * after a call to {@link #reset()}.
+ */
+ public CachingTokenFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (cache == null) {
+ // fill cache lazily
+ cache = new LinkedList<AttributeSource.State>();
+ fillCache();
+ iterator = cache.iterator();
+ }
+
+ if (!iterator.hasNext()) {
+ // the cache is exhausted, return false
+ return false;
+ }
+ // Since the TokenFilter can be reset, the tokens need to be preserved as immutable.
+ restoreState(iterator.next());
+ return true;
+ }
+
+ @Override
+ public final void end() {
+ if (finalState != null) {
+ restoreState(finalState);
+ }
+ }
+
+ /**
+ * Rewinds the iterator to the beginning of the cached list.
+ * <p>
+ * Note that this does not call reset() on the wrapped tokenstream ever, even
+ * the first time. You should reset() the inner tokenstream before wrapping
+ * it with CachingTokenFilter.
+ */
+ @Override
+ public void reset() {
+ if(cache != null) {
+ iterator = cache.iterator();
+ }
+ }
+
+ private void fillCache() throws IOException {
+ while(input.incrementToken()) {
+ cache.add(captureState());
+ }
+ // capture final state
+ input.end();
+ finalState = captureState();
+ }
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/CharFilter.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/CharFilter.java
new file mode 100644
index 0000000000..467550d041
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/CharFilter.java
@@ -0,0 +1,92 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Subclasses of CharFilter can be chained to filter a Reader
+ * They can be used as {@link java.io.Reader} with additional offset
+ * correction. {@link Tokenizer}s will automatically use {@link #correctOffset}
+ * if a CharFilter subclass is used.
+ * <p>
+ * This class is abstract: at a minimum you must implement {@link #read(char[], int, int)},
+ * transforming the input in some way from {@link #input}, and {@link #correct(int)}
+ * to adjust the offsets to match the originals.
+ * <p>
+ * You can optionally provide more efficient implementations of additional methods
+ * like {@link #read()}, {@link #read(char[])}, {@link #read(java.nio.CharBuffer)},
+ * but this is not required.
+ * <p>
+ * For examples and integration with {@link Analyzer}, see the
+ * {@link org.apache.lucene.analysis Analysis package documentation}.
+ */
+// the way java.io.FilterReader should work!
+public abstract class CharFilter extends Reader {
+ /**
+ * The underlying character-input stream.
+ */
+ protected final Reader input;
+
+ /**
+ * Create a new CharFilter wrapping the provided reader.
+ * @param input a Reader, can also be a CharFilter for chaining.
+ */
+ public CharFilter(Reader input) {
+ super(input);
+ this.input = input;
+ }
+
+ /**
+ * Closes the underlying input stream.
+ * <p>
+ * <b>NOTE:</b>
+ * The default implementation closes the input Reader, so
+ * be sure to call <code>super.close()</code> when overriding this method.
+ */
+ @Override
+ public void close() throws IOException {
+ input.close();
+ }
+
+ /**
+ * Subclasses override to correct the current offset.
+ *
+ * @param currentOff current offset
+ * @return corrected offset
+ */
+ protected abstract int correct(int currentOff);
+
+ /**
+ * Chains the corrected offset through the input
+ * CharFilter(s).
+ */
+ public final int correctOffset(int currentOff) {
+ final int corrected = correct(currentOff);
+ return (input instanceof CharFilter) ? ((CharFilter) input).correctOffset(corrected) : corrected;
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/NumericTokenStream.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/NumericTokenStream.java
new file mode 100644
index 0000000000..fbad0d1907
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/NumericTokenStream.java
@@ -0,0 +1,331 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.document.DoubleField; // for javadocs
+import org.apache.lucene.document.FloatField; // for javadocs
+import org.apache.lucene.document.IntField; // for javadocs
+import org.apache.lucene.document.LongField; // for javadocs
+import org.apache.lucene.search.NumericRangeFilter; // for javadocs
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+
+/**
+ * <b>Expert:</b> This class provides a {@link TokenStream}
+ * for indexing numeric values that can be used by {@link
+ * NumericRangeQuery} or {@link NumericRangeFilter}.
+ *
+ * <p>Note that for simple usage, {@link IntField}, {@link
+ * LongField}, {@link FloatField} or {@link DoubleField} is
+ * recommended. These fields disable norms and
+ * term freqs, as they are not usually needed during
+ * searching. If you need to change these settings, you
+ * should use this class.
+ *
+ * <p>Here's an example usage, for an <code>int</code> field:
+ *
+ * <pre class="prettyprint">
+ * FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ * fieldType.setOmitNorms(true);
+ * fieldType.setIndexOptions(IndexOptions.DOCS_ONLY);
+ * Field field = new Field(name, new NumericTokenStream(precisionStep).setIntValue(value), fieldType);
+ * document.add(field);
+ * </pre>
+ *
+ * <p>For optimal performance, re-use the TokenStream and Field instance
+ * for more than one document:
+ *
+ * <pre class="prettyprint">
+ * NumericTokenStream stream = new NumericTokenStream(precisionStep);
+ * FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ * fieldType.setOmitNorms(true);
+ * fieldType.setIndexOptions(IndexOptions.DOCS_ONLY);
+ * Field field = new Field(name, stream, fieldType);
+ * Document document = new Document();
+ * document.add(field);
+ *
+ * for(all documents) {
+ * stream.setIntValue(value)
+ * writer.addDocument(document);
+ * }
+ * </pre>
+ *
+ * <p>This stream is not intended to be used in analyzers;
+ * it's more for iterating the different precisions during
+ * indexing a specific numeric value.</p>
+
+ * <p><b>NOTE</b>: as token streams are only consumed once
+ * the document is added to the index, if you index more
+ * than one numeric field, use a separate <code>NumericTokenStream</code>
+ * instance for each.</p>
+ *
+ * <p>See {@link NumericRangeQuery} for more details on the
+ * <a
+ * href="../search/NumericRangeQuery.html#precisionStepDesc"><code>precisionStep</code></a>
+ * parameter as well as how numeric fields work under the hood.</p>
+ *
+ * @since 2.9
+ */
+public final class NumericTokenStream extends TokenStream {
+
+ /** The full precision token gets this token type assigned. */
+ public static final String TOKEN_TYPE_FULL_PREC = "fullPrecNumeric";
+
+ /** The lower precision tokens gets this token type assigned. */
+ public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecNumeric";
+
+ /** <b>Expert:</b> Use this attribute to get the details of the currently generated token.
+ * @lucene.experimental
+ * @since 4.0
+ */
+ public interface NumericTermAttribute extends Attribute {
+ /** Returns current shift value, undefined before first token */
+ int getShift();
+ /** Returns current token's raw value as {@code long} with all {@link #getShift} applied, undefined before first token */
+ long getRawValue();
+ /** Returns value size in bits (32 for {@code float}, {@code int}; 64 for {@code double}, {@code long}) */
+ int getValueSize();
+
+ /** <em>Don't call this method!</em>
+ * @lucene.internal */
+ void init(long value, int valSize, int precisionStep, int shift);
+
+ /** <em>Don't call this method!</em>
+ * @lucene.internal */
+ void setShift(int shift);
+
+ /** <em>Don't call this method!</em>
+ * @lucene.internal */
+ int incShift();
+ }
+
+ // just a wrapper to prevent adding CTA
+ private static final class NumericAttributeFactory extends AttributeFactory {
+ private final AttributeFactory delegate;
+
+ NumericAttributeFactory(AttributeFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+ if (CharTermAttribute.class.isAssignableFrom(attClass))
+ throw new IllegalArgumentException("NumericTokenStream does not support CharTermAttribute.");
+ return delegate.createAttributeInstance(attClass);
+ }
+ }
+
+ /** Implementation of {@link NumericTermAttribute}.
+ * @lucene.internal
+ * @since 4.0
+ */
+ public static final class NumericTermAttributeImpl extends AttributeImpl implements NumericTermAttribute,TermToBytesRefAttribute {
+ private long value = 0L;
+ private int valueSize = 0, shift = 0, precisionStep = 0;
+ private BytesRef bytes = new BytesRef();
+
+ /**
+ * Creates, but does not yet initialize this attribute instance
+ * @see #init(long, int, int, int)
+ */
+ public NumericTermAttributeImpl() {}
+
+ @Override
+ public BytesRef getBytesRef() {
+ return bytes;
+ }
+
+ @Override
+ public int fillBytesRef() {
+ assert valueSize == 64 || valueSize == 32;
+ return (valueSize == 64) ?
+ NumericUtils.longToPrefixCoded(value, shift, bytes) :
+ NumericUtils.intToPrefixCoded((int) value, shift, bytes);
+ }
+
+ @Override
+ public int getShift() { return shift; }
+ @Override
+ public void setShift(int shift) { this.shift = shift; }
+ @Override
+ public int incShift() {
+ return (shift += precisionStep);
+ }
+
+ @Override
+ public long getRawValue() { return value & ~((1L << shift) - 1L); }
+ @Override
+ public int getValueSize() { return valueSize; }
+
+ @Override
+ public void init(long value, int valueSize, int precisionStep, int shift) {
+ this.value = value;
+ this.valueSize = valueSize;
+ this.precisionStep = precisionStep;
+ this.shift = shift;
+ }
+
+ @Override
+ public void clear() {
+ // this attribute has no contents to clear!
+ // we keep it untouched as it's fully controlled by outer class.
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ fillBytesRef();
+ reflector.reflect(TermToBytesRefAttribute.class, "bytes", BytesRef.deepCopyOf(bytes));
+ reflector.reflect(NumericTermAttribute.class, "shift", shift);
+ reflector.reflect(NumericTermAttribute.class, "rawValue", getRawValue());
+ reflector.reflect(NumericTermAttribute.class, "valueSize", valueSize);
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ final NumericTermAttribute a = (NumericTermAttribute) target;
+ a.init(value, valueSize, precisionStep, shift);
+ }
+ }
+
+ /**
+ * Creates a token stream for numeric values using the default <code>precisionStep</code>
+ * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The stream is not yet initialized,
+ * before using set a value using the various set<em>???</em>Value() methods.
+ */
+ public NumericTokenStream() {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, NumericUtils.PRECISION_STEP_DEFAULT);
+ }
+
+ /**
+ * Creates a token stream for numeric values with the specified
+ * <code>precisionStep</code>. The stream is not yet initialized,
+ * before using set a value using the various set<em>???</em>Value() methods.
+ */
+ public NumericTokenStream(final int precisionStep) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, precisionStep);
+ }
+
+ /**
+ * Expert: Creates a token stream for numeric values with the specified
+ * <code>precisionStep</code> using the given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ * The stream is not yet initialized,
+ * before using set a value using the various set<em>???</em>Value() methods.
+ */
+ public NumericTokenStream(AttributeFactory factory, final int precisionStep) {
+ super(new NumericAttributeFactory(factory));
+ if (precisionStep < 1)
+ throw new IllegalArgumentException("precisionStep must be >=1");
+ this.precisionStep = precisionStep;
+ numericAtt.setShift(-precisionStep);
+ }
+
+ /**
+ * Initializes the token stream with the supplied <code>long</code> value.
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * <code>new Field(name, new NumericTokenStream(precisionStep).setLongValue(value))</code>
+ */
+ public NumericTokenStream setLongValue(final long value) {
+ numericAtt.init(value, valSize = 64, precisionStep, -precisionStep);
+ return this;
+ }
+
+ /**
+ * Initializes the token stream with the supplied <code>int</code> value.
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * <code>new Field(name, new NumericTokenStream(precisionStep).setIntValue(value))</code>
+ */
+ public NumericTokenStream setIntValue(final int value) {
+ numericAtt.init(value, valSize = 32, precisionStep, -precisionStep);
+ return this;
+ }
+
+ /**
+ * Initializes the token stream with the supplied <code>double</code> value.
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * <code>new Field(name, new NumericTokenStream(precisionStep).setDoubleValue(value))</code>
+ */
+ public NumericTokenStream setDoubleValue(final double value) {
+ numericAtt.init(NumericUtils.doubleToSortableLong(value), valSize = 64, precisionStep, -precisionStep);
+ return this;
+ }
+
+ /**
+ * Initializes the token stream with the supplied <code>float</code> value.
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * <code>new Field(name, new NumericTokenStream(precisionStep).setFloatValue(value))</code>
+ */
+ public NumericTokenStream setFloatValue(final float value) {
+ numericAtt.init(NumericUtils.floatToSortableInt(value), valSize = 32, precisionStep, -precisionStep);
+ return this;
+ }
+
+ @Override
+ public void reset() {
+ if (valSize == 0)
+ throw new IllegalStateException("call set???Value() before usage");
+ numericAtt.setShift(-precisionStep);
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (valSize == 0)
+ throw new IllegalStateException("call set???Value() before usage");
+
+ // this will only clear all other attributes in this TokenStream
+ clearAttributes();
+
+ final int shift = numericAtt.incShift();
+ typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
+ posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
+ return (shift < valSize);
+ }
+
+ /** Returns the precision step. */
+ public int getPrecisionStep() {
+ return precisionStep;
+ }
+
+ // members
+ private final NumericTermAttribute numericAtt = addAttribute(NumericTermAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ private int valSize = 0; // valSize==0 means not initialized
+ private final int precisionStep;
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/ReusableStringReader.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/ReusableStringReader.java
new file mode 100644
index 0000000000..5c0fcadc7e
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/ReusableStringReader.java
@@ -0,0 +1,69 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.Reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Internal class to enable reuse of the string reader by {@link Analyzer#tokenStream(String,String)} */
+final class ReusableStringReader extends Reader {
+ private int pos = 0, size = 0;
+ private String s = null;
+
+ void setValue(String s) {
+ this.s = s;
+ this.size = s.length();
+ this.pos = 0;
+ }
+
+ @Override
+ public int read() {
+ if (pos < size) {
+ return s.charAt(pos++);
+ } else {
+ s = null;
+ return -1;
+ }
+ }
+
+ @Override
+ public int read(char[] c, int off, int len) {
+ if (pos < size) {
+ len = Math.min(len, size-pos);
+ s.getChars(pos, pos+len, c, off);
+ pos += len;
+ return len;
+ } else {
+ s = null;
+ return -1;
+ }
+ }
+
+ @Override
+ public void close() {
+ pos = size; // this prevents NPE when reading after close!
+ s = null;
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/Token.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/Token.java
new file mode 100644
index 0000000000..ea387a4ac8
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/Token.java
@@ -0,0 +1,670 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.index.DocsAndPositionsEnum; // for javadoc
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ A Token is an occurrence of a term from the text of a field. It consists of
+ a term's text, the start and end offset of the term in the text of the field,
+ and a type string.
+ <p>
+ The start and end offsets permit applications to re-associate a token with
+ its source text, e.g., to display highlighted query terms in a document
+ browser, or to show matching text fragments in a <abbr title="KeyWord In Context">KWIC</abbr>
+ display, etc.
+ <p>
+ The type is a string, assigned by a lexical analyzer
+ (a.k.a. tokenizer), naming the lexical or syntactic class that the token
+ belongs to. For example an end of sentence marker token might be implemented
+ with type "eos". The default token type is "word".
+ <p>
+ A Token can optionally have metadata (a.k.a. payload) in the form of a variable
+ length byte array. Use {@link DocsAndPositionsEnum#getPayload()} to retrieve the
+ payloads from the index.
+
+ <br><br>
+
+ <p><b>NOTE:</b> As of 2.9, Token implements all {@link Attribute} interfaces
+ that are part of core Lucene and can be found in the {@code tokenattributes} subpackage.
+ Even though it is not necessary to use Token anymore, with the new TokenStream API it can
+ be used as convenience class that implements all {@link Attribute}s, which is especially useful
+ to easily switch from the old to the new TokenStream API.
+
+ <br><br>
+
+ <p>Tokenizers and TokenFilters should try to re-use a Token
+ instance when possible for best performance, by
+ implementing the {@link TokenStream#incrementToken()} API.
+ Failing that, to create a new Token you should first use
+ one of the constructors that starts with null text. To load
+ the token from a char[] use {@link #copyBuffer(char[], int, int)}.
+ To load from a String use {@link #setEmpty} followed by {@link #append(CharSequence)} or {@link #append(CharSequence, int, int)}.
+ Alternatively you can get the Token's termBuffer by calling either {@link #buffer()},
+ if you know that your text is shorter than the capacity of the termBuffer
+ or {@link #resizeBuffer(int)}, if there is any possibility
+ that you may need to grow the buffer. Fill in the characters of your term into this
+ buffer, with {@link String#getChars(int, int, char[], int)} if loading from a string,
+ or with {@link System#arraycopy(Object, int, Object, int, int)}, and finally call {@link #setLength(int)} to
+ set the length of the term text. See <a target="_top"
+ href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
+ for details.</p>
+ <p>Typical Token reuse patterns:
+ <ul>
+ <li> Copying text from a string (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre class="prettyprint">
+ return reusableToken.reinit(string, startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ <li> Copying some text from a string (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre class="prettyprint">
+ return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ </li>
+ <li> Copying text from char[] buffer (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre class="prettyprint">
+ return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ <li> Copying some text from a char[] buffer (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre class="prettyprint">
+ return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ <li> Copying from one one Token to another (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre class="prettyprint">
+ return reusableToken.reinit(source.buffer(), 0, source.length(), source.startOffset(), source.endOffset()[, source.type()]);
+ </pre>
+ </li>
+ </ul>
+ A few things to note:
+ <ul>
+ <li>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</li>
+ <li>Because <code>TokenStreams</code> can be chained, one cannot assume that the <code>Token's</code> current type is correct.</li>
+ <li>The startOffset and endOffset represent the start and offset in the source text, so be careful in adjusting them.</li>
+ <li>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</li>
+ </ul>
+ </p>
+ <p>
+ <b>Please note:</b> With Lucene 3.1, the <code>{@linkplain #toString toString()}</code> method had to be changed to match the
+ {@link CharSequence} interface introduced by the interface {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}.
+ This method now only prints the term text, no additional information anymore.
+ </p>
+*/
+public class Token extends CharTermAttributeImpl
+ implements TypeAttribute, PositionIncrementAttribute,
+ FlagsAttribute, OffsetAttribute, PayloadAttribute, PositionLengthAttribute {
+
+ private int startOffset,endOffset;
+ private String type = DEFAULT_TYPE;
+ private int flags;
+ private BytesRef payload;
+ private int positionIncrement = 1;
+ private int positionLength = 1;
+
+ /** Constructs a Token will null text. */
+ public Token() {
+ }
+
+ /** Constructs a Token with null text and start & end
+ * offsets.
+ * @param start start offset in the source text
+ * @param end end offset in the source text */
+ public Token(int start, int end) {
+ checkOffsets(start, end);
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /** Constructs a Token with null text and start & end
+ * offsets plus the Token type.
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ * @param typ the lexical type of this Token */
+ public Token(int start, int end, String typ) {
+ checkOffsets(start, end);
+ startOffset = start;
+ endOffset = end;
+ type = typ;
+ }
+
+ /**
+ * Constructs a Token with null text and start & end
+ * offsets plus flags. NOTE: flags is EXPERIMENTAL.
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ * @param flags The bits to set for this token
+ */
+ public Token(int start, int end, int flags) {
+ checkOffsets(start, end);
+ startOffset = start;
+ endOffset = end;
+ this.flags = flags;
+ }
+
+ /** Constructs a Token with the given term text, and start
+ * & end offsets. The type defaults to "word."
+ * <b>NOTE:</b> for better indexing speed you should
+ * instead use the char[] termBuffer methods to set the
+ * term text.
+ * @param text term text
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ */
+ public Token(String text, int start, int end) {
+ checkOffsets(start, end);
+ append(text);
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /** Constructs a Token with the given text, start and end
+ * offsets, & type. <b>NOTE:</b> for better indexing
+ * speed you should instead use the char[] termBuffer
+ * methods to set the term text.
+ * @param text term text
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ * @param typ token type
+ */
+ public Token(String text, int start, int end, String typ) {
+ checkOffsets(start, end);
+ append(text);
+ startOffset = start;
+ endOffset = end;
+ type = typ;
+ }
+
+ /**
+ * Constructs a Token with the given text, start and end
+ * offsets, & type. <b>NOTE:</b> for better indexing
+ * speed you should instead use the char[] termBuffer
+ * methods to set the term text.
+ * @param text term text
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ * @param flags token type bits
+ */
+ public Token(String text, int start, int end, int flags) {
+ checkOffsets(start, end);
+ append(text);
+ startOffset = start;
+ endOffset = end;
+ this.flags = flags;
+ }
+
+ /**
+ * Constructs a Token with the given term buffer (offset
+ * & length), start and end
+ * offsets
+ * @param startTermBuffer buffer containing term text
+ * @param termBufferOffset the index in the buffer of the first character
+ * @param termBufferLength number of valid characters in the buffer
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ */
+ public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end) {
+ checkOffsets(start, end);
+ copyBuffer(startTermBuffer, termBufferOffset, termBufferLength);
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see PositionIncrementAttribute
+ */
+ @Override
+ public void setPositionIncrement(int positionIncrement) {
+ if (positionIncrement < 0)
+ throw new IllegalArgumentException
+ ("Increment must be zero or greater: " + positionIncrement);
+ this.positionIncrement = positionIncrement;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see PositionIncrementAttribute
+ */
+ @Override
+ public int getPositionIncrement() {
+ return positionIncrement;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see PositionLengthAttribute
+ */
+ @Override
+ public void setPositionLength(int positionLength) {
+ this.positionLength = positionLength;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see PositionLengthAttribute
+ */
+ @Override
+ public int getPositionLength() {
+ return positionLength;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see OffsetAttribute
+ */
+ @Override
+ public final int startOffset() {
+ return startOffset;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see OffsetAttribute
+ */
+ @Override
+ public final int endOffset() {
+ return endOffset;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see OffsetAttribute
+ */
+ @Override
+ public void setOffset(int startOffset, int endOffset) {
+ checkOffsets(startOffset, endOffset);
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see TypeAttribute
+ */
+ @Override
+ public final String type() {
+ return type;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see TypeAttribute
+ */
+ @Override
+ public final void setType(String type) {
+ this.type = type;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see FlagsAttribute
+ */
+ @Override
+ public int getFlags() {
+ return flags;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see FlagsAttribute
+ */
+ @Override
+ public void setFlags(int flags) {
+ this.flags = flags;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see PayloadAttribute
+ */
+ @Override
+ public BytesRef getPayload() {
+ return this.payload;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see PayloadAttribute
+ */
+ @Override
+ public void setPayload(BytesRef payload) {
+ this.payload = payload;
+ }
+
+ /** Resets the term text, payload, flags, and positionIncrement,
+ * startOffset, endOffset and token type to default.
+ */
+ @Override
+ public void clear() {
+ super.clear();
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ @Override
+ public Token clone() {
+ Token t = (Token)super.clone();
+ // Do a deep clone
+ if (payload != null) {
+ t.payload = payload.clone();
+ }
+ return t;
+ }
+
+ /** Makes a clone, but replaces the term buffer &
+ * start/end offset in the process. This is more
+ * efficient than doing a full clone (and then calling
+ * {@link #copyBuffer}) because it saves a wasted copy of the old
+ * termBuffer. */
+ public Token clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ final Token t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset);
+ t.positionIncrement = positionIncrement;
+ t.flags = flags;
+ t.type = type;
+ if (payload != null)
+ t.payload = payload.clone();
+ return t;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj instanceof Token) {
+ final Token other = (Token) obj;
+ return (startOffset == other.startOffset &&
+ endOffset == other.endOffset &&
+ flags == other.flags &&
+ positionIncrement == other.positionIncrement &&
+ (type == null ? other.type == null : type.equals(other.type)) &&
+ (payload == null ? other.payload == null : payload.equals(other.payload)) &&
+ super.equals(obj)
+ );
+ } else
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ int code = super.hashCode();
+ code = code * 31 + startOffset;
+ code = code * 31 + endOffset;
+ code = code * 31 + flags;
+ code = code * 31 + positionIncrement;
+ if (type != null)
+ code = code * 31 + type.hashCode();
+ if (payload != null)
+ code = code * 31 + payload.hashCode();
+ return code;
+ }
+
+ // like clear() but doesn't clear termBuffer/text
+ private void clearNoTermBuffer() {
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #copyBuffer(char[], int, int)},
+ * {@link #setOffset},
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
+ checkOffsets(newStartOffset, newEndOffset);
+ clearNoTermBuffer();
+ copyBuffer(newTermBuffer, newTermOffset, newTermLength);
+ payload = null;
+ positionIncrement = 1;
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #copyBuffer(char[], int, int)},
+ * {@link #setOffset},
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ checkOffsets(newStartOffset, newEndOffset);
+ clearNoTermBuffer();
+ copyBuffer(newTermBuffer, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence)},
+ * {@link #setOffset},
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newStartOffset, int newEndOffset, String newType) {
+ checkOffsets(newStartOffset, newEndOffset);
+ clear();
+ append(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence, int, int)},
+ * {@link #setOffset},
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
+ checkOffsets(newStartOffset, newEndOffset);
+ clear();
+ append(newTerm, newTermOffset, newTermOffset + newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence)},
+ * {@link #setOffset},
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newStartOffset, int newEndOffset) {
+ checkOffsets(newStartOffset, newEndOffset);
+ clear();
+ append(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence, int, int)},
+ * {@link #setOffset},
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ checkOffsets(newStartOffset, newEndOffset);
+ clear();
+ append(newTerm, newTermOffset, newTermOffset + newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one. Note: Payloads are shared.
+ * @param prototype source Token to copy fields from
+ */
+ public void reinit(Token prototype) {
+ copyBuffer(prototype.buffer(), 0, prototype.length());
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.
+ * @param prototype existing Token
+ * @param newTerm new term text
+ */
+ public void reinit(Token prototype, String newTerm) {
+ setEmpty().append(newTerm);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.
+ * @param prototype existing Token
+ * @param newTermBuffer buffer containing new term text
+ * @param offset the index in the buffer of the first character
+ * @param length number of valid characters in the buffer
+ */
+ public void reinit(Token prototype, char[] newTermBuffer, int offset, int length) {
+ copyBuffer(newTermBuffer, offset, length);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ if (target instanceof Token) {
+ final Token to = (Token) target;
+ to.reinit(this);
+ // reinit shares the payload, so clone it:
+ if (payload !=null) {
+ to.payload = payload.clone();
+ }
+ } else {
+ super.copyTo(target);
+ ((OffsetAttribute) target).setOffset(startOffset, endOffset);
+ ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement);
+ ((PayloadAttribute) target).setPayload((payload == null) ? null : payload.clone());
+ ((FlagsAttribute) target).setFlags(flags);
+ ((TypeAttribute) target).setType(type);
+ }
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ super.reflectWith(reflector);
+ reflector.reflect(OffsetAttribute.class, "startOffset", startOffset);
+ reflector.reflect(OffsetAttribute.class, "endOffset", endOffset);
+ reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement);
+ reflector.reflect(PayloadAttribute.class, "payload", payload);
+ reflector.reflect(FlagsAttribute.class, "flags", flags);
+ reflector.reflect(TypeAttribute.class, "type", type);
+ }
+
+ private void checkOffsets(int startOffset, int endOffset) {
+ if (startOffset < 0 || endOffset < startOffset) {
+ throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
+ + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+ }
+ }
+
+ /** Convenience factory that returns <code>Token</code> as implementation for the basic
+ * attributes and return the default impl (with "Impl" appended) for all other
+ * attributes.
+ * @since 3.0
+ */
+ public static final AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
+ new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
+
+ /** <b>Expert:</b> Creates a TokenAttributeFactory returning {@link Token} as instance for the basic attributes
+ * and for all other attributes calls the given delegate factory.
+ * @since 3.0
+ */
+ public static final class TokenAttributeFactory extends AttributeSource.AttributeFactory {
+
+ private final AttributeSource.AttributeFactory delegate;
+
+ /** <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
+ * and for all other attributes calls the given delegate factory. */
+ public TokenAttributeFactory(AttributeSource.AttributeFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+ return attClass.isAssignableFrom(Token.class)
+ ? new Token() : delegate.createAttributeInstance(attClass);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other instanceof TokenAttributeFactory) {
+ final TokenAttributeFactory af = (TokenAttributeFactory) other;
+ return this.delegate.equals(af.delegate);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return delegate.hashCode() ^ 0x0a45aa31;
+ }
+ }
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenFilter.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenFilter.java
new file mode 100644
index 0000000000..f9cadf9c08
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenFilter.java
@@ -0,0 +1,80 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/** A TokenFilter is a TokenStream whose input is another TokenStream.
+ <p>
+ This is an abstract class; subclasses must override {@link #incrementToken()}.
+ @see TokenStream
+ */
+public abstract class TokenFilter extends TokenStream {
+ /** The source of tokens for this filter. */
+ protected final TokenStream input;
+
+ /** Construct a token stream filtering the given input. */
+ protected TokenFilter(TokenStream input) {
+ super(input);
+ this.input = input;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <b>NOTE:</b>
+ * The default implementation chains the call to the input TokenStream, so
+ * be sure to call <code>super.end()</code> first when overriding this method.
+ */
+ @Override
+ public void end() throws IOException {
+ input.end();
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <b>NOTE:</b>
+ * The default implementation chains the call to the input TokenStream, so
+ * be sure to call <code>super.close()</code> when overriding this method.
+ */
+ @Override
+ public void close() throws IOException {
+ input.close();
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <b>NOTE:</b>
+ * The default implementation chains the call to the input TokenStream, so
+ * be sure to call <code>super.reset()</code> when overriding this method.
+ */
+ @Override
+ public void reset() throws IOException {
+ input.reset();
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenStream.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenStream.java
new file mode 100644
index 0000000000..a7d4e24954
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenStream.java
@@ -0,0 +1,209 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Closeable;
+import java.lang.reflect.Modifier;
+
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource;
+
+/**
+ * A <code>TokenStream</code> enumerates the sequence of tokens, either from
+ * {@link Field}s of a {@link Document} or from query text.
+ * <p>
+ * This is an abstract class; concrete subclasses are:
+ * <ul>
+ * <li>{@link Tokenizer}, a <code>TokenStream</code> whose input is a Reader; and
+ * <li>{@link TokenFilter}, a <code>TokenStream</code> whose input is another
+ * <code>TokenStream</code>.
+ * </ul>
+ * A new <code>TokenStream</code> API has been introduced with Lucene 2.9. This API
+ * has moved from being {@link Token}-based to {@link Attribute}-based. While
+ * {@link Token} still exists in 2.9 as a convenience class, the preferred way
+ * to store the information of a {@link Token} is to use {@link AttributeImpl}s.
+ * <p>
+ * <code>TokenStream</code> now extends {@link AttributeSource}, which provides
+ * access to all of the token {@link Attribute}s for the <code>TokenStream</code>.
+ * Note that only one instance per {@link AttributeImpl} is created and reused
+ * for every token. This approach reduces object creation and allows local
+ * caching of references to the {@link AttributeImpl}s. See
+ * {@link #incrementToken()} for further details.
+ * <p>
+ * <b>The workflow of the new <code>TokenStream</code> API is as follows:</b>
+ * <ol>
+ * <li>Instantiation of <code>TokenStream</code>/{@link TokenFilter}s which add/get
+ * attributes to/from the {@link AttributeSource}.
+ * <li>The consumer calls {@link TokenStream#reset()}.
+ * <li>The consumer retrieves attributes from the stream and stores local
+ * references to all attributes it wants to access.
+ * <li>The consumer calls {@link #incrementToken()} until it returns false
+ * consuming the attributes after each call.
+ * <li>The consumer calls {@link #end()} so that any end-of-stream operations
+ * can be performed.
+ * <li>The consumer calls {@link #close()} to release any resource when finished
+ * using the <code>TokenStream</code>.
+ * </ol>
+ * To make sure that filters and consumers know which attributes are available,
+ * the attributes must be added during instantiation. Filters and consumers are
+ * not required to check for availability of attributes in
+ * {@link #incrementToken()}.
+ * <p>
+ * You can find some example code for the new API in the analysis package level
+ * Javadoc.
+ * <p>
+ * Sometimes it is desirable to capture a current state of a <code>TokenStream</code>,
+ * e.g., for buffering purposes (see {@link CachingTokenFilter},
+ * TeeSinkTokenFilter). For this usecase
+ * {@link AttributeSource#captureState} and {@link AttributeSource#restoreState}
+ * can be used.
+ * <p>The {@code TokenStream}-API in Lucene is based on the decorator pattern.
+ * Therefore all non-abstract subclasses must be final or have at least a final
+ * implementation of {@link #incrementToken}! This is checked when Java
+ * assertions are enabled.
+ */
+public abstract class TokenStream extends AttributeSource implements Closeable {
+
+ /**
+ * A TokenStream using the default attribute factory.
+ */
+ protected TokenStream() {
+ super();
+ assert assertFinal();
+ }
+
+ /**
+ * A TokenStream that uses the same attributes as the supplied one.
+ */
+ protected TokenStream(AttributeSource input) {
+ super(input);
+ assert assertFinal();
+ }
+
+ /**
+ * A TokenStream using the supplied AttributeFactory for creating new {@link Attribute} instances.
+ */
+ protected TokenStream(AttributeFactory factory) {
+ super(factory);
+ assert assertFinal();
+ }
+
+ private boolean assertFinal() {
+ try {
+ final Class<?> clazz = getClass();
+ if (!clazz.desiredAssertionStatus())
+ return true;
+ assert clazz.isAnonymousClass() ||
+ (clazz.getModifiers() & (Modifier.FINAL | Modifier.PRIVATE)) != 0 ||
+ Modifier.isFinal(clazz.getMethod("incrementToken").getModifiers()) :
+ "TokenStream implementation classes or at least their incrementToken() implementation must be final";
+ return true;
+ } catch (NoSuchMethodException nsme) {
+ return false;
+ }
+ }
+
+ /**
+ * Consumers (i.e., {@link IndexWriter}) use this method to advance the stream to
+ * the next token. Implementing classes must implement this method and update
+ * the appropriate {@link AttributeImpl}s with the attributes of the next
+ * token.
+ * <P>
+ * The producer must make no assumptions about the attributes after the method
+ * has been returned: the caller may arbitrarily change it. If the producer
+ * needs to preserve the state for subsequent calls, it can use
+ * {@link #captureState} to create a copy of the current attribute state.
+ * <p>
+ * This method is called for every token of a document, so an efficient
+ * implementation is crucial for good performance. To avoid calls to
+ * {@link #addAttribute(Class)} and {@link #getAttribute(Class)},
+ * references to all {@link AttributeImpl}s that this stream uses should be
+ * retrieved during instantiation.
+ * <p>
+ * To ensure that filters and consumers know which attributes are available,
+ * the attributes must be added during instantiation. Filters and consumers
+ * are not required to check for availability of attributes in
+ * {@link #incrementToken()}.
+ *
+ * @return false for end of stream; true otherwise
+ */
+ public abstract boolean incrementToken() throws IOException;
+
+ /**
+ * This method is called by the consumer after the last token has been
+ * consumed, after {@link #incrementToken()} returned <code>false</code>
+ * (using the new <code>TokenStream</code> API). Streams implementing the old API
+ * should upgrade to use this feature.
+ * <p/>
+ * This method can be used to perform any end-of-stream operations, such as
+ * setting the final offset of a stream. The final offset of a stream might
+ * differ from the offset of the last token eg in case one or more whitespaces
+ * followed after the last token, but a WhitespaceTokenizer was used.
+ * <p>
+ * Additionally any skipped positions (such as those removed by a stopfilter)
+ * can be applied to the position increment, or any adjustment of other
+ * attributes where the end-of-stream value may be important.
+ * <p>
+ * If you override this method, always call {@code super.end()}.
+ *
+ * @throws IOException If an I/O error occurs
+ */
+ public void end() throws IOException {
+ clearAttributes(); // LUCENE-3849: don't consume dirty atts
+ if (hasAttribute(PositionIncrementAttribute.class)) {
+ getAttribute(PositionIncrementAttribute.class).setPositionIncrement(0);
+ }
+ }
+
+ /**
+ * This method is called by a consumer before it begins consumption using
+ * {@link #incrementToken()}.
+ * <p>
+ * Resets this stream to a clean state. Stateful implementations must implement
+ * this method so that they can be reused, just as if they had been created fresh.
+ * <p>
+ * If you override this method, always call {@code super.reset()}, otherwise
+ * some internal state will not be correctly reset (e.g., {@link Tokenizer} will
+ * throw {@link IllegalStateException} on further usage).
+ */
+ public void reset() throws IOException {}
+
+ /** Releases resources associated with this stream.
+ * <p>
+ * If you override this method, always call {@code super.close()}, otherwise
+ * some internal state will not be correctly reset (e.g., {@link Tokenizer} will
+ * throw {@link IllegalStateException} on reuse).
+ */
+ @Override
+ public void close() throws IOException {}
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
new file mode 100644
index 0000000000..c4f6f950f8
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
@@ -0,0 +1,274 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RollingBuffer;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.State;
+import org.apache.lucene.util.automaton.Transition;
+
+// TODO: maybe also toFST? then we can translate atts into FST outputs/weights
+
+/** Consumes a TokenStream and creates an {@link Automaton}
+ * where the transition labels are UTF8 bytes (or Unicode
+ * code points if unicodeArcs is true) from the {@link
+ * TermToBytesRefAttribute}. Between tokens we insert
+ * POS_SEP and for holes we insert HOLE.
+ *
+ * @lucene.experimental */
+public class TokenStreamToAutomaton {
+
+ private boolean preservePositionIncrements;
+ private boolean unicodeArcs;
+
+ /** Sole constructor. */
+ public TokenStreamToAutomaton() {
+ this.preservePositionIncrements = true;
+ }
+
+ /** Whether to generate holes in the automaton for missing positions, <code>true</code> by default. */
+ public void setPreservePositionIncrements(boolean enablePositionIncrements) {
+ this.preservePositionIncrements = enablePositionIncrements;
+ }
+
+ /** Whether to make transition labels Unicode code points instead of UTF8 bytes,
+ * <code>false</code> by default */
+ public void setUnicodeArcs(boolean unicodeArcs) {
+ this.unicodeArcs = unicodeArcs;
+ }
+
+ private static class Position implements RollingBuffer.Resettable {
+ // Any tokens that ended at our position arrive to this state:
+ State arriving;
+
+ // Any tokens that start at our position leave from this state:
+ State leaving;
+
+ @Override
+ public void reset() {
+ arriving = null;
+ leaving = null;
+ }
+ }
+
+ private static class Positions extends RollingBuffer<Position> {
+ @Override
+ protected Position newInstance() {
+ return new Position();
+ }
+ }
+
+ /** Subclass & implement this if you need to change the
+ * token (such as escaping certain bytes) before it's
+ * turned into a graph. */
+ protected BytesRef changeToken(BytesRef in) {
+ return in;
+ }
+
+ /** We create transition between two adjacent tokens. */
+ public static final int POS_SEP = 0x001f;
+
+ /** We add this arc to represent a hole. */
+ public static final int HOLE = 0x001e;
+
+ /** Pulls the graph (including {@link
+ * PositionLengthAttribute}) from the provided {@link
+ * TokenStream}, and creates the corresponding
+ * automaton where arcs are bytes (or Unicode code points
+ * if unicodeArcs = true) from each term. */
+ public Automaton toAutomaton(TokenStream in) throws IOException {
+ final Automaton a = new Automaton();
+ boolean deterministic = true;
+
+ final TermToBytesRefAttribute termBytesAtt = in.addAttribute(TermToBytesRefAttribute.class);
+ final PositionIncrementAttribute posIncAtt = in.addAttribute(PositionIncrementAttribute.class);
+ final PositionLengthAttribute posLengthAtt = in.addAttribute(PositionLengthAttribute.class);
+ final OffsetAttribute offsetAtt = in.addAttribute(OffsetAttribute.class);
+
+ final BytesRef term = termBytesAtt.getBytesRef();
+
+ in.reset();
+
+ // Only temporarily holds states ahead of our current
+ // position:
+
+ final RollingBuffer<Position> positions = new Positions();
+
+ int pos = -1;
+ Position posData = null;
+ int maxOffset = 0;
+ while (in.incrementToken()) {
+ int posInc = posIncAtt.getPositionIncrement();
+ if (!preservePositionIncrements && posInc > 1) {
+ posInc = 1;
+ }
+ assert pos > -1 || posInc > 0;
+
+ if (posInc > 0) {
+
+ // New node:
+ pos += posInc;
+
+ posData = positions.get(pos);
+ assert posData.leaving == null;
+
+ if (posData.arriving == null) {
+ // No token ever arrived to this position
+ if (pos == 0) {
+ // OK: this is the first token
+ posData.leaving = a.getInitialState();
+ } else {
+ // This means there's a hole (eg, StopFilter
+ // does this):
+ posData.leaving = new State();
+ addHoles(a.getInitialState(), positions, pos);
+ }
+ } else {
+ posData.leaving = new State();
+ posData.arriving.addTransition(new Transition(POS_SEP, posData.leaving));
+ if (posInc > 1) {
+ // A token spanned over a hole; add holes
+ // "under" it:
+ addHoles(a.getInitialState(), positions, pos);
+ }
+ }
+ positions.freeBefore(pos);
+ } else {
+ // note: this isn't necessarily true. its just that we aren't surely det.
+ // we could optimize this further (e.g. buffer and sort synonyms at a position)
+ // but thats probably overkill. this is cheap and dirty
+ deterministic = false;
+ }
+
+ final int endPos = pos + posLengthAtt.getPositionLength();
+
+ termBytesAtt.fillBytesRef();
+ final BytesRef termUTF8 = changeToken(term);
+ int[] termUnicode = null;
+ final Position endPosData = positions.get(endPos);
+ if (endPosData.arriving == null) {
+ endPosData.arriving = new State();
+ }
+
+ State state = posData.leaving;
+ int termLen;
+ if (unicodeArcs) {
+ final String utf16 = termUTF8.utf8ToString();
+ termUnicode = new int[utf16.codePointCount(0, utf16.length())];
+ termLen = termUnicode.length;
+ for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp))
+ termUnicode[j++] = cp = utf16.codePointAt(i);
+ } else {
+ termLen = termUTF8.length;
+ }
+
+ for(int byteIDX=0;byteIDX<termLen;byteIDX++) {
+ final State nextState = byteIDX == termLen-1 ? endPosData.arriving : new State();
+ int c;
+ if (unicodeArcs) {
+ c = termUnicode[byteIDX];
+ } else {
+ c = termUTF8.bytes[termUTF8.offset + byteIDX] & 0xff;
+ }
+ state.addTransition(new Transition(c, nextState));
+ state = nextState;
+ }
+
+ maxOffset = Math.max(maxOffset, offsetAtt.endOffset());
+ }
+
+ in.end();
+ State endState = null;
+ if (offsetAtt.endOffset() > maxOffset) {
+ endState = new State();
+ endState.setAccept(true);
+ }
+
+ pos++;
+ while (pos <= positions.getMaxPos()) {
+ posData = positions.get(pos);
+ if (posData.arriving != null) {
+ if (endState != null) {
+ posData.arriving.addTransition(new Transition(POS_SEP, endState));
+ } else {
+ posData.arriving.setAccept(true);
+ }
+ }
+ pos++;
+ }
+
+ //toDot(a);
+ a.setDeterministic(deterministic);
+ return a;
+ }
+
+ // for debugging!
+ /*
+ private static void toDot(Automaton a) throws IOException {
+ final String s = a.toDot();
+ Writer w = new OutputStreamWriter(new FileOutputStream("/tmp/out.dot"));
+ w.write(s);
+ w.close();
+ System.out.println("TEST: saved to /tmp/out.dot");
+ }
+ */
+
+ private static void addHoles(State startState, RollingBuffer<Position> positions, int pos) {
+ Position posData = positions.get(pos);
+ Position prevPosData = positions.get(pos-1);
+
+ while(posData.arriving == null || prevPosData.leaving == null) {
+ if (posData.arriving == null) {
+ posData.arriving = new State();
+ posData.arriving.addTransition(new Transition(POS_SEP, posData.leaving));
+ }
+ if (prevPosData.leaving == null) {
+ if (pos == 1) {
+ prevPosData.leaving = startState;
+ } else {
+ prevPosData.leaving = new State();
+ }
+ if (prevPosData.arriving != null) {
+ prevPosData.arriving.addTransition(new Transition(POS_SEP, prevPosData.leaving));
+ }
+ }
+ prevPosData.leaving.addTransition(new Transition(HOLE, posData.arriving));
+ pos--;
+ if (pos <= 0) {
+ break;
+ }
+ posData = prevPosData;
+ prevPosData = positions.get(pos-1);
+ }
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/Tokenizer.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/Tokenizer.java
new file mode 100644
index 0000000000..df6d811626
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/Tokenizer.java
@@ -0,0 +1,127 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeSource;
+
+import java.io.Reader;
+import java.io.IOException;
+
+/** A Tokenizer is a TokenStream whose input is a Reader.
+ <p>
+ This is an abstract class; subclasses must override {@link #incrementToken()}
+ <p>
+ NOTE: Subclasses overriding {@link #incrementToken()} must
+ call {@link AttributeSource#clearAttributes()} before
+ setting attributes.
+ */
+public abstract class Tokenizer extends TokenStream {
+ /** The text source for this Tokenizer. */
+ protected Reader input = ILLEGAL_STATE_READER;
+
+ /** Pending reader: not actually assigned to input until reset() */
+ private Reader inputPending = ILLEGAL_STATE_READER;
+
+ /** Construct a token stream processing the given input. */
+ protected Tokenizer(Reader input) {
+ if (input == null) {
+ throw new NullPointerException("input must not be null");
+ }
+ this.inputPending = input;
+ }
+
+ /** Construct a token stream processing the given input using the given AttributeFactory. */
+ protected Tokenizer(AttributeFactory factory, Reader input) {
+ super(factory);
+ if (input == null) {
+ throw new NullPointerException("input must not be null");
+ }
+ this.inputPending = input;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <b>NOTE:</b>
+ * The default implementation closes the input Reader, so
+ * be sure to call <code>super.close()</code> when overriding this method.
+ */
+ @Override
+ public void close() throws IOException {
+ input.close();
+ // LUCENE-2387: don't hold onto Reader after close, so
+ // GC can reclaim
+ inputPending = input = ILLEGAL_STATE_READER;
+ }
+
+ /** Return the corrected offset. If {@link #input} is a {@link CharFilter} subclass
+ * this method calls {@link CharFilter#correctOffset}, else returns <code>currentOff</code>.
+ * @param currentOff offset as seen in the output
+ * @return corrected offset based on the input
+ * @see CharFilter#correctOffset
+ */
+ protected final int correctOffset(int currentOff) {
+ return (input instanceof CharFilter) ? ((CharFilter) input).correctOffset(currentOff) : currentOff;
+ }
+
+ /** Expert: Set a new reader on the Tokenizer. Typically, an
+ * analyzer (in its tokenStream method) will use
+ * this to re-use a previously created tokenizer. */
+ public final void setReader(Reader input) throws IOException {
+ if (input == null) {
+ throw new NullPointerException("input must not be null");
+ } else if (this.input != ILLEGAL_STATE_READER) {
+ throw new IllegalStateException("TokenStream contract violation: close() call missing");
+ }
+ this.inputPending = input;
+ assert setReaderTestPoint();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ input = inputPending;
+ inputPending = ILLEGAL_STATE_READER;
+ }
+
+ // only used by assert, for testing
+ boolean setReaderTestPoint() {
+ return true;
+ }
+
+ private static final Reader ILLEGAL_STATE_READER = new Reader() {
+ @Override
+ public int read(char[] cbuf, int off, int len) {
+ throw new IllegalStateException("TokenStream contract violation: reset()/close() call missing, " +
+ "reset() called multiple times, or subclass does not call super.reset(). " +
+ "Please see Javadocs of TokenStream class for more information about the correct consuming workflow.");
+ }
+
+ @Override
+ public void close() {}
+ };
+}
+
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/package.html b/oak-lucene/src/main/java/org/apache/lucene/analysis/package.html
new file mode 100644
index 0000000000..be2d6e2587
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/package.html
@@ -0,0 +1,972 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>API and code to convert text into indexable/searchable tokens. Covers {@link org.apache.lucene.analysis.Analyzer} and related classes.</p>
+<h2>Parsing? Tokenization? Analysis!</h2>
+<p>
+Lucene, an indexing and search library, accepts only plain text input.
+<p>
+<h2>Parsing</h2>
+<p>
+Applications that build their search capabilities upon Lucene may support documents in various formats – HTML, XML, PDF, Word – just to name a few.
+Lucene does not care about the <i>Parsing</i> of these and other document formats, and it is the responsibility of the
+application using Lucene to use an appropriate <i>Parser</i> to convert the original format into plain text before passing that plain text to Lucene.
+<p>
+<h2>Tokenization</h2>
+<p>
+Plain text passed to Lucene for indexing goes through a process generally called tokenization. Tokenization is the process
+of breaking input text into small indexing elements – tokens.
+The way input text is broken into tokens heavily influences how people will then be able to search for that text.
+For instance, sentences beginnings and endings can be identified to provide for more accurate phrase
+and proximity searches (though sentence identification is not provided by Lucene).
+<p>
+ In some cases simply breaking the input text into tokens is not enough
+ – a deeper <i>Analysis</i> may be needed. Lucene includes both
+ pre- and post-tokenization analysis facilities.
+</p>
+<p>
+ Pre-tokenization analysis can include (but is not limited to) stripping
+ HTML markup, and transforming or removing text matching arbitrary patterns
+ or sets of fixed strings.
+</p>
+<p>
+ There are many post-tokenization steps that can be done, including
+ (but not limited to):
+</p>
+<ul>
+ <li><a href="http://en.wikipedia.org/wiki/Stemming">Stemming</a> –
+ Replacing words with their stems.
+ For instance with English stemming "bikes" is replaced with "bike";
+ now query "bike" can find both documents containing "bike" and those containing "bikes".
+ </li>
+ <li><a href="http://en.wikipedia.org/wiki/Stop_words">Stop Words Filtering</a> –
+ Common words like "the", "and" and "a" rarely add any value to a search.
+ Removing them shrinks the index size and increases performance.
+ It may also reduce some "noise" and actually improve search quality.
+ </li>
+ <li><a href="http://en.wikipedia.org/wiki/Text_normalization">Text Normalization</a> –
+ Stripping accents and other character markings can make for better searching.
+ </li>
+ <li><a href="http://en.wikipedia.org/wiki/Synonym">Synonym Expansion</a> –
+ Adding in synonyms at the same token position as the current word can mean better
+ matching when users search with words in the synonym set.
+ </li>
+</ul>
+<p>
+<h2>Core Analysis</h2>
+<p>
+ The analysis package provides the mechanism to convert Strings and Readers
+ into tokens that can be indexed by Lucene. There are four main classes in
+ the package from which all analysis processes are derived. These are:
+</p>
+<ul>
+ <li>
+ {@link org.apache.lucene.analysis.Analyzer} – An Analyzer is
+ responsible for building a
+ {@link org.apache.lucene.analysis.TokenStream} which can be consumed
+ by the indexing and searching processes. See below for more information
+ on implementing your own Analyzer.
+ </li>
+ <li>
+ CharFilter – CharFilter extends
+ {@link java.io.Reader} to perform pre-tokenization substitutions,
+ deletions, and/or insertions on an input Reader's text, while providing
+ corrected character offsets to account for these modifications. This
+ capability allows highlighting to function over the original text when
+ indexed tokens are created from CharFilter-modified text with offsets
+ that are not the same as those in the original text. Tokenizers'
+ constructors and reset() methods accept a CharFilter. CharFilters may
+ be chained to perform multiple pre-tokenization modifications.
+ </li>
+ <li>
+ {@link org.apache.lucene.analysis.Tokenizer} – A Tokenizer is a
+ {@link org.apache.lucene.analysis.TokenStream} and is responsible for
+ breaking up incoming text into tokens. In most cases, an Analyzer will
+ use a Tokenizer as the first step in the analysis process. However,
+ to modify text prior to tokenization, use a CharStream subclass (see
+ above).
+ </li>
+ <li>
+ {@link org.apache.lucene.analysis.TokenFilter} – A TokenFilter is
+ also a {@link org.apache.lucene.analysis.TokenStream} and is responsible
+ for modifying tokens that have been created by the Tokenizer. Common
+ modifications performed by a TokenFilter are: deletion, stemming, synonym
+ injection, and down casing. Not all Analyzers require TokenFilters.
+ </li>
+</ul>
+<h2>Hints, Tips and Traps</h2>
+<p>
+ The synergy between {@link org.apache.lucene.analysis.Analyzer} and
+ {@link org.apache.lucene.analysis.Tokenizer} is sometimes confusing. To ease
+ this confusion, some clarifications:
+</p>
+<ul>
+ <li>
+ The {@link org.apache.lucene.analysis.Analyzer} is responsible for the entire task of
+ <u>creating</u> tokens out of the input text, while the {@link org.apache.lucene.analysis.Tokenizer}
+ is only responsible for <u>breaking</u> the input text into tokens. Very likely, tokens created
+ by the {@link org.apache.lucene.analysis.Tokenizer} would be modified or even omitted
+ by the {@link org.apache.lucene.analysis.Analyzer} (via one or more
+ {@link org.apache.lucene.analysis.TokenFilter}s) before being returned.
+ </li>
+ <li>
+ {@link org.apache.lucene.analysis.Tokenizer} is a {@link org.apache.lucene.analysis.TokenStream},
+ but {@link org.apache.lucene.analysis.Analyzer} is not.
+ </li>
+ <li>
+ {@link org.apache.lucene.analysis.Analyzer} is "field aware", but
+ {@link org.apache.lucene.analysis.Tokenizer} is not.
+ </li>
+</ul>
+<p>
+ Lucene Java provides a number of analysis capabilities, the most commonly used one being the StandardAnalyzer.
+ Many applications will have a long and industrious life with nothing more
+ than the StandardAnalyzer. However, there are a few other classes/packages that are worth mentioning:
+</p>
+<ol>
+ <li>
+ PerFieldAnalyzerWrapper – Most Analyzers perform the same operation on all
+ {@link org.apache.lucene.document.Field}s. The PerFieldAnalyzerWrapper can be used to associate a different Analyzer with different
+ {@link org.apache.lucene.document.Field}s.
+ </li>
+ <li>
+ The analysis library located at the root of the Lucene distribution has a number of different Analyzer implementations to solve a variety
+ of different problems related to searching. Many of the Analyzers are designed to analyze non-English languages.
+ </li>
+ <li>
+ There are a variety of Tokenizer and TokenFilter implementations in this package. Take a look around, chances are someone has implemented what you need.
+ </li>
+</ol>
+<p>
+ Analysis is one of the main causes of performance degradation during indexing. Simply put, the more you analyze the slower the indexing (in most cases).
+ Perhaps your application would be just fine using the simple WhitespaceTokenizer combined with a StopFilter. The benchmark/ library can be useful
+ for testing out the speed of the analysis process.
+</p>
+<h2>Invoking the Analyzer</h2>
+<p>
+ Applications usually do not invoke analysis – Lucene does it for them:
+</p>
+<ul>
+ <li>
+ At indexing, as a consequence of
+ {@link org.apache.lucene.index.IndexWriter#addDocument(Iterable) addDocument(doc)},
+ the Analyzer in effect for indexing is invoked for each indexed field of the added document.
+ </li>
+ <li>
+ At search, a QueryParser may invoke the Analyzer during parsing. Note that for some queries, analysis does not
+ take place, e.g. wildcard queries.
+ </li>
+</ul>
+<p>
+ However an application might invoke Analysis of any text for testing or for any other purpose, something like:
+</p>
+<PRE class="prettyprint" id="analysis-workflow">
+ Version matchVersion = Version.LUCENE_XY; // Substitute desired Lucene version for XY
+ Analyzer analyzer = new StandardAnalyzer(matchVersion); // or any other analyzer
+ TokenStream ts = analyzer.tokenStream("myfield", new StringReader("some text goes here"));
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+
+ try {
+ ts.reset(); // Resets this stream to the beginning. (Required)
+ while (ts.incrementToken()) {
+ // Use {@link org.apache.lucene.util.AttributeSource#reflectAsString(boolean)}
+ // for token stream debugging.
+ System.out.println("token: " + ts.reflectAsString(true));
+
+ System.out.println("token start offset: " + offsetAtt.startOffset());
+ System.out.println(" token end offset: " + offsetAtt.endOffset());
+ }
+ ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
+ } finally {
+ ts.close(); // Release resources associated with this stream.
+ }
+</PRE>
+<h2>Indexing Analysis vs. Search Analysis</h2>
+<p>
+ Selecting the "correct" analyzer is crucial
+ for search quality, and can also affect indexing and search performance.
+ The "correct" analyzer differs between applications.
+ Lucene java's wiki page
+ <a href="http://wiki.apache.org/lucene-java/AnalysisParalysis">AnalysisParalysis</a>
+ provides some data on "analyzing your analyzer".
+ Here are some rules of thumb:
+ <ol>
+ <li>Test test test... (did we say test?)</li>
+ <li>Beware of over analysis – might hurt indexing performance.</li>
+ <li>Start with same analyzer for indexing and search, otherwise searches would not find what they are supposed to...</li>
+ <li>In some cases a different analyzer is required for indexing and search, for instance:
+ <ul>
+ <li>Certain searches require more stop words to be filtered. (I.e. more than those that were filtered at indexing.)</li>
+ <li>Query expansion by synonyms, acronyms, auto spell correction, etc.</li>
+ </ul>
+ This might sometimes require a modified analyzer – see the next section on how to do that.
+ </li>
+ </ol>
+</p>
+<h2>Implementing your own Analyzer</h2>
+<p>
+ Creating your own Analyzer is straightforward. Your Analyzer can wrap
+ existing analysis components — CharFilter(s) <i>(optional)</i>, a
+ Tokenizer, and TokenFilter(s) <i>(optional)</i> — or components you
+ create, or a combination of existing and newly created components. Before
+ pursuing this approach, you may find it worthwhile to explore the
+ <a href="{@docRoot}/../analyzers-common/overview-summary.html">analyzers-common</a> library and/or ask on the
+ <a href="http://lucene.apache.org/core/discussion.html"
+ >java-user@lucene.apache.org mailing list</a> first to see if what you
+ need already exists. If you are still committed to creating your own
+ Analyzer, have a look at the source code of any one of the many samples
+ located in this package.
+</p>
+<p>
+ The following sections discuss some aspects of implementing your own analyzer.
+</p>
+<h3>Field Section Boundaries</h3>
+<p>
+ When {@link org.apache.lucene.document.Document#add(org.apache.lucene.index.IndexableField) document.add(field)}
+ is called multiple times for the same field name, we could say that each such call creates a new
+ section for that field in that document.
+ In fact, a separate call to
+ {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)}
+ would take place for each of these so called "sections".
+ However, the default Analyzer behavior is to treat all these sections as one large section.
+ This allows phrase search and proximity search to seamlessly cross
+ boundaries between these "sections".
+ In other words, if a certain field "f" is added like this:
+</p>
+<PRE class="prettyprint">
+ document.add(new Field("f","first ends",...);
+ document.add(new Field("f","starts two",...);
+ indexWriter.addDocument(document);
+</PRE>
+<p>
+ Then, a phrase search for "ends starts" would find that document.
+ Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections",
+ simply by overriding
+ {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
+</p>
+<PRE class="prettyprint">
+ Version matchVersion = Version.LUCENE_XY; // Substitute desired Lucene version for XY
+ Analyzer myAnalyzer = new StandardAnalyzer(matchVersion) {
+ public int getPositionIncrementGap(String fieldName) {
+ return 10;
+ }
+ };
+</PRE>
+<h3>Token Position Increments</h3>
+<p>
+ By default, all tokens created by Analyzers and Tokenizers have a
+ {@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#getPositionIncrement() position increment} of one.
+ This means that the position stored for that token in the index would be one more than
+ that of the previous token.
+ Recall that phrase and proximity searches rely on position info.
+</p>
+<p>
+ If the selected analyzer filters the stop words "is" and "the", then for a document
+ containing the string "blue is the sky", only the tokens "blue", "sky" are indexed,
+ with position("sky") = 3 + position("blue"). Now, a phrase query "blue is the sky"
+ would find that document, because the same analyzer filters the same stop words from
+ that query. But the phrase query "blue sky" would not find that document because the
+ position increment between "blue" and "sky" is only 1.
+</p>
+<p>
+ If this behavior does not fit the application needs, the query parser needs to be
+ configured to not take position increments into account when generating phrase queries.
+</p>
+<p>
+ Note that a StopFilter MUST increment the position increment in order not to generate corrupt
+ tokenstream graphs. Here is the logic used by StopFilter to increment positions when filtering out tokens:
+</p>
+<PRE class="prettyprint">
+ public TokenStream tokenStream(final String fieldName, Reader reader) {
+ final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
+ TokenStream res = new TokenStream() {
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ public boolean incrementToken() throws IOException {
+ int extraIncrement = 0;
+ while (true) {
+ boolean hasNext = ts.incrementToken();
+ if (hasNext) {
+ if (stopWords.contains(termAtt.toString())) {
+ extraIncrement += posIncrAtt.getPositionIncrement(); // filter this word
+ continue;
+ }
+ if (extraIncrement>0) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+extraIncrement);
+ }
+ }
+ return hasNext;
+ }
+ }
+ };
+ return res;
+ }
+</PRE>
+<p>
+ A few more use cases for modifying position increments are:
+</p>
+<ol>
+ <li>Inhibiting phrase and proximity matches in sentence boundaries – for this, a tokenizer that
+ identifies a new sentence can add 1 to the position increment of the first token of the new sentence.</li>
+ <li>Injecting synonyms – here, synonyms of a token should be added after that token,
+ and their position increment should be set to 0.
+ As result, all synonyms of a token would be considered to appear in exactly the
+ same position as that token, and so would they be seen by phrase and proximity searches.</li>
+</ol>
+
+<h3>Token Position Length</h3>
+<p>
+ By default, all tokens created by Analyzers and Tokenizers have a
+ {@link org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute#getPositionLength() position length} of one.
+ This means that the token occupies a single position. This attribute is not indexed
+ and thus not taken into account for positional queries, but is used by eg. suggesters.
+</p>
+<p>
+ The main use case for positions lengths is multi-word synonyms. With single-word
+ synonyms, setting the position increment to 0 is enough to denote the fact that two
+ words are synonyms, for example:
+</p>
+<table>
+<tr><td>Term</td><td>red</td><td>magenta</td></tr>
+<tr><td>Position increment</td><td>1</td><td>0</td></tr>
+</table>
+<p>
+ Given that position(magenta) = 0 + position(red), they are at the same position, so anything
+ working with analyzers will return the exact same result if you replace "magenta" with "red"
+ in the input. However, multi-word synonyms are more tricky. Let's say that you want to build
+ a TokenStream where "IBM" is a synonym of "Internal Business Machines". Position increments
+ are not enough anymore:
+</p>
+<table>
+<tr><td>Term</td><td>IBM</td><td>International</td><td>Business</td><td>Machines</td></tr>
+<tr><td>Position increment</td><td>1</td><td>0</td><td>1</td><td>1</td></tr>
+</table>
+<p>
+ The problem with this token stream is that "IBM" is at the same position as "International"
+ although it is a synonym with "International Business Machines" as a whole. Setting
+ the position increment of "Business" and "Machines" to 0 wouldn't help as it would mean
+ than "International" is a synonym of "Business". The only way to solve this issue is to
+ make "IBM" span across 3 positions, this is where position lengths come to rescue.
+</p>
+<table>
+<tr><td>Term</td><td>IBM</td><td>International</td><td>Business</td><td>Machines</td></tr>
+<tr><td>Position increment</td><td>1</td><td>0</td><td>1</td><td>1</td></tr>
+<tr><td>Position length</td><td>3</td><td>1</td><td>1</td><td>1</td></tr>
+</table>
+<p>
+ This new attribute makes clear that "IBM" and "International Business Machines" start and end
+ at the same positions.
+</p>
+<a name="corrupt" />
+<h3>How to not write corrupt token streams</h3>
+<p>
+ There are a few rules to observe when writing custom Tokenizers and TokenFilters:
+</p>
+<ul>
+ <li>The first position increment must be > 0.</li>
+ <li>Positions must not go backward.</li>
+ <li>Tokens that have the same start position must have the same start offset.</li>
+ <li>Tokens that have the same end position (taking into account the
+ position length) must have the same end offset.</li>
+ <li>Tokenizers must call {@link
+ org.apache.lucene.util.AttributeSource#clearAttributes()} in
+ incrementToken().</li>
+ <li>Tokenizers must override {@link
+ org.apache.lucene.analysis.TokenStream#end()}, and pass the final
+ offset (the total number of input characters processed) to both
+ parameters of {@link org.apache.lucene.analysis.tokenattributes.OffsetAttribute#setOffset(int, int)}.</li>
+</ul>
+<p>
+ Although these rules might seem easy to follow, problems can quickly happen when chaining
+ badly implemented filters that play with positions and offsets, such as synonym or n-grams
+ filters. Here are good practices for writing correct filters:
+</p>
+<ul>
+ <li>Token filters should not modify offsets. If you feel that your filter would need to modify offsets, then it should probably be implemented as a tokenizer.</li>
+ <li>Token filters should not insert positions. If a filter needs to add tokens, then they should all have a position increment of 0.</li>
+ <li>When they add tokens, token filters should call {@link org.apache.lucene.util.AttributeSource#clearAttributes()} first.</li>
+ <li>When they remove tokens, token filters should increment the position increment of the following token.</li>
+ <li>Token filters should preserve position lengths.</li>
+</ul>
+<h2>TokenStream API</h2>
+<p>
+ "Flexible Indexing" summarizes the effort of making the Lucene indexer
+ pluggable and extensible for custom index formats. A fully customizable
+ indexer means that users will be able to store custom data structures on
+ disk. Therefore an API is necessary that can transport custom types of
+ data from the documents to the indexer.
+</p>
+<h3>Attribute and AttributeSource</h3>
+<p>
+ Classes {@link org.apache.lucene.util.Attribute} and
+ {@link org.apache.lucene.util.AttributeSource} serve as the basis upon which
+ the analysis elements of "Flexible Indexing" are implemented. An Attribute
+ holds a particular piece of information about a text token. For example,
+ {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}
+ contains the term text of a token, and
+ {@link org.apache.lucene.analysis.tokenattributes.OffsetAttribute} contains
+ the start and end character offsets of a token. An AttributeSource is a
+ collection of Attributes with a restriction: there may be only one instance
+ of each attribute type. TokenStream now extends AttributeSource, which means
+ that one can add Attributes to a TokenStream. Since TokenFilter extends
+ TokenStream, all filters are also AttributeSources.
+</p>
+<p>
+ Lucene provides seven Attributes out of the box:
+</p>
+<table rules="all" frame="box" cellpadding="3">
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}</td>
+ <td>
+ The term text of a token. Implements {@link java.lang.CharSequence}
+ (providing methods length() and charAt(), and allowing e.g. for direct
+ use with regular expression {@link java.util.regex.Matcher}s) and
+ {@link java.lang.Appendable} (allowing the term text to be appended to.)
+ </td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.OffsetAttribute}</td>
+ <td>The start and end offset of a token in characters.</td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute}</td>
+ <td>See above for detailed information about position increment.</td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute}</td>
+ <td>The number of positions occupied by a token.</td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.PayloadAttribute}</td>
+ <td>The payload that a Token can optionally have.</td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.TypeAttribute}</td>
+ <td>The type of the token. Default is 'word'.</td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.FlagsAttribute}</td>
+ <td>Optional flags a token can have.</td>
+ </tr>
+ <tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.KeywordAttribute}</td>
+ <td>
+ Keyword-aware TokenStreams/-Filters skip modification of tokens that
+ return true from this attribute's isKeyword() method.
+ </td>
+ </tr>
+</table>
+<h3>More Requirements for Analysis Component Classes</h3>
+Due to the historical development of the API, there are some perhaps
+less than obvious requirements to implement analysis components
+classes.
+<h4 id="analysis-lifetime">Token Stream Lifetime</h4>
+The code fragment of the <a href="#analysis-workflow">analysis workflow
+protocol</a> above shows a token stream being obtained, used, and then
+left for garbage. However, that does not mean that the components of
+that token stream will, in fact, be discarded. The default is just the
+opposite. {@link org.apache.lucene.analysis.Analyzer} applies a reuse
+strategy to the tokenizer and the token filters. It will reuse
+them. For each new input, it calls {@link org.apache.lucene.analysis.Tokenizer#setReader(java.io.Reader)}
+to set the input. Your components must be prepared for this scenario,
+as described below.
+<h4>Tokenizer</h4>
+<ul>
+ <li>
+ You should create your tokenizer class by extending {@link org.apache.lucene.analysis.Tokenizer}.
+ </li>
+ <li>
+ Your tokenizer must <strong>never</strong> make direct use of the
+ {@link java.io.Reader} supplied to its constructor(s). (A future
+ release of Apache Lucene may remove the reader parameters from the
+ Tokenizer constructors.)
+ {@link org.apache.lucene.analysis.Tokenizer} wraps the reader in an
+ object that helps enforce that applications comply with the <a
+ href="#analysis-workflow">analysis workflow</a>. Thus, your class
+ should only reference the input via the protected 'input' field
+ of Tokenizer.
+ </li>
+ <li>
+ Your tokenizer <strong>must</strong> override {@link org.apache.lucene.analysis.TokenStream#end()}.
+ Your implementation <strong>must</strong> call
+ <code>super.end()</code>. It must set a correct final offset into
+ the offset attribute, and finish up and other attributes to reflect
+ the end of the stream.
+ </li>
+ <li>
+ If your tokenizer overrides {@link org.apache.lucene.analysis.TokenStream#reset()}
+ or {@link org.apache.lucene.analysis.TokenStream#close()}, it
+ <strong>must</strong> call the corresponding superclass method.
+ </li>
+</ul>
+<h4>Token Filter</h4>
+ You should create your token filter class by extending {@link org.apache.lucene.analysis.TokenFilter}.
+ If your token filter overrides {@link org.apache.lucene.analysis.TokenStream#reset()},
+ {@link org.apache.lucene.analysis.TokenStream#end()}
+ or {@link org.apache.lucene.analysis.TokenStream#close()}, it
+ <strong>must</strong> call the corresponding superclass method.
+<h4>Creating delegates</h4>
+ Forwarding classes (those which extend {@link org.apache.lucene.analysis.Tokenizer} but delegate
+ selected logic to another tokenizer) must also set the reader to the delegate in the overridden
+ {@link org.apache.lucene.analysis.Tokenizer#reset()} method, e.g.:
+ <pre class="prettyprint">
+ public class ForwardingTokenizer extends Tokenizer {
+ private Tokenizer delegate;
+ ...
+ {@literal @Override}
+ public void reset() {
+ super.reset();
+ delegate.setReader(this.input);
+ delegate.reset();
+ }
+ }
+ </pre>
+<h3>Testing Your Analysis Component</h3>
+<p>
+ The lucene-test-framework component defines
+ <a href="{@docRoot}/../test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.html">BaseTokenStreamTestCase</a>. By extending
+ this class, you can create JUnit tests that validate that your
+ Analyzer and/or analysis components correctly implement the
+ protocol. The checkRandomData methods of that class are particularly effective in flushing out errors.
+</p>
+<h3>Using the TokenStream API</h3>
+There are a few important things to know in order to use the new API efficiently which are summarized here. You may want
+to walk through the example below first and come back to this section afterwards.
+<ol><li>
+Please keep in mind that an AttributeSource can only have one instance of a particular Attribute. Furthermore, if
+a chain of a TokenStream and multiple TokenFilters is used, then all TokenFilters in that chain share the Attributes
+with the TokenStream.
+</li>
+<br>
+<li>
+Attribute instances are reused for all tokens of a document. Thus, a TokenStream/-Filter needs to update
+the appropriate Attribute(s) in incrementToken(). The consumer, commonly the Lucene indexer, consumes the data in the
+Attributes and then calls incrementToken() again until it returns false, which indicates that the end of the stream
+was reached. This means that in each call of incrementToken() a TokenStream/-Filter can safely overwrite the data in
+the Attribute instances.
+</li>
+<br>
+<li>
+For performance reasons a TokenStream/-Filter should add/get Attributes during instantiation; i.e., create an attribute in the
+constructor and store references to it in an instance variable. Using an instance variable instead of calling addAttribute()/getAttribute()
+in incrementToken() will avoid attribute lookups for every token in the document.
+</li>
+<br>
+<li>
+All methods in AttributeSource are idempotent, which means calling them multiple times always yields the same
+result. This is especially important to know for addAttribute(). The method takes the <b>type</b> (<code>Class</code>)
+of an Attribute as an argument and returns an <b>instance</b>. If an Attribute of the same type was previously added, then
+the already existing instance is returned, otherwise a new instance is created and returned. Therefore TokenStreams/-Filters
+can safely call addAttribute() with the same Attribute type multiple times. Even consumers of TokenStreams should
+normally call addAttribute() instead of getAttribute(), because it would not fail if the TokenStream does not have this
+Attribute (getAttribute() would throw an IllegalArgumentException, if the Attribute is missing). More advanced code
+could simply check with hasAttribute(), if a TokenStream has it, and may conditionally leave out processing for
+extra performance.
+</li></ol>
+<h3>Example</h3>
+<p>
+ In this example we will create a WhiteSpaceTokenizer and use a LengthFilter to suppress all words that have
+ only two or fewer characters. The LengthFilter is part of the Lucene core and its implementation will be explained
+ here to illustrate the usage of the TokenStream API.
+</p>
+<p>
+ Then we will develop a custom Attribute, a PartOfSpeechAttribute, and add another filter to the chain which
+ utilizes the new custom attribute, and call it PartOfSpeechTaggingFilter.
+</p>
+<h4>Whitespace tokenization</h4>
+<pre class="prettyprint">
+public class MyAnalyzer extends Analyzer {
+
+ private Version matchVersion;
+
+ public MyAnalyzer(Version matchVersion) {
+ this.matchVersion = matchVersion;
+ }
+
+ {@literal @Override}
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new WhitespaceTokenizer(matchVersion, reader));
+ }
+
+ public static void main(String[] args) throws IOException {
+ // text to tokenize
+ final String text = "This is a demo of the TokenStream API";
+
+ Version matchVersion = Version.LUCENE_XY; // Substitute desired Lucene version for XY
+ MyAnalyzer analyzer = new MyAnalyzer(matchVersion);
+ TokenStream stream = analyzer.tokenStream("field", new StringReader(text));
+
+ // get the CharTermAttribute from the TokenStream
+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+
+ try {
+ stream.reset();
+
+ // print all tokens until stream is exhausted
+ while (stream.incrementToken()) {
+ System.out.println(termAtt.toString());
+ }
+
+ stream.end();
+ } finally {
+ stream.close();
+ }
+ }
+}
+</pre>
+In this easy example a simple white space tokenization is performed. In main() a loop consumes the stream and
+prints the term text of the tokens by accessing the CharTermAttribute that the WhitespaceTokenizer provides.
+Here is the output:
+<pre>
+This
+is
+a
+demo
+of
+the
+new
+TokenStream
+API
+</pre>
+<h4>Adding a LengthFilter</h4>
+We want to suppress all tokens that have 2 or less characters. We can do that
+easily by adding a LengthFilter to the chain. Only the
+<code>createComponents()</code> method in our analyzer needs to be changed:
+<pre class="prettyprint">
+ {@literal @Override}
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ final Tokenizer source = new WhitespaceTokenizer(matchVersion, reader);
+ TokenStream result = new LengthFilter(true, source, 3, Integer.MAX_VALUE);
+ return new TokenStreamComponents(source, result);
+ }
+</pre>
+Note how now only words with 3 or more characters are contained in the output:
+<pre>
+This
+demo
+the
+new
+TokenStream
+API
+</pre>
+Now let's take a look how the LengthFilter is implemented:
+<pre class="prettyprint">
+public final class LengthFilter extends FilteringTokenFilter {
+
+ private final int min;
+ private final int max;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ /**
+ * Create a new LengthFilter. This will filter out tokens whose
+ * CharTermAttribute is either too short
+ * (< min) or too long (> max).
+ * @param version the Lucene match version
+ * @param in the TokenStream to consume
+ * @param min the minimum length
+ * @param max the maximum length
+ */
+ public LengthFilter(Version version, TokenStream in, int min, int max) {
+ super(version, in);
+ this.min = min;
+ this.max = max;
+ }
+
+ {@literal @Override}
+ public boolean accept() {
+ final int len = termAtt.length();
+ return (len >= min && len <= max);
+ }
+
+}
+</pre>
+<p>
+ In LengthFilter, the CharTermAttribute is added and stored in the instance
+ variable <code>termAtt</code>. Remember that there can only be a single
+ instance of CharTermAttribute in the chain, so in our example the
+ <code>addAttribute()</code> call in LengthFilter returns the
+ CharTermAttribute that the WhitespaceTokenizer already added.
+</p>
+<p>
+ The tokens are retrieved from the input stream in FilteringTokenFilter's
+ <code>incrementToken()</code> method (see below), which calls LengthFilter's
+ <code>accept()</code> method. By looking at the term text in the
+ CharTermAttribute, the length of the term can be determined and tokens that
+ are either too short or too long are skipped. Note how
+ <code>accept()</code> can efficiently access the instance variable; no
+ attribute lookup is necessary. The same is true for the consumer, which can
+ simply use local references to the Attributes.
+</p>
+<p>
+ LengthFilter extends FilteringTokenFilter:
+</p>
+
+<pre class="prettyprint">
+public abstract class FilteringTokenFilter extends TokenFilter {
+
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ /**
+ * Create a new FilteringTokenFilter.
+ * @param in the TokenStream to consume
+ */
+ public FilteringTokenFilter(Version version, TokenStream in) {
+ super(in);
+ }
+
+ /** Override this method and return if the current input token should be returned by incrementToken. */
+ protected abstract boolean accept() throws IOException;
+
+ {@literal @Override}
+ public final boolean incrementToken() throws IOException {
+ int skippedPositions = 0;
+ while (input.incrementToken()) {
+ if (accept()) {
+ if (skippedPositions != 0) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
+ }
+ return true;
+ }
+ skippedPositions += posIncrAtt.getPositionIncrement();
+ }
+ // reached EOS -- return false
+ return false;
+ }
+
+ {@literal @Override}
+ public void reset() throws IOException {
+ super.reset();
+ }
+
+}
+</pre>
+
+<h4>Adding a custom Attribute</h4>
+Now we're going to implement our own custom Attribute for part-of-speech tagging and call it consequently
+<code>PartOfSpeechAttribute</code>. First we need to define the interface of the new Attribute:
+<pre class="prettyprint">
+ public interface PartOfSpeechAttribute extends Attribute {
+ public static enum PartOfSpeech {
+ Noun, Verb, Adjective, Adverb, Pronoun, Preposition, Conjunction, Article, Unknown
+ }
+
+ public void setPartOfSpeech(PartOfSpeech pos);
+
+ public PartOfSpeech getPartOfSpeech();
+ }
+</pre>
+<p>
+ Now we also need to write the implementing class. The name of that class is important here: By default, Lucene
+ checks if there is a class with the name of the Attribute with the suffix 'Impl'. In this example, we would
+ consequently call the implementing class <code>PartOfSpeechAttributeImpl</code>.
+</p>
+<p>
+ This should be the usual behavior. However, there is also an expert-API that allows changing these naming conventions:
+ {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. The factory accepts an Attribute interface as argument
+ and returns an actual instance. You can implement your own factory if you need to change the default behavior.
+</p>
+<p>
+ Now here is the actual class that implements our new Attribute. Notice that the class has to extend
+ {@link org.apache.lucene.util.AttributeImpl}:
+</p>
+<pre class="prettyprint">
+public final class PartOfSpeechAttributeImpl extends AttributeImpl
+ implements PartOfSpeechAttribute {
+
+ private PartOfSpeech pos = PartOfSpeech.Unknown;
+
+ public void setPartOfSpeech(PartOfSpeech pos) {
+ this.pos = pos;
+ }
+
+ public PartOfSpeech getPartOfSpeech() {
+ return pos;
+ }
+
+ {@literal @Override}
+ public void clear() {
+ pos = PartOfSpeech.Unknown;
+ }
+
+ {@literal @Override}
+ public void copyTo(AttributeImpl target) {
+ ((PartOfSpeechAttribute) target).setPartOfSpeech(pos);
+ }
+}
+</pre>
+<p>
+ This is a simple Attribute implementation has only a single variable that
+ stores the part-of-speech of a token. It extends the
+ <code>AttributeImpl</code> class and therefore implements its abstract methods
+ <code>clear()</code> and <code>copyTo()</code>. Now we need a TokenFilter that
+ can set this new PartOfSpeechAttribute for each token. In this example we
+ show a very naive filter that tags every word with a leading upper-case letter
+ as a 'Noun' and all other words as 'Unknown'.
+</p>
+<pre class="prettyprint">
+ public static class PartOfSpeechTaggingFilter extends TokenFilter {
+ PartOfSpeechAttribute posAtt = addAttribute(PartOfSpeechAttribute.class);
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ protected PartOfSpeechTaggingFilter(TokenStream input) {
+ super(input);
+ }
+
+ public boolean incrementToken() throws IOException {
+ if (!input.incrementToken()) {return false;}
+ posAtt.setPartOfSpeech(determinePOS(termAtt.buffer(), 0, termAtt.length()));
+ return true;
+ }
+
+ // determine the part of speech for the given term
+ protected PartOfSpeech determinePOS(char[] term, int offset, int length) {
+ // naive implementation that tags every uppercased word as noun
+ if (length > 0 && Character.isUpperCase(term[0])) {
+ return PartOfSpeech.Noun;
+ }
+ return PartOfSpeech.Unknown;
+ }
+ }
+</pre>
+<p>
+ Just like the LengthFilter, this new filter stores references to the
+ attributes it needs in instance variables. Notice how you only need to pass
+ in the interface of the new Attribute and instantiating the correct class
+ is automatically taken care of.
+</p>
+<p>Now we need to add the filter to the chain in MyAnalyzer:</p>
+<pre class="prettyprint">
+ {@literal @Override}
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ final Tokenizer source = new WhitespaceTokenizer(matchVersion, reader);
+ TokenStream result = new LengthFilter(true, source, 3, Integer.MAX_VALUE);
+ result = new PartOfSpeechTaggingFilter(result);
+ return new TokenStreamComponents(source, result);
+ }
+</pre>
+Now let's look at the output:
+<pre>
+This
+demo
+the
+new
+TokenStream
+API
+</pre>
+Apparently it hasn't changed, which shows that adding a custom attribute to a TokenStream/Filter chain does not
+affect any existing consumers, simply because they don't know the new Attribute. Now let's change the consumer
+to make use of the new PartOfSpeechAttribute and print it out:
+<pre class="prettyprint">
+ public static void main(String[] args) throws IOException {
+ // text to tokenize
+ final String text = "This is a demo of the TokenStream API";
+
+ MyAnalyzer analyzer = new MyAnalyzer();
+ TokenStream stream = analyzer.tokenStream("field", new StringReader(text));
+
+ // get the CharTermAttribute from the TokenStream
+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+
+ // get the PartOfSpeechAttribute from the TokenStream
+ PartOfSpeechAttribute posAtt = stream.addAttribute(PartOfSpeechAttribute.class);
+
+ try {
+ stream.reset();
+
+ // print all tokens until stream is exhausted
+ while (stream.incrementToken()) {
+ System.out.println(termAtt.toString() + ": " + posAtt.getPartOfSpeech());
+ }
+
+ stream.end();
+ } finally {
+ stream.close();
+ }
+ }
+</pre>
+The change that was made is to get the PartOfSpeechAttribute from the TokenStream and print out its contents in
+the while loop that consumes the stream. Here is the new output:
+<pre>
+This: Noun
+demo: Unknown
+the: Unknown
+new: Unknown
+TokenStream: Noun
+API: Noun
+</pre>
+Each word is now followed by its assigned PartOfSpeech tag. Of course this is a naive
+part-of-speech tagging. The word 'This' should not even be tagged as noun; it is only spelled capitalized because it
+is the first word of a sentence. Actually this is a good opportunity for an exercise. To practice the usage of the new
+API the reader could now write an Attribute and TokenFilter that can specify for each word if it was the first token
+of a sentence or not. Then the PartOfSpeechTaggingFilter can make use of this knowledge and only tag capitalized words
+as nouns if not the first word of a sentence (we know, this is still not a correct behavior, but hey, it's a good exercise).
+As a small hint, this is how the new Attribute class could begin:
+<pre class="prettyprint">
+ public class FirstTokenOfSentenceAttributeImpl extends AttributeImpl
+ implements FirstTokenOfSentenceAttribute {
+
+ private boolean firstToken;
+
+ public void setFirstToken(boolean firstToken) {
+ this.firstToken = firstToken;
+ }
+
+ public boolean getFirstToken() {
+ return firstToken;
+ }
+
+ {@literal @Override}
+ public void clear() {
+ firstToken = false;
+ }
+
+ ...
+</pre>
+<h4>Adding a CharFilter chain</h4>
+Analyzers take Java {@link java.io.Reader}s as input. Of course you can wrap your Readers with {@link java.io.FilterReader}s
+to manipulate content, but this would have the big disadvantage that character offsets might be inconsistent with your original
+text.
+<p>
+{@link org.apache.lucene.analysis.CharFilter} is designed to allow you to pre-process input like a FilterReader would, but also
+preserve the original offsets associated with those characters. This way mechanisms like highlighting still work correctly.
+CharFilters can be chained.
+<p>
+Example:
+<pre class="prettyprint">
+public class MyAnalyzer extends Analyzer {
+
+ {@literal @Override}
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MyTokenizer(reader));
+ }
+
+ {@literal @Override}
+ protected Reader initReader(String fieldName, Reader reader) {
+ // wrap the Reader in a CharFilter chain.
+ return new SecondCharFilter(new FirstCharFilter(reader));
+ }
+}
+</pre>
+</body>
+</html>
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java
new file mode 100644
index 0000000000..073987d218
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java
@@ -0,0 +1,106 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+/**
+ * The term text of a Token.
+ */
+public interface CharTermAttribute extends Attribute, CharSequence, Appendable {
+
+ /** Copies the contents of buffer, starting at offset for
+ * length characters, into the termBuffer array.
+ * @param buffer the buffer to copy
+ * @param offset the index in the buffer of the first character to copy
+ * @param length the number of characters to copy
+ */
+ public void copyBuffer(char[] buffer, int offset, int length);
+
+ /** Returns the internal termBuffer character array which
+ * you can then directly alter. If the array is too
+ * small for your token, use {@link
+ * #resizeBuffer(int)} to increase it. After
+ * altering the buffer be sure to call {@link
+ * #setLength} to record the number of valid
+ * characters that were placed into the termBuffer.
+ * <p>
+ * <b>NOTE</b>: The returned buffer may be larger than
+ * the valid {@link #length()}.
+ */
+ public char[] buffer();
+
+ /** Grows the termBuffer to at least size newSize, preserving the
+ * existing content.
+ * @param newSize minimum size of the new termBuffer
+ * @return newly created termBuffer with length >= newSize
+ */
+ public char[] resizeBuffer(int newSize);
+
+ /** Set number of valid characters (length of the term) in
+ * the termBuffer array. Use this to truncate the termBuffer
+ * or to synchronize with external manipulation of the termBuffer.
+ * Note: to grow the size of the array,
+ * use {@link #resizeBuffer(int)} first.
+ * @param length the truncated length
+ */
+ public CharTermAttribute setLength(int length);
+
+ /** Sets the length of the termBuffer to zero.
+ * Use this method before appending contents
+ * using the {@link Appendable} interface.
+ */
+ public CharTermAttribute setEmpty();
+
+ // the following methods are redefined to get rid of IOException declaration:
+ @Override
+ public CharTermAttribute append(CharSequence csq);
+ @Override
+ public CharTermAttribute append(CharSequence csq, int start, int end);
+ @Override
+ public CharTermAttribute append(char c);
+
+ /** Appends the specified {@code String} to this character sequence.
+ * <p>The characters of the {@code String} argument are appended, in order, increasing the length of
+ * this sequence by the length of the argument. If argument is {@code null}, then the four
+ * characters {@code "null"} are appended.
+ */
+ public CharTermAttribute append(String s);
+
+ /** Appends the specified {@code StringBuilder} to this character sequence.
+ * <p>The characters of the {@code StringBuilder} argument are appended, in order, increasing the length of
+ * this sequence by the length of the argument. If argument is {@code null}, then the four
+ * characters {@code "null"} are appended.
+ */
+ public CharTermAttribute append(StringBuilder sb);
+
+ /** Appends the contents of the other {@code CharTermAttribute} to this character sequence.
+ * <p>The characters of the {@code CharTermAttribute} argument are appended, in order, increasing the length of
+ * this sequence by the length of the argument. If argument is {@code null}, then the four
+ * characters {@code "null"} are appended.
+ */
+ public CharTermAttribute append(CharTermAttribute termAtt);
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
new file mode 100644
index 0000000000..5e964f7f70
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
@@ -0,0 +1,291 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.CharBuffer;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.AttributeReflector;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.UnicodeUtil;
+
+/** Default implementation of {@link CharTermAttribute}. */
+public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttribute, TermToBytesRefAttribute, Cloneable {
+ private static int MIN_BUFFER_SIZE = 10;
+
+ private char[] termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
+ private int termLength = 0;
+
+ /** Initialize this attribute with empty term text */
+ public CharTermAttributeImpl() {}
+
+ @Override
+ public final void copyBuffer(char[] buffer, int offset, int length) {
+ growTermBuffer(length);
+ System.arraycopy(buffer, offset, termBuffer, 0, length);
+ termLength = length;
+ }
+
+ @Override
+ public final char[] buffer() {
+ return termBuffer;
+ }
+
+ @Override
+ public final char[] resizeBuffer(int newSize) {
+ if(termBuffer.length < newSize){
+ // Not big enough; create a new array with slight
+ // over allocation and preserve content
+ final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
+ System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
+ termBuffer = newCharBuffer;
+ }
+ return termBuffer;
+ }
+
+ private void growTermBuffer(int newSize) {
+ if(termBuffer.length < newSize){
+ // Not big enough; create a new array with slight
+ // over allocation:
+ termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
+ }
+ }
+
+ @Override
+ public final CharTermAttribute setLength(int length) {
+ if (length > termBuffer.length)
+ throw new IllegalArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.length + ")");
+ termLength = length;
+ return this;
+ }
+
+ @Override
+ public final CharTermAttribute setEmpty() {
+ termLength = 0;
+ return this;
+ }
+
+ // *** TermToBytesRefAttribute interface ***
+ private BytesRef bytes = new BytesRef(MIN_BUFFER_SIZE);
+
+ @Override
+ public int fillBytesRef() {
+ return UnicodeUtil.UTF16toUTF8WithHash(termBuffer, 0, termLength, bytes);
+ }
+
+ @Override
+ public BytesRef getBytesRef() {
+ return bytes;
+ }
+
+ // *** CharSequence interface ***
+ @Override
+ public final int length() {
+ return termLength;
+ }
+
+ @Override
+ public final char charAt(int index) {
+ if (index >= termLength)
+ throw new IndexOutOfBoundsException();
+ return termBuffer[index];
+ }
+
+ @Override
+ public final CharSequence subSequence(final int start, final int end) {
+ if (start > termLength || end > termLength)
+ throw new IndexOutOfBoundsException();
+ return new String(termBuffer, start, end - start);
+ }
+
+ // *** Appendable interface ***
+
+ @Override
+ public final CharTermAttribute append(CharSequence csq) {
+ if (csq == null) // needed for Appendable compliance
+ return appendNull();
+ return append(csq, 0, csq.length());
+ }
+
+ @Override
+ public final CharTermAttribute append(CharSequence csq, int start, int end) {
+ if (csq == null) // needed for Appendable compliance
+ csq = "null";
+ final int len = end - start, csqlen = csq.length();
+ if (len < 0 || start > csqlen || end > csqlen)
+ throw new IndexOutOfBoundsException();
+ if (len == 0)
+ return this;
+ resizeBuffer(termLength + len);
+ if (len > 4) { // only use instanceof check series for longer CSQs, else simply iterate
+ if (csq instanceof String) {
+ ((String) csq).getChars(start, end, termBuffer, termLength);
+ } else if (csq instanceof StringBuilder) {
+ ((StringBuilder) csq).getChars(start, end, termBuffer, termLength);
+ } else if (csq instanceof CharTermAttribute) {
+ System.arraycopy(((CharTermAttribute) csq).buffer(), start, termBuffer, termLength, len);
+ } else if (csq instanceof CharBuffer && ((CharBuffer) csq).hasArray()) {
+ final CharBuffer cb = (CharBuffer) csq;
+ System.arraycopy(cb.array(), cb.arrayOffset() + cb.position() + start, termBuffer, termLength, len);
+ } else if (csq instanceof StringBuffer) {
+ ((StringBuffer) csq).getChars(start, end, termBuffer, termLength);
+ } else {
+ while (start < end)
+ termBuffer[termLength++] = csq.charAt(start++);
+ // no fall-through here, as termLength is updated!
+ return this;
+ }
+ termLength += len;
+ return this;
+ } else {
+ while (start < end)
+ termBuffer[termLength++] = csq.charAt(start++);
+ return this;
+ }
+ }
+
+ @Override
+ public final CharTermAttribute append(char c) {
+ resizeBuffer(termLength + 1)[termLength++] = c;
+ return this;
+ }
+
+ // *** For performance some convenience methods in addition to CSQ's ***
+
+ @Override
+ public final CharTermAttribute append(String s) {
+ if (s == null) // needed for Appendable compliance
+ return appendNull();
+ final int len = s.length();
+ s.getChars(0, len, resizeBuffer(termLength + len), termLength);
+ termLength += len;
+ return this;
+ }
+
+ @Override
+ public final CharTermAttribute append(StringBuilder s) {
+ if (s == null) // needed for Appendable compliance
+ return appendNull();
+ final int len = s.length();
+ s.getChars(0, len, resizeBuffer(termLength + len), termLength);
+ termLength += len;
+ return this;
+ }
+
+ @Override
+ public final CharTermAttribute append(CharTermAttribute ta) {
+ if (ta == null) // needed for Appendable compliance
+ return appendNull();
+ final int len = ta.length();
+ System.arraycopy(ta.buffer(), 0, resizeBuffer(termLength + len), termLength, len);
+ termLength += len;
+ return this;
+ }
+
+ private CharTermAttribute appendNull() {
+ resizeBuffer(termLength + 4);
+ termBuffer[termLength++] = 'n';
+ termBuffer[termLength++] = 'u';
+ termBuffer[termLength++] = 'l';
+ termBuffer[termLength++] = 'l';
+ return this;
+ }
+
+ // *** AttributeImpl ***
+
+ @Override
+ public int hashCode() {
+ int code = termLength;
+ code = code * 31 + ArrayUtil.hashCode(termBuffer, 0, termLength);
+ return code;
+ }
+
+ @Override
+ public void clear() {
+ termLength = 0;
+ }
+
+ @Override
+ public CharTermAttributeImpl clone() {
+ CharTermAttributeImpl t = (CharTermAttributeImpl)super.clone();
+ // Do a deep clone
+ t.termBuffer = new char[this.termLength];
+ System.arraycopy(this.termBuffer, 0, t.termBuffer, 0, this.termLength);
+ t.bytes = BytesRef.deepCopyOf(bytes);
+ return t;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof CharTermAttributeImpl) {
+ final CharTermAttributeImpl o = ((CharTermAttributeImpl) other);
+ if (termLength != o.termLength)
+ return false;
+ for(int i=0;i<termLength;i++) {
+ if (termBuffer[i] != o.termBuffer[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Returns solely the term text as specified by the
+ * {@link CharSequence} interface.
+ * <p>This method changed the behavior with Lucene 3.1,
+ * before it returned a String representation of the whole
+ * term with all attributes.
+ * This affects especially the
+ * {@link org.apache.lucene.analysis.Token} subclass.
+ */
+ @Override
+ public String toString() {
+ return new String(termBuffer, 0, termLength);
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ reflector.reflect(CharTermAttribute.class, "term", toString());
+ fillBytesRef();
+ reflector.reflect(TermToBytesRefAttribute.class, "bytes", BytesRef.deepCopyOf(bytes));
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ CharTermAttribute t = (CharTermAttribute) target;
+ t.copyBuffer(termBuffer, 0, termLength);
+ }
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java
new file mode 100644
index 0000000000..77b3c9809b
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java
@@ -0,0 +1,53 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.Attribute;
+
+/**
+ * This attribute can be used to pass different flags down the {@link Tokenizer} chain,
+ * e.g. from one TokenFilter to another one.
+ * <p>
+ * This is completely distinct from {@link TypeAttribute}, although they do share similar purposes.
+ * The flags can be used to encode information about the token for use by other
+ * {@link org.apache.lucene.analysis.TokenFilter}s.
+ * @lucene.experimental While we think this is here to stay, we may want to change it to be a long.
+ */
+public interface FlagsAttribute extends Attribute {
+ /**
+ * Get the bitset for any bits that have been set.
+ * @return The bits
+ * @see #getFlags()
+ */
+ public int getFlags();
+
+ /**
+ * Set the flags to a new bitset.
+ * @see #getFlags()
+ */
+ public void setFlags(int flags);
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java
new file mode 100644
index 0000000000..c6a730e32c
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java
@@ -0,0 +1,75 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+
+/** Default implementation of {@link FlagsAttribute}. */
+public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute, Cloneable {
+ private int flags = 0;
+
+ /** Initialize this attribute with no bits set */
+ public FlagsAttributeImpl() {}
+
+ @Override
+ public int getFlags() {
+ return flags;
+ }
+
+ @Override
+ public void setFlags(int flags) {
+ this.flags = flags;
+ }
+
+ @Override
+ public void clear() {
+ flags = 0;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof FlagsAttributeImpl) {
+ return ((FlagsAttributeImpl) other).flags == flags;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return flags;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ FlagsAttribute t = (FlagsAttribute) target;
+ t.setFlags(flags);
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java
new file mode 100644
index 0000000000..151b63116f
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java
@@ -0,0 +1,59 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Attribute;
+
+/**
+ * This attribute can be used to mark a token as a keyword. Keyword aware
+ * {@link TokenStream}s can decide to modify a token based on the return value
+ * of {@link #isKeyword()} if the token is modified. Stemming filters for
+ * instance can use this attribute to conditionally skip a term if
+ * {@link #isKeyword()} returns <code>true</code>.
+ */
+public interface KeywordAttribute extends Attribute {
+
+ /**
+ * Returns <code>true</code> if the current token is a keyword, otherwise
+ * <code>false</code>
+ *
+ * @return <code>true</code> if the current token is a keyword, otherwise
+ * <code>false</code>
+ * @see #setKeyword(boolean)
+ */
+ public boolean isKeyword();
+
+ /**
+ * Marks the current token as keyword if set to <code>true</code>.
+ *
+ * @param isKeyword
+ * <code>true</code> if the current token is a keyword, otherwise
+ * <code>false</code>.
+ * @see #isKeyword()
+ */
+ public void setKeyword(boolean isKeyword);
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java
new file mode 100644
index 0000000000..18f6288448
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java
@@ -0,0 +1,74 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+
+/** Default implementation of {@link KeywordAttribute}. */
+public final class KeywordAttributeImpl extends AttributeImpl implements
+ KeywordAttribute {
+ private boolean keyword;
+
+ /** Initialize this attribute with the keyword value as false. */
+ public KeywordAttributeImpl() {}
+
+ @Override
+ public void clear() {
+ keyword = false;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ KeywordAttribute attr = (KeywordAttribute) target;
+ attr.setKeyword(keyword);
+ }
+
+ @Override
+ public int hashCode() {
+ return keyword ? 31 : 37;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (getClass() != obj.getClass())
+ return false;
+ final KeywordAttributeImpl other = (KeywordAttributeImpl) obj;
+ return keyword == other.keyword;
+ }
+
+ @Override
+ public boolean isKeyword() {
+ return keyword;
+ }
+
+ @Override
+ public void setKeyword(boolean isKeyword) {
+ keyword = isKeyword;
+ }
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java
new file mode 100644
index 0000000000..8b23b5c689
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java
@@ -0,0 +1,64 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+/**
+ * The start and end character offset of a Token.
+ */
+public interface OffsetAttribute extends Attribute {
+ /**
+ * Returns this Token's starting offset, the position of the first character
+ * corresponding to this token in the source text.
+ * <p>
+ * Note that the difference between {@link #endOffset()} and <code>startOffset()</code>
+ * may not be equal to termText.length(), as the term text may have been altered by a
+ * stemmer or some other filter.
+ * @see #setOffset(int, int)
+ */
+ public int startOffset();
+
+
+ /**
+ * Set the starting and ending offset.
+ * @throws IllegalArgumentException If <code>startOffset</code> or <code>endOffset</code>
+ * are negative, or if <code>startOffset</code> is greater than
+ * <code>endOffset</code>
+ * @see #startOffset()
+ * @see #endOffset()
+ */
+ public void setOffset(int startOffset, int endOffset);
+
+
+ /**
+ * Returns this Token's ending offset, one greater than the position of the
+ * last character corresponding to this token in the source text. The length
+ * of the token in the source text is (<code>endOffset()</code> - {@link #startOffset()}).
+ * @see #setOffset(int, int)
+ */
+ public int endOffset();
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java
new file mode 100644
index 0000000000..9df11d2640
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java
@@ -0,0 +1,101 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+
+/** Default implementation of {@link OffsetAttribute}. */
+public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttribute, Cloneable {
+ private int startOffset;
+ private int endOffset;
+
+ /** Initialize this attribute with startOffset and endOffset of 0. */
+ public OffsetAttributeImpl() {}
+
+ @Override
+ public int startOffset() {
+ return startOffset;
+ }
+
+ @Override
+ public void setOffset(int startOffset, int endOffset) {
+
+ // TODO: we could assert that this is set-once, ie,
+ // current values are -1? Very few token filters should
+ // change offsets once set by the tokenizer... and
+ // tokenizer should call clearAtts before re-using
+ // OffsetAtt
+
+ if (startOffset < 0 || endOffset < startOffset) {
+ throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
+ + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+ }
+
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ @Override
+ public int endOffset() {
+ return endOffset;
+ }
+
+
+ @Override
+ public void clear() {
+ // TODO: we could use -1 as default here? Then we can
+ // assert in setOffset...
+ startOffset = 0;
+ endOffset = 0;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof OffsetAttributeImpl) {
+ OffsetAttributeImpl o = (OffsetAttributeImpl) other;
+ return o.startOffset == startOffset && o.endOffset == endOffset;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ int code = startOffset;
+ code = code * 31 + endOffset;
+ return code;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ OffsetAttribute t = (OffsetAttribute) target;
+ t.setOffset(startOffset, endOffset);
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java
new file mode 100644
index 0000000000..0d537df866
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java
@@ -0,0 +1,58 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.DocsAndPositionsEnum; // javadocs
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * The payload of a Token.
+ * <p>
+ * The payload is stored in the index at each position, and can
+ * be used to influence scoring when using Payload-based queries
+ * in the {@link org.apache.lucene.search.payloads} and
+ * {@link org.apache.lucene.search.spans} packages.
+ * <p>
+ * NOTE: because the payload will be stored at each position, its usually
+ * best to use the minimum number of bytes necessary. Some codec implementations
+ * may optimize payload storage when all payloads have the same length.
+ *
+ * @see DocsAndPositionsEnum
+ */
+public interface PayloadAttribute extends Attribute {
+ /**
+ * Returns this Token's payload.
+ * @see #setPayload(BytesRef)
+ */
+ public BytesRef getPayload();
+
+ /**
+ * Sets this Token's payload.
+ * @see #getPayload()
+ */
+ public void setPayload(BytesRef payload);
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java
new file mode 100644
index 0000000000..c9f6d06b4c
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java
@@ -0,0 +1,101 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+
+/** Default implementation of {@link PayloadAttribute}. */
+public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribute, Cloneable {
+ private BytesRef payload;
+
+ /**
+ * Initialize this attribute with no payload.
+ */
+ public PayloadAttributeImpl() {}
+
+ /**
+ * Initialize this attribute with the given payload.
+ */
+ public PayloadAttributeImpl(BytesRef payload) {
+ this.payload = payload;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ return this.payload;
+ }
+
+ @Override
+ public void setPayload(BytesRef payload) {
+ this.payload = payload;
+ }
+
+ @Override
+ public void clear() {
+ payload = null;
+ }
+
+ @Override
+ public PayloadAttributeImpl clone() {
+ PayloadAttributeImpl clone = (PayloadAttributeImpl) super.clone();
+ if (payload != null) {
+ clone.payload = payload.clone();
+ }
+ return clone;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof PayloadAttribute) {
+ PayloadAttributeImpl o = (PayloadAttributeImpl) other;
+ if (o.payload == null || payload == null) {
+ return o.payload == null && payload == null;
+ }
+
+ return o.payload.equals(payload);
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return (payload == null) ? 0 : payload.hashCode();
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ PayloadAttribute t = (PayloadAttribute) target;
+ t.setPayload((payload == null) ? null : payload.clone());
+ }
+
+
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java
new file mode 100644
index 0000000000..5b0314e328
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java
@@ -0,0 +1,70 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+/** Determines the position of this token
+ * relative to the previous Token in a TokenStream, used in phrase
+ * searching.
+ *
+ * <p>The default value is one.
+ *
+ * <p>Some common uses for this are:<ul>
+ *
+ * <li>Set it to zero to put multiple terms in the same position. This is
+ * useful if, e.g., a word has multiple stems. Searches for phrases
+ * including either stem will match. In this case, all but the first stem's
+ * increment should be set to zero: the increment of the first instance
+ * should be one. Repeating a token with an increment of zero can also be
+ * used to boost the scores of matches on that token.
+ *
+ * <li>Set it to values greater than one to inhibit exact phrase matches.
+ * If, for example, one does not want phrases to match across removed stop
+ * words, then one could build a stop word filter that removes stop words and
+ * also sets the increment to the number of stop words removed before each
+ * non-stop word. Then exact phrase queries will only match when the terms
+ * occur with no intervening stop words.
+ *
+ * </ul>
+ *
+ * @see org.apache.lucene.index.DocsAndPositionsEnum
+ */
+public interface PositionIncrementAttribute extends Attribute {
+ /** Set the position increment. The default value is one.
+ *
+ * @param positionIncrement the distance from the prior term
+ * @throws IllegalArgumentException if <code>positionIncrement</code>
+ * is negative.
+ * @see #getPositionIncrement()
+ */
+ public void setPositionIncrement(int positionIncrement);
+
+ /** Returns the position increment of this Token.
+ * @see #setPositionIncrement(int)
+ */
+ public int getPositionIncrement();
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
new file mode 100644
index 0000000000..59247a7a42
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
@@ -0,0 +1,80 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+
+/** Default implementation of {@link PositionIncrementAttribute}. */
+public class PositionIncrementAttributeImpl extends AttributeImpl implements PositionIncrementAttribute, Cloneable {
+ private int positionIncrement = 1;
+
+ /** Initialize this attribute with position increment of 1 */
+ public PositionIncrementAttributeImpl() {}
+
+ @Override
+ public void setPositionIncrement(int positionIncrement) {
+ if (positionIncrement < 0) {
+ throw new IllegalArgumentException
+ ("Increment must be zero or greater: got " + positionIncrement);
+ }
+ this.positionIncrement = positionIncrement;
+ }
+
+ @Override
+ public int getPositionIncrement() {
+ return positionIncrement;
+ }
+
+ @Override
+ public void clear() {
+ this.positionIncrement = 1;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof PositionIncrementAttributeImpl) {
+ PositionIncrementAttributeImpl _other = (PositionIncrementAttributeImpl) other;
+ return positionIncrement == _other.positionIncrement;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return positionIncrement;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ PositionIncrementAttribute t = (PositionIncrementAttribute) target;
+ t.setPositionIncrement(positionIncrement);
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java
new file mode 100644
index 0000000000..6c0de31864
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java
@@ -0,0 +1,58 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+/** Determines how many positions this
+ * token spans. Very few analyzer components actually
+ * produce this attribute, and indexing ignores it, but
+ * it's useful to express the graph structure naturally
+ * produced by decompounding, word splitting/joining,
+ * synonym filtering, etc.
+ *
+ * <p>NOTE: this is optional, and most analyzers
+ * don't change the default value (1). */
+
+public interface PositionLengthAttribute extends Attribute {
+ /**
+ * Set the position length of this Token.
+ * <p>
+ * The default value is one.
+ * @param positionLength how many positions this token
+ * spans.
+ * @throws IllegalArgumentException if <code>positionLength</code>
+ * is zero or negative.
+ * @see #getPositionLength()
+ */
+ public void setPositionLength(int positionLength);
+
+ /** Returns the position length of this Token.
+ * @see #setPositionLength
+ */
+ public int getPositionLength();
+}
+
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java
new file mode 100644
index 0000000000..66f595ee4e
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java
@@ -0,0 +1,80 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+
+/** Default implementation of {@link PositionLengthAttribute}. */
+public class PositionLengthAttributeImpl extends AttributeImpl implements PositionLengthAttribute, Cloneable {
+ private int positionLength = 1;
+
+ /** Initializes this attribute with position length of 1. */
+ public PositionLengthAttributeImpl() {}
+
+ @Override
+ public void setPositionLength(int positionLength) {
+ if (positionLength < 1) {
+ throw new IllegalArgumentException
+ ("Position length must be 1 or greater: got " + positionLength);
+ }
+ this.positionLength = positionLength;
+ }
+
+ @Override
+ public int getPositionLength() {
+ return positionLength;
+ }
+
+ @Override
+ public void clear() {
+ this.positionLength = 1;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof PositionLengthAttributeImpl) {
+ PositionLengthAttributeImpl _other = (PositionLengthAttributeImpl) other;
+ return positionLength == _other.positionLength;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return positionLength;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ PositionLengthAttribute t = (PositionLengthAttribute) target;
+ t.setPositionLength(positionLength);
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
new file mode 100644
index 0000000000..6e9314b2cb
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
@@ -0,0 +1,85 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * This attribute is requested by TermsHashPerField to index the contents.
+ * This attribute can be used to customize the final byte[] encoding of terms.
+ * <p>
+ * Consumers of this attribute call {@link #getBytesRef()} up-front, and then
+ * invoke {@link #fillBytesRef()} for each term. Example:
+ * <pre class="prettyprint">
+ * final TermToBytesRefAttribute termAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
+ * final BytesRef bytes = termAtt.getBytesRef();
+ *
+ * while (tokenStream.incrementToken() {
+ *
+ * // you must call termAtt.fillBytesRef() before doing something with the bytes.
+ * // this encodes the term value (internally it might be a char[], etc) into the bytes.
+ * int hashCode = termAtt.fillBytesRef();
+ *
+ * if (isInteresting(bytes)) {
+ *
+ * // because the bytes are reused by the attribute (like CharTermAttribute's char[] buffer),
+ * // you should make a copy if you need persistent access to the bytes, otherwise they will
+ * // be rewritten across calls to incrementToken()
+ *
+ * doSomethingWith(new BytesRef(bytes));
+ * }
+ * }
+ * ...
+ * </pre>
+ * @lucene.experimental This is a very expert API, please use
+ * {@link CharTermAttributeImpl} and its implementation of this method
+ * for UTF-8 terms.
+ */
+public interface TermToBytesRefAttribute extends Attribute {
+ /**
+ * Updates the bytes {@link #getBytesRef()} to contain this term's
+ * final encoding, and returns its hashcode.
+ * @return the hashcode as defined by {@link BytesRef#hashCode}:
+ * <pre class="prettyprint">
+ * int hash = 0;
+ * for (int i = termBytes.offset; i < termBytes.offset+termBytes.length; i++) {
+ * hash = 31*hash + termBytes.bytes[i];
+ * }
+ * </pre>
+ * Implement this for performance reasons, if your code can calculate
+ * the hash on-the-fly. If this is not the case, just return
+ * {@code termBytes.hashCode()}.
+ */
+ public int fillBytesRef();
+
+ /**
+ * Retrieve this attribute's BytesRef. The bytes are updated
+ * from the current term when the consumer calls {@link #fillBytesRef()}.
+ * @return this Attributes internal BytesRef.
+ */
+ public BytesRef getBytesRef();
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java
new file mode 100644
index 0000000000..48bddb3c7e
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java
@@ -0,0 +1,49 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+/**
+ * A Token's lexical type. The Default value is "word".
+ */
+public interface TypeAttribute extends Attribute {
+
+ /** the default type */
+ public static final String DEFAULT_TYPE = "word";
+
+ /**
+ * Returns this Token's lexical type. Defaults to "word".
+ * @see #setType(String)
+ */
+ public String type();
+
+ /**
+ * Set the lexical type.
+ * @see #type()
+ */
+ public void setType(String type);
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java
new file mode 100644
index 0000000000..14cb8d0c57
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java
@@ -0,0 +1,83 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+
+/** Default implementation of {@link TypeAttribute}. */
+public class TypeAttributeImpl extends AttributeImpl implements TypeAttribute, Cloneable {
+ private String type;
+
+ /** Initialize this attribute with {@link TypeAttribute#DEFAULT_TYPE} */
+ public TypeAttributeImpl() {
+ this(DEFAULT_TYPE);
+ }
+
+ /** Initialize this attribute with <code>type</code> */
+ public TypeAttributeImpl(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String type() {
+ return type;
+ }
+
+ @Override
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public void clear() {
+ type = DEFAULT_TYPE;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof TypeAttributeImpl) {
+ final TypeAttributeImpl o = (TypeAttributeImpl) other;
+ return (this.type == null ? o.type == null : this.type.equals(o.type));
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return (type == null) ? 0 : type.hashCode();
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ TypeAttribute t = (TypeAttribute) target;
+ t.setType(type);
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/package.html b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/package.html
new file mode 100644
index 0000000000..d988488c33
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/analysis/tokenattributes/package.html
@@ -0,0 +1,25 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+General-purpose attributes for text analysis.
+</body>
+</html>
diff --git a/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTermState.java b/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTermState.java
new file mode 100644
index 0000000000..83c10a1e54
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTermState.java
@@ -0,0 +1,68 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.codecs;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.DocsEnum; // javadocs
+import org.apache.lucene.index.OrdTermState;
+import org.apache.lucene.index.TermState;
+
+/**
+ * Holds all state required for {@link PostingsReaderBase}
+ * to produce a {@link DocsEnum} without re-seeking the
+ * terms dict.
+ */
+public class BlockTermState extends OrdTermState {
+ /** how many docs have this term */
+ public int docFreq;
+ /** total number of occurrences of this term */
+ public long totalTermFreq;
+
+ /** the term's ord in the current block */
+ public int termBlockOrd;
+ /** fp into the terms dict primary file (_X.tim) that holds this term */
+ // TODO: update BTR to nuke this
+ public long blockFilePointer;
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected BlockTermState() {
+ }
+
+ @Override
+ public void copyFrom(TermState _other) {
+ assert _other instanceof BlockTermState : "can not copy from " + _other.getClass().getName();
+ BlockTermState other = (BlockTermState) _other;
+ super.copyFrom(_other);
+ docFreq = other.docFreq;
+ totalTermFreq = other.totalTermFreq;
+ termBlockOrd = other.termBlockOrd;
+ blockFilePointer = other.blockFilePointer;
+ }
+
+ @Override
+ public String toString() {
+ return "docFreq=" + docFreq + " totalTermFreq=" + totalTermFreq + " termBlockOrd=" + termBlockOrd + " blockFP=" + blockFilePointer;
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTreeTermsReader.java b/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
new file mode 100644
index 0000000000..2daff85abd
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
@@ -0,0 +1,3010 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.codecs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.TreeMap;
+
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.RunAutomaton;
+import org.apache.lucene.util.automaton.Transition;
+import org.apache.lucene.util.fst.ByteSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.Outputs;
+import org.apache.lucene.util.fst.Util;
+
+/** A block-based terms index and dictionary that assigns
+ * terms to variable length blocks according to how they
+ * share prefixes. The terms index is a prefix trie
+ * whose leaves are term blocks. The advantage of this
+ * approach is that seekExact is often able to
+ * determine a term cannot exist without doing any IO, and
+ * intersection with Automata is very fast. Note that this
+ * terms dictionary has it's own fixed terms index (ie, it
+ * does not support a pluggable terms index
+ * implementation).
+ *
+ * <p><b>NOTE</b>: this terms dictionary does not support
+ * index divisor when opening an IndexReader. Instead, you
+ * can change the min/maxItemsPerBlock during indexing.</p>
+ *
+ * <p>The data structure used by this implementation is very
+ * similar to a burst trie
+ * (http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.18.3499),
+ * but with added logic to break up too-large blocks of all
+ * terms sharing a given prefix into smaller ones.</p>
+ *
+ * <p>Use {@link org.apache.lucene.index.CheckIndex} with the <code>-verbose</code>
+ * option to see summary statistics on the blocks in the
+ * dictionary.
+ *
+ * See {@link BlockTreeTermsWriter}.
+ *
+ * @lucene.experimental
+ */
+
+public class BlockTreeTermsReader extends FieldsProducer {
+
+ // Open input to the main terms dict file (_X.tib)
+ private final IndexInput in;
+
+ //private static final boolean DEBUG = BlockTreeTermsWriter.DEBUG;
+
+ // Reads the terms dict entries, to gather state to
+ // produce DocsEnum on demand
+ private final PostingsReaderBase postingsReader;
+
+ private final TreeMap<String,FieldReader> fields = new TreeMap<String,FieldReader>();
+
+ /** File offset where the directory starts in the terms file. */
+ private long dirOffset;
+
+ /** File offset where the directory starts in the index file. */
+ private long indexDirOffset;
+
+ private String segment;
+
+ private final int version;
+
+ /** Sole constructor. */
+ public BlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
+ PostingsReaderBase postingsReader, IOContext ioContext,
+ String segmentSuffix, int indexDivisor)
+ throws IOException {
+
+ this.postingsReader = postingsReader;
+
+ this.segment = info.name;
+ in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION),
+ ioContext);
+
+ boolean success = false;
+ IndexInput indexIn = null;
+
+ try {
+ version = readHeader(in);
+ if (indexDivisor != -1) {
+ indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
+ ioContext);
+ int indexVersion = readIndexHeader(indexIn);
+ if (indexVersion != version) {
+ throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion);
+ }
+ }
+
+ // Have PostingsReader init itself
+ postingsReader.init(in);
+
+ // Read per-field details
+ seekDir(in, dirOffset);
+ if (indexDivisor != -1) {
+ seekDir(indexIn, indexDirOffset);
+ }
+
+ final int numFields = in.readVInt();
+ if (numFields < 0) {
+ throw new CorruptIndexException("invalid numFields: " + numFields + " (resource=" + in + ")");
+ }
+
+ for(int i=0;i<numFields;i++) {
+ final int field = in.readVInt();
+ final long numTerms = in.readVLong();
+ assert numTerms >= 0;
+ final int numBytes = in.readVInt();
+ final BytesRef rootCode = new BytesRef(new byte[numBytes]);
+ in.readBytes(rootCode.bytes, 0, numBytes);
+ rootCode.length = numBytes;
+ final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ assert fieldInfo != null: "field=" + field;
+ final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
+ final long sumDocFreq = in.readVLong();
+ final int docCount = in.readVInt();
+ final int longsSize = version >= BlockTreeTermsWriter.TERMS_VERSION_META_ARRAY ? in.readVInt() : 0;
+ if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
+ throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")");
+ }
+ if (sumDocFreq < docCount) { // #postings must be >= #docs with field
+ throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount + " (resource=" + in + ")");
+ }
+ if (sumTotalTermFreq != -1 && sumTotalTermFreq < sumDocFreq) { // #positions must be >= #postings
+ throw new CorruptIndexException("invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq + " (resource=" + in + ")");
+ }
+ final long indexStartFP = indexDivisor != -1 ? indexIn.readVLong() : 0;
+ FieldReader previous = fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount, indexStartFP, longsSize, indexIn));
+ if (previous != null) {
+ throw new CorruptIndexException("duplicate field: " + fieldInfo.name + " (resource=" + in + ")");
+ }
+ }
+ if (indexDivisor != -1) {
+ indexIn.close();
+ }
+
+ success = true;
+ } finally {
+ if (!success) {
+ // this.close() will close in:
+ IOUtils.closeWhileHandlingException(indexIn, this);
+ }
+ }
+ }
+
+ /** Reads terms file header. */
+ protected int readHeader(IndexInput input) throws IOException {
+ int version = CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_CODEC_NAME,
+ BlockTreeTermsWriter.TERMS_VERSION_START,
+ BlockTreeTermsWriter.TERMS_VERSION_CURRENT);
+ if (version < BlockTreeTermsWriter.TERMS_VERSION_APPEND_ONLY) {
+ dirOffset = input.readLong();
+ }
+ return version;
+ }
+
+ /** Reads index file header. */
+ protected int readIndexHeader(IndexInput input) throws IOException {
+ int version = CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
+ BlockTreeTermsWriter.TERMS_INDEX_VERSION_START,
+ BlockTreeTermsWriter.TERMS_INDEX_VERSION_CURRENT);
+ if (version < BlockTreeTermsWriter.TERMS_INDEX_VERSION_APPEND_ONLY) {
+ indexDirOffset = input.readLong();
+ }
+ return version;
+ }
+
+ /** Seek {@code input} to the directory offset. */
+ protected void seekDir(IndexInput input, long dirOffset)
+ throws IOException {
+ if (version >= BlockTreeTermsWriter.TERMS_INDEX_VERSION_APPEND_ONLY) {
+ input.seek(input.length() - 8);
+ dirOffset = input.readLong();
+ }
+ input.seek(dirOffset);
+ }
+
+ // for debugging
+ // private static String toHex(int v) {
+ // return "0x" + Integer.toHexString(v);
+ // }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ IOUtils.close(in, postingsReader);
+ } finally {
+ // Clear so refs to terms index is GCable even if
+ // app hangs onto us:
+ fields.clear();
+ }
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return Collections.unmodifiableSet(fields.keySet()).iterator();
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ assert field != null;
+ return fields.get(field);
+ }
+
+ @Override
+ public int size() {
+ return fields.size();
+ }
+
+ // for debugging
+ String brToString(BytesRef b) {
+ if (b == null) {
+ return "null";
+ } else {
+ try {
+ return b.utf8ToString() + " " + b;
+ } catch (Throwable t) {
+ // If BytesRef isn't actually UTF8, or it's eg a
+ // prefix of UTF8 that ends mid-unicode-char, we
+ // fallback to hex:
+ return b.toString();
+ }
+ }
+ }
+
+ /**
+ * BlockTree statistics for a single field
+ * returned by {@link FieldReader#computeStats()}.
+ */
+ public static class Stats {
+ /** How many nodes in the index FST. */
+ public long indexNodeCount;
+
+ /** How many arcs in the index FST. */
+ public long indexArcCount;
+
+ /** Byte size of the index. */
+ public long indexNumBytes;
+
+ /** Total number of terms in the field. */
+ public long totalTermCount;
+
+ /** Total number of bytes (sum of term lengths) across all terms in the field. */
+ public long totalTermBytes;
+
+ /** The number of normal (non-floor) blocks in the terms file. */
+ public int nonFloorBlockCount;
+
+ /** The number of floor blocks (meta-blocks larger than the
+ * allowed {@code maxItemsPerBlock}) in the terms file. */
+ public int floorBlockCount;
+
+ /** The number of sub-blocks within the floor blocks. */
+ public int floorSubBlockCount;
+
+ /** The number of "internal" blocks (that have both
+ * terms and sub-blocks). */
+ public int mixedBlockCount;
+
+ /** The number of "leaf" blocks (blocks that have only
+ * terms). */
+ public int termsOnlyBlockCount;
+
+ /** The number of "internal" blocks that do not contain
+ * terms (have only sub-blocks). */
+ public int subBlocksOnlyBlockCount;
+
+ /** Total number of blocks. */
+ public int totalBlockCount;
+
+ /** Number of blocks at each prefix depth. */
+ public int[] blockCountByPrefixLen = new int[10];
+ private int startBlockCount;
+ private int endBlockCount;
+
+ /** Total number of bytes used to store term suffixes. */
+ public long totalBlockSuffixBytes;
+
+ /** Total number of bytes used to store term stats (not
+ * including what the {@link PostingsBaseFormat}
+ * stores. */
+ public long totalBlockStatsBytes;
+
+ /** Total bytes stored by the {@link PostingsBaseFormat},
+ * plus the other few vInts stored in the frame. */
+ public long totalBlockOtherBytes;
+
+ /** Segment name. */
+ public final String segment;
+
+ /** Field name. */
+ public final String field;
+
+ Stats(String segment, String field) {
+ this.segment = segment;
+ this.field = field;
+ }
+
+ void startBlock(FieldReader.SegmentTermsEnum.Frame frame, boolean isFloor) {
+ totalBlockCount++;
+ if (isFloor) {
+ if (frame.fp == frame.fpOrig) {
+ floorBlockCount++;
+ }
+ floorSubBlockCount++;
+ } else {
+ nonFloorBlockCount++;
+ }
+
+ if (blockCountByPrefixLen.length <= frame.prefix) {
+ blockCountByPrefixLen = ArrayUtil.grow(blockCountByPrefixLen, 1+frame.prefix);
+ }
+ blockCountByPrefixLen[frame.prefix]++;
+ startBlockCount++;
+ totalBlockSuffixBytes += frame.suffixesReader.length();
+ totalBlockStatsBytes += frame.statsReader.length();
+ }
+
+ void endBlock(FieldReader.SegmentTermsEnum.Frame frame) {
+ final int termCount = frame.isLeafBlock ? frame.entCount : frame.state.termBlockOrd;
+ final int subBlockCount = frame.entCount - termCount;
+ totalTermCount += termCount;
+ if (termCount != 0 && subBlockCount != 0) {
+ mixedBlockCount++;
+ } else if (termCount != 0) {
+ termsOnlyBlockCount++;
+ } else if (subBlockCount != 0) {
+ subBlocksOnlyBlockCount++;
+ } else {
+ throw new IllegalStateException();
+ }
+ endBlockCount++;
+ final long otherBytes = frame.fpEnd - frame.fp - frame.suffixesReader.length() - frame.statsReader.length();
+ assert otherBytes > 0 : "otherBytes=" + otherBytes + " frame.fp=" + frame.fp + " frame.fpEnd=" + frame.fpEnd;
+ totalBlockOtherBytes += otherBytes;
+ }
+
+ void term(BytesRef term) {
+ totalTermBytes += term.length;
+ }
+
+ void finish() {
+ assert startBlockCount == endBlockCount: "startBlockCount=" + startBlockCount + " endBlockCount=" + endBlockCount;
+ assert totalBlockCount == floorSubBlockCount + nonFloorBlockCount: "floorSubBlockCount=" + floorSubBlockCount + " nonFloorBlockCount=" + nonFloorBlockCount + " totalBlockCount=" + totalBlockCount;
+ assert totalBlockCount == mixedBlockCount + termsOnlyBlockCount + subBlocksOnlyBlockCount: "totalBlockCount=" + totalBlockCount + " mixedBlockCount=" + mixedBlockCount + " subBlocksOnlyBlockCount=" + subBlocksOnlyBlockCount + " termsOnlyBlockCount=" + termsOnlyBlockCount;
+ }
+
+ @Override
+ public String toString() {
+ final ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+ PrintStream out;
+ try {
+ out = new PrintStream(bos, false, "UTF-8");
+ } catch (UnsupportedEncodingException bogus) {
+ throw new RuntimeException(bogus);
+ }
+
+ out.println(" index FST:");
+ out.println(" " + indexNodeCount + " nodes");
+ out.println(" " + indexArcCount + " arcs");
+ out.println(" " + indexNumBytes + " bytes");
+ out.println(" terms:");
+ out.println(" " + totalTermCount + " terms");
+ out.println(" " + totalTermBytes + " bytes" + (totalTermCount != 0 ? " (" + String.format(Locale.ROOT, "%.1f", ((double) totalTermBytes)/totalTermCount) + " bytes/term)" : ""));
+ out.println(" blocks:");
+ out.println(" " + totalBlockCount + " blocks");
+ out.println(" " + termsOnlyBlockCount + " terms-only blocks");
+ out.println(" " + subBlocksOnlyBlockCount + " sub-block-only blocks");
+ out.println(" " + mixedBlockCount + " mixed blocks");
+ out.println(" " + floorBlockCount + " floor blocks");
+ out.println(" " + (totalBlockCount-floorSubBlockCount) + " non-floor blocks");
+ out.println(" " + floorSubBlockCount + " floor sub-blocks");
+ out.println(" " + totalBlockSuffixBytes + " term suffix bytes" + (totalBlockCount != 0 ? " (" + String.format(Locale.ROOT, "%.1f", ((double) totalBlockSuffixBytes)/totalBlockCount) + " suffix-bytes/block)" : ""));
+ out.println(" " + totalBlockStatsBytes + " term stats bytes" + (totalBlockCount != 0 ? " (" + String.format(Locale.ROOT, "%.1f", ((double) totalBlockStatsBytes)/totalBlockCount) + " stats-bytes/block)" : ""));
+ out.println(" " + totalBlockOtherBytes + " other bytes" + (totalBlockCount != 0 ? " (" + String.format(Locale.ROOT, "%.1f", ((double) totalBlockOtherBytes)/totalBlockCount) + " other-bytes/block)" : ""));
+ if (totalBlockCount != 0) {
+ out.println(" by prefix length:");
+ int total = 0;
+ for(int prefix=0;prefix<blockCountByPrefixLen.length;prefix++) {
+ final int blockCount = blockCountByPrefixLen[prefix];
+ total += blockCount;
+ if (blockCount != 0) {
+ out.println(" " + String.format(Locale.ROOT, "%2d", prefix) + ": " + blockCount);
+ }
+ }
+ assert totalBlockCount == total;
+ }
+
+ try {
+ return bos.toString("UTF-8");
+ } catch (UnsupportedEncodingException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ }
+ }
+
+ final Outputs<BytesRef> fstOutputs = ByteSequenceOutputs.getSingleton();
+ final BytesRef NO_OUTPUT = fstOutputs.getNoOutput();
+
+ /** BlockTree's implementation of {@link Terms}. */
+ public final class FieldReader extends Terms {
+ final long numTerms;
+ final FieldInfo fieldInfo;
+ final long sumTotalTermFreq;
+ final long sumDocFreq;
+ final int docCount;
+ final long indexStartFP;
+ final long rootBlockFP;
+ final BytesRef rootCode;
+ final int longsSize;
+
+ private final FST<BytesRef> index;
+ //private boolean DEBUG;
+
+ FieldReader(FieldInfo fieldInfo, long numTerms, BytesRef rootCode, long sumTotalTermFreq, long sumDocFreq, int docCount, long indexStartFP, int longsSize, IndexInput indexIn) throws IOException {
+ assert numTerms > 0;
+ this.fieldInfo = fieldInfo;
+ //DEBUG = BlockTreeTermsReader.DEBUG && fieldInfo.name.equals("id");
+ this.numTerms = numTerms;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+ this.indexStartFP = indexStartFP;
+ this.rootCode = rootCode;
+ this.longsSize = longsSize;
+ // if (DEBUG) {
+ // System.out.println("BTTR: seg=" + segment + " field=" + fieldInfo.name + " rootBlockCode=" + rootCode + " divisor=" + indexDivisor);
+ // }
+
+ rootBlockFP = (new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length)).readVLong() >>> BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS;
+
+ if (indexIn != null) {
+ final IndexInput clone = indexIn.clone();
+ //System.out.println("start=" + indexStartFP + " field=" + fieldInfo.name);
+ clone.seek(indexStartFP);
+ index = new FST<BytesRef>(clone, ByteSequenceOutputs.getSingleton());
+
+ /*
+ if (false) {
+ final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
+ Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));
+ Util.toDot(index, w, false, false);
+ System.out.println("FST INDEX: SAVED to " + dotFileName);
+ w.close();
+ }
+ */
+ } else {
+ index = null;
+ }
+ }
+
+ /** For debugging -- used by CheckIndex too*/
+ // TODO: maybe push this into Terms?
+ public Stats computeStats() throws IOException {
+ return new SegmentTermsEnum().computeBlockStats();
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public boolean hasFreqs() {
+ return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ }
+
+ @Override
+ public boolean hasOffsets() {
+ return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return fieldInfo.hasPayloads();
+ }
+
+ @Override
+ public TermsEnum iterator(TermsEnum reuse) throws IOException {
+ return new SegmentTermsEnum();
+ }
+
+ @Override
+ public long size() {
+ return numTerms;
+ }
+
+ @Override
+ public long getSumTotalTermFreq() {
+ return sumTotalTermFreq;
+ }
+
+ @Override
+ public long getSumDocFreq() {
+ return sumDocFreq;
+ }
+
+ @Override
+ public int getDocCount() {
+ return docCount;
+ }
+
+ @Override
+ public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException {
+ if (compiled.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
+ throw new IllegalArgumentException("please use CompiledAutomaton.getTermsEnum instead");
+ }
+ return new IntersectEnum(compiled, startTerm);
+ }
+
+ /** Returns approximate RAM bytes used */
+ public long ramBytesUsed() {
+ return ((index!=null)? index.sizeInBytes() : 0);
+ }
+
+ // NOTE: cannot seek!
+ private final class IntersectEnum extends TermsEnum {
+ private final IndexInput in;
+
+ private Frame[] stack;
+
+ @SuppressWarnings({"rawtypes","unchecked"}) private FST.Arc<BytesRef>[] arcs = new FST.Arc[5];
+
+ private final RunAutomaton runAutomaton;
+ private final CompiledAutomaton compiledAutomaton;
+
+ private Frame currentFrame;
+
+ private final BytesRef term = new BytesRef();
+
+ private final FST.BytesReader fstReader;
+
+ // TODO: can we share this with the frame in STE?
+ private final class Frame {
+ final int ord;
+ long fp;
+ long fpOrig;
+ long fpEnd;
+ long lastSubFP;
+
+ // State in automaton
+ int state;
+
+ int metaDataUpto;
+
+ byte[] suffixBytes = new byte[128];
+ final ByteArrayDataInput suffixesReader = new ByteArrayDataInput();
+
+ byte[] statBytes = new byte[64];
+ final ByteArrayDataInput statsReader = new ByteArrayDataInput();
+
+ byte[] floorData = new byte[32];
+ final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();
+
+ // Length of prefix shared by all terms in this block
+ int prefix;
+
+ // Number of entries (term or sub-block) in this block
+ int entCount;
+
+ // Which term we will next read
+ int nextEnt;
+
+ // True if this block is either not a floor block,
+ // or, it's the last sub-block of a floor block
+ boolean isLastInFloor;
+
+ // True if all entries are terms
+ boolean isLeafBlock;
+
+ int numFollowFloorBlocks;
+ int nextFloorLabel;
+
+ Transition[] transitions;
+ int curTransitionMax;
+ int transitionIndex;
+
+ FST.Arc<BytesRef> arc;
+
+ final BlockTermState termState;
+
+ // metadata buffer, holding monotonic values
+ public long[] longs;
+ // metadata buffer, holding general values
+ public byte[] bytes;
+ ByteArrayDataInput bytesReader;
+
+ // Cumulative output so far
+ BytesRef outputPrefix;
+
+ private int startBytePos;
+ private int suffix;
+
+ public Frame(int ord) throws IOException {
+ this.ord = ord;
+ this.termState = postingsReader.newTermState();
+ this.termState.totalTermFreq = -1;
+ this.longs = new long[longsSize];
+ }
+
+ void loadNextFloorBlock() throws IOException {
+ assert numFollowFloorBlocks > 0;
+ //if (DEBUG) System.out.println(" loadNextFoorBlock trans=" + transitions[transitionIndex]);
+
+ do {
+ fp = fpOrig + (floorDataReader.readVLong() >>> 1);
+ numFollowFloorBlocks--;
+ // if (DEBUG) System.out.println(" skip floor block2! nextFloorLabel=" + (char) nextFloorLabel + " vs target=" + (char) transitions[transitionIndex].getMin() + " newFP=" + fp + " numFollowFloorBlocks=" + numFollowFloorBlocks);
+ if (numFollowFloorBlocks != 0) {
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ } else {
+ nextFloorLabel = 256;
+ }
+ // if (DEBUG) System.out.println(" nextFloorLabel=" + (char) nextFloorLabel);
+ } while (numFollowFloorBlocks != 0 && nextFloorLabel <= transitions[transitionIndex].getMin());
+
+ load(null);
+ }
+
+ public void setState(int state) {
+ this.state = state;
+ transitionIndex = 0;
+ transitions = compiledAutomaton.sortedTransitions[state];
+ if (transitions.length != 0) {
+ curTransitionMax = transitions[0].getMax();
+ } else {
+ curTransitionMax = -1;
+ }
+ }
+
+ void load(BytesRef frameIndexData) throws IOException {
+
+ // if (DEBUG) System.out.println(" load fp=" + fp + " fpOrig=" + fpOrig + " frameIndexData=" + frameIndexData + " trans=" + (transitions.length != 0 ? transitions[0] : "n/a" + " state=" + state));
+
+ if (frameIndexData != null && transitions.length != 0) {
+ // Floor frame
+ if (floorData.length < frameIndexData.length) {
+ this.floorData = new byte[ArrayUtil.oversize(frameIndexData.length, 1)];
+ }
+ System.arraycopy(frameIndexData.bytes, frameIndexData.offset, floorData, 0, frameIndexData.length);
+ floorDataReader.reset(floorData, 0, frameIndexData.length);
+ // Skip first long -- has redundant fp, hasTerms
+ // flag, isFloor flag
+ final long code = floorDataReader.readVLong();
+ if ((code & BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR) != 0) {
+ numFollowFloorBlocks = floorDataReader.readVInt();
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ // if (DEBUG) System.out.println(" numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + nextFloorLabel);
+
+ // If current state is accept, we must process
+ // first block in case it has empty suffix:
+ if (!runAutomaton.isAccept(state)) {
+ // Maybe skip floor blocks:
+ while (numFollowFloorBlocks != 0 && nextFloorLabel <= transitions[0].getMin()) {
+ fp = fpOrig + (floorDataReader.readVLong() >>> 1);
+ numFollowFloorBlocks--;
+ // if (DEBUG) System.out.println(" skip floor block! nextFloorLabel=" + (char) nextFloorLabel + " vs target=" + (char) transitions[0].getMin() + " newFP=" + fp + " numFollowFloorBlocks=" + numFollowFloorBlocks);
+ if (numFollowFloorBlocks != 0) {
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ } else {
+ nextFloorLabel = 256;
+ }
+ }
+ }
+ }
+ }
+
+ in.seek(fp);
+ int code = in.readVInt();
+ entCount = code >>> 1;
+ assert entCount > 0;
+ isLastInFloor = (code & 1) != 0;
+
+ // term suffixes:
+ code = in.readVInt();
+ isLeafBlock = (code & 1) != 0;
+ int numBytes = code >>> 1;
+ // if (DEBUG) System.out.println(" entCount=" + entCount + " lastInFloor?=" + isLastInFloor + " leafBlock?=" + isLeafBlock + " numSuffixBytes=" + numBytes);
+ if (suffixBytes.length < numBytes) {
+ suffixBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ in.readBytes(suffixBytes, 0, numBytes);
+ suffixesReader.reset(suffixBytes, 0, numBytes);
+
+ // stats
+ numBytes = in.readVInt();
+ if (statBytes.length < numBytes) {
+ statBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ in.readBytes(statBytes, 0, numBytes);
+ statsReader.reset(statBytes, 0, numBytes);
+ metaDataUpto = 0;
+
+ termState.termBlockOrd = 0;
+ nextEnt = 0;
+
+ // metadata
+ numBytes = in.readVInt();
+ if (bytes == null) {
+ bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ bytesReader = new ByteArrayDataInput();
+ } else if (bytes.length < numBytes) {
+ bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ in.readBytes(bytes, 0, numBytes);
+ bytesReader.reset(bytes, 0, numBytes);
+
+ if (!isLastInFloor) {
+ // Sub-blocks of a single floor block are always
+ // written one after another -- tail recurse:
+ fpEnd = in.getFilePointer();
+ }
+ }
+
+ // TODO: maybe add scanToLabel; should give perf boost
+
+ public boolean next() {
+ return isLeafBlock ? nextLeaf() : nextNonLeaf();
+ }
+
+ // Decodes next entry; returns true if it's a sub-block
+ public boolean nextLeaf() {
+ //if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt + " entCount=" + entCount);
+ assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
+ nextEnt++;
+ suffix = suffixesReader.readVInt();
+ startBytePos = suffixesReader.getPosition();
+ suffixesReader.skipBytes(suffix);
+ return false;
+ }
+
+ public boolean nextNonLeaf() {
+ //if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt + " entCount=" + entCount);
+ assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
+ nextEnt++;
+ final int code = suffixesReader.readVInt();
+ suffix = code >>> 1;
+ startBytePos = suffixesReader.getPosition();
+ suffixesReader.skipBytes(suffix);
+ if ((code & 1) == 0) {
+ // A normal term
+ termState.termBlockOrd++;
+ return false;
+ } else {
+ // A sub-block; make sub-FP absolute:
+ lastSubFP = fp - suffixesReader.readVLong();
+ return true;
+ }
+ }
+
+ public int getTermBlockOrd() {
+ return isLeafBlock ? nextEnt : termState.termBlockOrd;
+ }
+
+ public void decodeMetaData() throws IOException {
+
+ // lazily catch up on metadata decode:
+ final int limit = getTermBlockOrd();
+ boolean absolute = metaDataUpto == 0;
+ assert limit > 0;
+
+ // TODO: better API would be "jump straight to term=N"???
+ while (metaDataUpto < limit) {
+
+ // TODO: we could make "tiers" of metadata, ie,
+ // decode docFreq/totalTF but don't decode postings
+ // metadata; this way caller could get
+ // docFreq/totalTF w/o paying decode cost for
+ // postings
+
+ // TODO: if docFreq were bulk decoded we could
+ // just skipN here:
+
+ // stats
+ termState.docFreq = statsReader.readVInt();
+ //if (DEBUG) System.out.println(" dF=" + state.docFreq);
+ if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
+ termState.totalTermFreq = termState.docFreq + statsReader.readVLong();
+ //if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq);
+ }
+ // metadata
+ for (int i = 0; i < longsSize; i++) {
+ longs[i] = bytesReader.readVLong();
+ }
+ postingsReader.decodeTerm(longs, bytesReader, fieldInfo, termState, absolute);
+
+ metaDataUpto++;
+ absolute = false;
+ }
+ termState.termBlockOrd = metaDataUpto;
+ }
+ }
+
+ private BytesRef savedStartTerm;
+
+ // TODO: in some cases we can filter by length? eg
+ // regexp foo*bar must be at least length 6 bytes
+ public IntersectEnum(CompiledAutomaton compiled, BytesRef startTerm) throws IOException {
+ // if (DEBUG) {
+ // System.out.println("\nintEnum.init seg=" + segment + " commonSuffix=" + brToString(compiled.commonSuffixRef));
+ // }
+ runAutomaton = compiled.runAutomaton;
+ compiledAutomaton = compiled;
+ in = BlockTreeTermsReader.this.in.clone();
+ stack = new Frame[5];
+ for(int idx=0;idx<stack.length;idx++) {
+ stack[idx] = new Frame(idx);
+ }
+ for(int arcIdx=0;arcIdx<arcs.length;arcIdx++) {
+ arcs[arcIdx] = new FST.Arc<BytesRef>();
+ }
+
+ if (index == null) {
+ fstReader = null;
+ } else {
+ fstReader = index.getBytesReader();
+ }
+
+ // TODO: if the automaton is "smallish" we really
+ // should use the terms index to seek at least to
+ // the initial term and likely to subsequent terms
+ // (or, maybe just fallback to ATE for such cases).
+ // Else the seek cost of loading the frames will be
+ // too costly.
+
+ final FST.Arc<BytesRef> arc = index.getFirstArc(arcs[0]);
+ // Empty string prefix must have an output in the index!
+ assert arc.isFinal();
+
+ // Special pushFrame since it's the first one:
+ final Frame f = stack[0];
+ f.fp = f.fpOrig = rootBlockFP;
+ f.prefix = 0;
+ f.setState(runAutomaton.getInitialState());
+ f.arc = arc;
+ f.outputPrefix = arc.output;
+ f.load(rootCode);
+
+ // for assert:
+ assert setSavedStartTerm(startTerm);
+
+ currentFrame = f;
+ if (startTerm != null) {
+ seekToStartTerm(startTerm);
+ }
+ }
+
+ // only for assert:
+ private boolean setSavedStartTerm(BytesRef startTerm) {
+ savedStartTerm = startTerm == null ? null : BytesRef.deepCopyOf(startTerm);
+ return true;
+ }
+
+ @Override
+ public TermState termState() throws IOException {
+ currentFrame.decodeMetaData();
+ return currentFrame.termState.clone();
+ }
+
+ private Frame getFrame(int ord) throws IOException {
+ if (ord >= stack.length) {
+ final Frame[] next = new Frame[ArrayUtil.oversize(1+ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(stack, 0, next, 0, stack.length);
+ for(int stackOrd=stack.length;stackOrd<next.length;stackOrd++) {
+ next[stackOrd] = new Frame(stackOrd);
+ }
+ stack = next;
+ }
+ assert stack[ord].ord == ord;
+ return stack[ord];
+ }
+
+ private FST.Arc<BytesRef> getArc(int ord) {
+ if (ord >= arcs.length) {
+ @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<BytesRef>[] next =
+ new FST.Arc[ArrayUtil.oversize(1+ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(arcs, 0, next, 0, arcs.length);
+ for(int arcOrd=arcs.length;arcOrd<next.length;arcOrd++) {
+ next[arcOrd] = new FST.Arc<BytesRef>();
+ }
+ arcs = next;
+ }
+ return arcs[ord];
+ }
+
+ private Frame pushFrame(int state) throws IOException {
+ final Frame f = getFrame(currentFrame == null ? 0 : 1+currentFrame.ord);
+
+ f.fp = f.fpOrig = currentFrame.lastSubFP;
+ f.prefix = currentFrame.prefix + currentFrame.suffix;
+ // if (DEBUG) System.out.println(" pushFrame state=" + state + " prefix=" + f.prefix);
+ f.setState(state);
+
+ // Walk the arc through the index -- we only
+ // "bother" with this so we can get the floor data
+ // from the index and skip floor blocks when
+ // possible:
+ FST.Arc<BytesRef> arc = currentFrame.arc;
+ int idx = currentFrame.prefix;
+ assert currentFrame.suffix > 0;
+ BytesRef output = currentFrame.outputPrefix;
+ while (idx < f.prefix) {
+ final int target = term.bytes[idx] & 0xff;
+ // TODO: we could be more efficient for the next()
+ // case by using current arc as starting point,
+ // passed to findTargetArc
+ arc = index.findTargetArc(target, arc, getArc(1+idx), fstReader);
+ assert arc != null;
+ output = fstOutputs.add(output, arc.output);
+ idx++;
+ }
+
+ f.arc = arc;
+ f.outputPrefix = output;
+ assert arc.isFinal();
+ f.load(fstOutputs.add(output, arc.nextFinalOutput));
+ return f;
+ }
+
+ @Override
+ public BytesRef term() {
+ return term;
+ }
+
+ @Override
+ public int docFreq() throws IOException {
+ //if (DEBUG) System.out.println("BTIR.docFreq");
+ currentFrame.decodeMetaData();
+ //if (DEBUG) System.out.println(" return " + currentFrame.termState.docFreq);
+ return currentFrame.termState.docFreq;
+ }
+
+ @Override
+ public long totalTermFreq() throws IOException {
+ currentFrame.decodeMetaData();
+ return currentFrame.termState.totalTermFreq;
+ }
+
+ @Override
+ public DocsEnum docs(Bits skipDocs, DocsEnum reuse, int flags) throws IOException {
+ currentFrame.decodeMetaData();
+ return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse, flags);
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+ // Positions were not indexed:
+ return null;
+ }
+
+ currentFrame.decodeMetaData();
+ return postingsReader.docsAndPositions(fieldInfo, currentFrame.termState, skipDocs, reuse, flags);
+ }
+
+ private int getState() {
+ int state = currentFrame.state;
+ for(int idx=0;idx<currentFrame.suffix;idx++) {
+ state = runAutomaton.step(state, currentFrame.suffixBytes[currentFrame.startBytePos+idx] & 0xff);
+ assert state != -1;
+ }
+ return state;
+ }
+
+ // NOTE: specialized to only doing the first-time
+ // seek, but we could generalize it to allow
+ // arbitrary seekExact/Ceil. Note that this is a
+ // seekFloor!
+ private void seekToStartTerm(BytesRef target) throws IOException {
+ //if (DEBUG) System.out.println("seek to startTerm=" + target.utf8ToString());
+ assert currentFrame.ord == 0;
+ if (term.length < target.length) {
+ term.bytes = ArrayUtil.grow(term.bytes, target.length);
+ }
+ FST.Arc<BytesRef> arc = arcs[0];
+ assert arc == currentFrame.arc;
+
+ for(int idx=0;idx<=target.length;idx++) {
+
+ while (true) {
+ final int savePos = currentFrame.suffixesReader.getPosition();
+ final int saveStartBytePos = currentFrame.startBytePos;
+ final int saveSuffix = currentFrame.suffix;
+ final long saveLastSubFP = currentFrame.lastSubFP;
+ final int saveTermBlockOrd = currentFrame.termState.termBlockOrd;
+
+ final boolean isSubBlock = currentFrame.next();
+
+ //if (DEBUG) System.out.println(" cycle ent=" + currentFrame.nextEnt + " (of " + currentFrame.entCount + ") prefix=" + currentFrame.prefix + " suffix=" + currentFrame.suffix + " isBlock=" + isSubBlock + " firstLabel=" + (currentFrame.suffix == 0 ? "" : (currentFrame.suffixBytes[currentFrame.startBytePos])&0xff));
+ term.length = currentFrame.prefix + currentFrame.suffix;
+ if (term.bytes.length < term.length) {
+ term.bytes = ArrayUtil.grow(term.bytes, term.length);
+ }
+ System.arraycopy(currentFrame.suffixBytes, currentFrame.startBytePos, term.bytes, currentFrame.prefix, currentFrame.suffix);
+
+ if (isSubBlock && StringHelper.startsWith(target, term)) {
+ // Recurse
+ //if (DEBUG) System.out.println(" recurse!");
+ currentFrame = pushFrame(getState());
+ break;
+ } else {
+ final int cmp = term.compareTo(target);
+ if (cmp < 0) {
+ if (currentFrame.nextEnt == currentFrame.entCount) {
+ if (!currentFrame.isLastInFloor) {
+ //if (DEBUG) System.out.println(" load floorBlock");
+ currentFrame.loadNextFloorBlock();
+ continue;
+ } else {
+ //if (DEBUG) System.out.println(" return term=" + brToString(term));
+ return;
+ }
+ }
+ continue;
+ } else if (cmp == 0) {
+ //if (DEBUG) System.out.println(" return term=" + brToString(term));
+ return;
+ } else {
+ // Fallback to prior entry: the semantics of
+ // this method is that the first call to
+ // next() will return the term after the
+ // requested term
+ currentFrame.nextEnt--;
+ currentFrame.lastSubFP = saveLastSubFP;
+ currentFrame.startBytePos = saveStartBytePos;
+ currentFrame.suffix = saveSuffix;
+ currentFrame.suffixesReader.setPosition(savePos);
+ currentFrame.termState.termBlockOrd = saveTermBlockOrd;
+ System.arraycopy(currentFrame.suffixBytes, currentFrame.startBytePos, term.bytes, currentFrame.prefix, currentFrame.suffix);
+ term.length = currentFrame.prefix + currentFrame.suffix;
+ // If the last entry was a block we don't
+ // need to bother recursing and pushing to
+ // the last term under it because the first
+ // next() will simply skip the frame anyway
+ return;
+ }
+ }
+ }
+ }
+
+ assert false;
+ }
+
+ @Override
+ public BytesRef next() throws IOException {
+
+ // if (DEBUG) {
+ // System.out.println("\nintEnum.next seg=" + segment);
+ // System.out.println(" frame ord=" + currentFrame.ord + " prefix=" + brToString(new BytesRef(term.bytes, term.offset, currentFrame.prefix)) + " state=" + currentFrame.state + " lastInFloor?=" + currentFrame.isLastInFloor + " fp=" + currentFrame.fp + " trans=" + (currentFrame.transitions.length == 0 ? "n/a" : currentFrame.transitions[currentFrame.transitionIndex]) + " outputPrefix=" + currentFrame.outputPrefix);
+ // }
+
+ nextTerm:
+ while(true) {
+ // Pop finished frames
+ while (currentFrame.nextEnt == currentFrame.entCount) {
+ if (!currentFrame.isLastInFloor) {
+ //if (DEBUG) System.out.println(" next-floor-block");
+ currentFrame.loadNextFloorBlock();
+ //if (DEBUG) System.out.println("\n frame ord=" + currentFrame.ord + " prefix=" + brToString(new BytesRef(term.bytes, term.offset, currentFrame.prefix)) + " state=" + currentFrame.state + " lastInFloor?=" + currentFrame.isLastInFloor + " fp=" + currentFrame.fp + " trans=" + (currentFrame.transitions.length == 0 ? "n/a" : currentFrame.transitions[currentFrame.transitionIndex]) + " outputPrefix=" + currentFrame.outputPrefix);
+ } else {
+ //if (DEBUG) System.out.println(" pop frame");
+ if (currentFrame.ord == 0) {
+ return null;
+ }
+ final long lastFP = currentFrame.fpOrig;
+ currentFrame = stack[currentFrame.ord-1];
+ assert currentFrame.lastSubFP == lastFP;
+ //if (DEBUG) System.out.println("\n frame ord=" + currentFrame.ord + " prefix=" + brToString(new BytesRef(term.bytes, term.offset, currentFrame.prefix)) + " state=" + currentFrame.state + " lastInFloor?=" + currentFrame.isLastInFloor + " fp=" + currentFrame.fp + " trans=" + (currentFrame.transitions.length == 0 ? "n/a" : currentFrame.transitions[currentFrame.transitionIndex]) + " outputPrefix=" + currentFrame.outputPrefix);
+ }
+ }
+
+ final boolean isSubBlock = currentFrame.next();
+ // if (DEBUG) {
+ // final BytesRef suffixRef = new BytesRef();
+ // suffixRef.bytes = currentFrame.suffixBytes;
+ // suffixRef.offset = currentFrame.startBytePos;
+ // suffixRef.length = currentFrame.suffix;
+ // System.out.println(" " + (isSubBlock ? "sub-block" : "term") + " " + currentFrame.nextEnt + " (of " + currentFrame.entCount + ") suffix=" + brToString(suffixRef));
+ // }
+
+ if (currentFrame.suffix != 0) {
+ final int label = currentFrame.suffixBytes[currentFrame.startBytePos] & 0xff;
+ while (label > currentFrame.curTransitionMax) {
+ if (currentFrame.transitionIndex >= currentFrame.transitions.length-1) {
+ // Stop processing this frame -- no further
+ // matches are possible because we've moved
+ // beyond what the max transition will allow
+ //if (DEBUG) System.out.println(" break: trans=" + (currentFrame.transitions.length == 0 ? "n/a" : currentFrame.transitions[currentFrame.transitionIndex]));
+
+ // sneaky! forces a pop above
+ currentFrame.isLastInFloor = true;
+ currentFrame.nextEnt = currentFrame.entCount;
+ continue nextTerm;
+ }
+ currentFrame.transitionIndex++;
+ currentFrame.curTransitionMax = currentFrame.transitions[currentFrame.transitionIndex].getMax();
+ //if (DEBUG) System.out.println(" next trans=" + currentFrame.transitions[currentFrame.transitionIndex]);
+ }
+ }
+
+ // First test the common suffix, if set:
+ if (compiledAutomaton.commonSuffixRef != null && !isSubBlock) {
+ final int termLen = currentFrame.prefix + currentFrame.suffix;
+ if (termLen < compiledAutomaton.commonSuffixRef.length) {
+ // No match
+ // if (DEBUG) {
+ // System.out.println(" skip: common suffix length");
+ // }
+ continue nextTerm;
+ }
+
+ final byte[] suffixBytes = currentFrame.suffixBytes;
+ final byte[] commonSuffixBytes = compiledAutomaton.commonSuffixRef.bytes;
+
+ final int lenInPrefix = compiledAutomaton.commonSuffixRef.length - currentFrame.suffix;
+ assert compiledAutomaton.commonSuffixRef.offset == 0;
+ int suffixBytesPos;
+ int commonSuffixBytesPos = 0;
+
+ if (lenInPrefix > 0) {
+ // A prefix of the common suffix overlaps with
+ // the suffix of the block prefix so we first
+ // test whether the prefix part matches:
+ final byte[] termBytes = term.bytes;
+ int termBytesPos = currentFrame.prefix - lenInPrefix;
+ assert termBytesPos >= 0;
+ final int termBytesPosEnd = currentFrame.prefix;
+ while (termBytesPos < termBytesPosEnd) {
+ if (termBytes[termBytesPos++] != commonSuffixBytes[commonSuffixBytesPos++]) {
+ // if (DEBUG) {
+ // System.out.println(" skip: common suffix mismatch (in prefix)");
+ // }
+ continue nextTerm;
+ }
+ }
+ suffixBytesPos = currentFrame.startBytePos;
+ } else {
+ suffixBytesPos = currentFrame.startBytePos + currentFrame.suffix - compiledAutomaton.commonSuffixRef.length;
+ }
+
+ // Test overlapping suffix part:
+ final int commonSuffixBytesPosEnd = compiledAutomaton.commonSuffixRef.length;
+ while (commonSuffixBytesPos < commonSuffixBytesPosEnd) {
+ if (suffixBytes[suffixBytesPos++] != commonSuffixBytes[commonSuffixBytesPos++]) {
+ // if (DEBUG) {
+ // System.out.println(" skip: common suffix mismatch");
+ // }
+ continue nextTerm;
+ }
+ }
+ }
+
+ // TODO: maybe we should do the same linear test
+ // that AutomatonTermsEnum does, so that if we
+ // reach a part of the automaton where .* is
+ // "temporarily" accepted, we just blindly .next()
+ // until the limit
+
+ // See if the term prefix matches the automaton:
+ int state = currentFrame.state;
+ for (int idx=0;idx<currentFrame.suffix;idx++) {
+ state = runAutomaton.step(state, currentFrame.suffixBytes[currentFrame.startBytePos+idx] & 0xff);
+ if (state == -1) {
+ // No match
+ //System.out.println(" no s=" + state);
+ continue nextTerm;
+ } else {
+ //System.out.println(" c s=" + state);
+ }
+ }
+
+ if (isSubBlock) {
+ // Match! Recurse:
+ //if (DEBUG) System.out.println(" sub-block match to state=" + state + "; recurse fp=" + currentFrame.lastSubFP);
+ copyTerm();
+ currentFrame = pushFrame(state);
+ //if (DEBUG) System.out.println("\n frame ord=" + currentFrame.ord + " prefix=" + brToString(new BytesRef(term.bytes, term.offset, currentFrame.prefix)) + " state=" + currentFrame.state + " lastInFloor?=" + currentFrame.isLastInFloor + " fp=" + currentFrame.fp + " trans=" + (currentFrame.transitions.length == 0 ? "n/a" : currentFrame.transitions[currentFrame.transitionIndex]) + " outputPrefix=" + currentFrame.outputPrefix);
+ } else if (runAutomaton.isAccept(state)) {
+ copyTerm();
+ //if (DEBUG) System.out.println(" term match to state=" + state + "; return term=" + brToString(term));
+ assert savedStartTerm == null || term.compareTo(savedStartTerm) > 0: "saveStartTerm=" + savedStartTerm.utf8ToString() + " term=" + term.utf8ToString();
+ return term;
+ } else {
+ //System.out.println(" no s=" + state);
+ }
+ }
+ }
+
+ private void copyTerm() {
+ //System.out.println(" copyTerm cur.prefix=" + currentFrame.prefix + " cur.suffix=" + currentFrame.suffix + " first=" + (char) currentFrame.suffixBytes[currentFrame.startBytePos]);
+ final int len = currentFrame.prefix + currentFrame.suffix;
+ if (term.bytes.length < len) {
+ term.bytes = ArrayUtil.grow(term.bytes, len);
+ }
+ System.arraycopy(currentFrame.suffixBytes, currentFrame.startBytePos, term.bytes, currentFrame.prefix, currentFrame.suffix);
+ term.length = len;
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public boolean seekExact(BytesRef text) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void seekExact(long ord) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long ord() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ // Iterates through terms in this field
+ private final class SegmentTermsEnum extends TermsEnum {
+ private IndexInput in;
+
+ private Frame[] stack;
+ private final Frame staticFrame;
+ private Frame currentFrame;
+ private boolean termExists;
+
+ private int targetBeforeCurrentLength;
+
+ private final ByteArrayDataInput scratchReader = new ByteArrayDataInput();
+
+ // What prefix of the current term was present in the index:
+ private int validIndexPrefix;
+
+ // assert only:
+ private boolean eof;
+
+ final BytesRef term = new BytesRef();
+ private final FST.BytesReader fstReader;
+
+ @SuppressWarnings({"rawtypes","unchecked"}) private FST.Arc<BytesRef>[] arcs =
+ new FST.Arc[1];
+
+ public SegmentTermsEnum() throws IOException {
+ //if (DEBUG) System.out.println("BTTR.init seg=" + segment);
+ stack = new Frame[0];
+
+ // Used to hold seek by TermState, or cached seek
+ staticFrame = new Frame(-1);
+
+ if (index == null) {
+ fstReader = null;
+ } else {
+ fstReader = index.getBytesReader();
+ }
+
+ // Init w/ root block; don't use index since it may
+ // not (and need not) have been loaded
+ for(int arcIdx=0;arcIdx<arcs.length;arcIdx++) {
+ arcs[arcIdx] = new FST.Arc<BytesRef>();
+ }
+
+ currentFrame = staticFrame;
+ final FST.Arc<BytesRef> arc;
+ if (index != null) {
+ arc = index.getFirstArc(arcs[0]);
+ // Empty string prefix must have an output in the index!
+ assert arc.isFinal();
+ } else {
+ arc = null;
+ }
+ currentFrame = staticFrame;
+ //currentFrame = pushFrame(arc, rootCode, 0);
+ //currentFrame.loadBlock();
+ validIndexPrefix = 0;
+ // if (DEBUG) {
+ // System.out.println("init frame state " + currentFrame.ord);
+ // printSeekState();
+ // }
+
+ //System.out.println();
+ // computeBlockStats().print(System.out);
+ }
+
+ // Not private to avoid synthetic access$NNN methods
+ void initIndexInput() {
+ if (this.in == null) {
+ this.in = BlockTreeTermsReader.this.in.clone();
+ }
+ }
+
+ /** Runs next() through the entire terms dict,
+ * computing aggregate statistics. */
+ public Stats computeBlockStats() throws IOException {
+
+ Stats stats = new Stats(segment, fieldInfo.name);
+ if (index != null) {
+ stats.indexNodeCount = index.getNodeCount();
+ stats.indexArcCount = index.getArcCount();
+ stats.indexNumBytes = index.sizeInBytes();
+ }
+
+ currentFrame = staticFrame;
+ FST.Arc<BytesRef> arc;
+ if (index != null) {
+ arc = index.getFirstArc(arcs[0]);
+ // Empty string prefix must have an output in the index!
+ assert arc.isFinal();
+ } else {
+ arc = null;
+ }
+
+ // Empty string prefix must have an output in the
+ // index!
+ currentFrame = pushFrame(arc, rootCode, 0);
+ currentFrame.fpOrig = currentFrame.fp;
+ currentFrame.loadBlock();
+ validIndexPrefix = 0;
+
+ stats.startBlock(currentFrame, !currentFrame.isLastInFloor);
+
+ allTerms:
+ while (true) {
+
+ // Pop finished blocks
+ while (currentFrame.nextEnt == currentFrame.entCount) {
+ stats.endBlock(currentFrame);
+ if (!currentFrame.isLastInFloor) {
+ currentFrame.loadNextFloorBlock();
+ stats.startBlock(currentFrame, true);
+ } else {
+ if (currentFrame.ord == 0) {
+ break allTerms;
+ }
+ final long lastFP = currentFrame.fpOrig;
+ currentFrame = stack[currentFrame.ord-1];
+ assert lastFP == currentFrame.lastSubFP;
+ // if (DEBUG) {
+ // System.out.println(" reset validIndexPrefix=" + validIndexPrefix);
+ // }
+ }
+ }
+
+ while(true) {
+ if (currentFrame.next()) {
+ // Push to new block:
+ currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length);
+ currentFrame.fpOrig = currentFrame.fp;
+ // This is a "next" frame -- even if it's
+ // floor'd we must pretend it isn't so we don't
+ // try to scan to the right floor frame:
+ currentFrame.isFloor = false;
+ //currentFrame.hasTerms = true;
+ currentFrame.loadBlock();
+ stats.startBlock(currentFrame, !currentFrame.isLastInFloor);
+ } else {
+ stats.term(term);
+ break;
+ }
+ }
+ }
+
+ stats.finish();
+
+ // Put root frame back:
+ currentFrame = staticFrame;
+ if (index != null) {
+ arc = index.getFirstArc(arcs[0]);
+ // Empty string prefix must have an output in the index!
+ assert arc.isFinal();
+ } else {
+ arc = null;
+ }
+ currentFrame = pushFrame(arc, rootCode, 0);
+ currentFrame.rewind();
+ currentFrame.loadBlock();
+ validIndexPrefix = 0;
+ term.length = 0;
+
+ return stats;
+ }
+
+ private Frame getFrame(int ord) throws IOException {
+ if (ord >= stack.length) {
+ final Frame[] next = new Frame[ArrayUtil.oversize(1+ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(stack, 0, next, 0, stack.length);
+ for(int stackOrd=stack.length;stackOrd<next.length;stackOrd++) {
+ next[stackOrd] = new Frame(stackOrd);
+ }
+ stack = next;
+ }
+ assert stack[ord].ord == ord;
+ return stack[ord];
+ }
+
+ private FST.Arc<BytesRef> getArc(int ord) {
+ if (ord >= arcs.length) {
+ @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<BytesRef>[] next =
+ new FST.Arc[ArrayUtil.oversize(1+ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(arcs, 0, next, 0, arcs.length);
+ for(int arcOrd=arcs.length;arcOrd<next.length;arcOrd++) {
+ next[arcOrd] = new FST.Arc<BytesRef>();
+ }
+ arcs = next;
+ }
+ return arcs[ord];
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ // Pushes a frame we seek'd to
+ Frame pushFrame(FST.Arc<BytesRef> arc, BytesRef frameData, int length) throws IOException {
+ scratchReader.reset(frameData.bytes, frameData.offset, frameData.length);
+ final long code = scratchReader.readVLong();
+ final long fpSeek = code >>> BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS;
+ final Frame f = getFrame(1+currentFrame.ord);
+ f.hasTerms = (code & BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS) != 0;
+ f.hasTermsOrig = f.hasTerms;
+ f.isFloor = (code & BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR) != 0;
+ if (f.isFloor) {
+ f.setFloorData(scratchReader, frameData);
+ }
+ pushFrame(arc, fpSeek, length);
+
+ return f;
+ }
+
+ // Pushes next'd frame or seek'd frame; we later
+ // lazy-load the frame only when needed
+ Frame pushFrame(FST.Arc<BytesRef> arc, long fp, int length) throws IOException {
+ final Frame f = getFrame(1+currentFrame.ord);
+ f.arc = arc;
+ if (f.fpOrig == fp && f.nextEnt != -1) {
+ //if (DEBUG) System.out.println(" push reused frame ord=" + f.ord + " fp=" + f.fp + " isFloor?=" + f.isFloor + " hasTerms=" + f.hasTerms + " pref=" + term + " nextEnt=" + f.nextEnt + " targetBeforeCurrentLength=" + targetBeforeCurrentLength + " term.length=" + term.length + " vs prefix=" + f.prefix);
+ if (f.prefix > targetBeforeCurrentLength) {
+ f.rewind();
+ } else {
+ // if (DEBUG) {
+ // System.out.println(" skip rewind!");
+ // }
+ }
+ assert length == f.prefix;
+ } else {
+ f.nextEnt = -1;
+ f.prefix = length;
+ f.state.termBlockOrd = 0;
+ f.fpOrig = f.fp = fp;
+ f.lastSubFP = -1;
+ // if (DEBUG) {
+ // final int sav = term.length;
+ // term.length = length;
+ // System.out.println(" push new frame ord=" + f.ord + " fp=" + f.fp + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " pref=" + brToString(term));
+ // term.length = sav;
+ // }
+ }
+
+ return f;
+ }
+
+ // asserts only
+ private boolean clearEOF() {
+ eof = false;
+ return true;
+ }
+
+ // asserts only
+ private boolean setEOF() {
+ eof = true;
+ return true;
+ }
+
+ @Override
+ public boolean seekExact(final BytesRef target) throws IOException {
+
+ if (index == null) {
+ throw new IllegalStateException("terms index was not loaded");
+ }
+
+ if (term.bytes.length <= target.length) {
+ term.bytes = ArrayUtil.grow(term.bytes, 1+target.length);
+ }
+
+ assert clearEOF();
+
+ // if (DEBUG) {
+ // System.out.println("\nBTTR.seekExact seg=" + segment + " target=" + fieldInfo.name + ":" + brToString(target) + " current=" + brToString(term) + " (exists?=" + termExists + ") validIndexPrefix=" + validIndexPrefix);
+ // printSeekState();
+ // }
+
+ FST.Arc<BytesRef> arc;
+ int targetUpto;
+ BytesRef output;
+
+ targetBeforeCurrentLength = currentFrame.ord;
+
+ if (currentFrame != staticFrame) {
+
+ // We are already seek'd; find the common
+ // prefix of new seek term vs current term and
+ // re-use the corresponding seek state. For
+ // example, if app first seeks to foobar, then
+ // seeks to foobaz, we can re-use the seek state
+ // for the first 5 bytes.
+
+ // if (DEBUG) {
+ // System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);
+ // }
+
+ arc = arcs[0];
+ assert arc.isFinal();
+ output = arc.output;
+ targetUpto = 0;
+
+ Frame lastFrame = stack[0];
+ assert validIndexPrefix <= term.length;
+
+ final int targetLimit = Math.min(target.length, validIndexPrefix);
+
+ int cmp = 0;
+
+ // TODO: reverse vLong byte order for better FST
+ // prefix output sharing
+
+ // First compare up to valid seek frames:
+ while (targetUpto < targetLimit) {
+ cmp = (term.bytes[targetUpto]&0xFF) - (target.bytes[target.offset + targetUpto]&0xFF);
+ // if (DEBUG) {
+ // System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) + " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")" + " arc.output=" + arc.output + " output=" + output);
+ // }
+ if (cmp != 0) {
+ break;
+ }
+ arc = arcs[1+targetUpto];
+ //if (arc.label != (target.bytes[target.offset + targetUpto] & 0xFF)) {
+ //System.out.println("FAIL: arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF));
+ //}
+ assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF);
+ if (arc.output != NO_OUTPUT) {
+ output = fstOutputs.add(output, arc.output);
+ }
+ if (arc.isFinal()) {
+ lastFrame = stack[1+lastFrame.ord];
+ }
+ targetUpto++;
+ }
+
+ if (cmp == 0) {
+ final int targetUptoMid = targetUpto;
+
+ // Second compare the rest of the term, but
+ // don't save arc/output/frame; we only do this
+ // to find out if the target term is before,
+ // equal or after the current term
+ final int targetLimit2 = Math.min(target.length, term.length);
+ while (targetUpto < targetLimit2) {
+ cmp = (term.bytes[targetUpto]&0xFF) - (target.bytes[target.offset + targetUpto]&0xFF);
+ // if (DEBUG) {
+ // System.out.println(" cycle2 targetUpto=" + targetUpto + " (vs limit=" + targetLimit + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) + " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")");
+ // }
+ if (cmp != 0) {
+ break;
+ }
+ targetUpto++;
+ }
+
+ if (cmp == 0) {
+ cmp = term.length - target.length;
+ }
+ targetUpto = targetUptoMid;
+ }
+
+ if (cmp < 0) {
+ // Common case: target term is after current
+ // term, ie, app is seeking multiple terms
+ // in sorted order
+ // if (DEBUG) {
+ // System.out.println(" target is after current (shares prefixLen=" + targetUpto + "); frame.ord=" + lastFrame.ord);
+ // }
+ currentFrame = lastFrame;
+
+ } else if (cmp > 0) {
+ // Uncommon case: target term
+ // is before current term; this means we can
+ // keep the currentFrame but we must rewind it
+ // (so we scan from the start)
+ targetBeforeCurrentLength = 0;
+ // if (DEBUG) {
+ // System.out.println(" target is before current (shares prefixLen=" + targetUpto + "); rewind frame ord=" + lastFrame.ord);
+ // }
+ currentFrame = lastFrame;
+ currentFrame.rewind();
+ } else {
+ // Target is exactly the same as current term
+ assert term.length == target.length;
+ if (termExists) {
+ // if (DEBUG) {
+ // System.out.println(" target is same as current; return true");
+ // }
+ return true;
+ } else {
+ // if (DEBUG) {
+ // System.out.println(" target is same as current but term doesn't exist");
+ // }
+ }
+ //validIndexPrefix = currentFrame.depth;
+ //term.length = target.length;
+ //return termExists;
+ }
+
+ } else {
+
+ targetBeforeCurrentLength = -1;
+ arc = index.getFirstArc(arcs[0]);
+
+ // Empty string prefix must have an output (block) in the index!
+ assert arc.isFinal();
+ assert arc.output != null;
+
+ // if (DEBUG) {
+ // System.out.println(" no seek state; push root frame");
+ // }
+
+ output = arc.output;
+
+ currentFrame = staticFrame;
+
+ //term.length = 0;
+ targetUpto = 0;
+ currentFrame = pushFrame(arc, fstOutputs.add(output, arc.nextFinalOutput), 0);
+ }
+
+ // if (DEBUG) {
+ // System.out.println(" start index loop targetUpto=" + targetUpto + " output=" + output + " currentFrame.ord=" + currentFrame.ord + " targetBeforeCurrentLength=" + targetBeforeCurrentLength);
+ // }
+
+ while (targetUpto < target.length) {
+
+ final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;
+
+ final FST.Arc<BytesRef> nextArc = index.findTargetArc(targetLabel, arc, getArc(1+targetUpto), fstReader);
+
+ if (nextArc == null) {
+
+ // Index is exhausted
+ // if (DEBUG) {
+ // System.out.println(" index: index exhausted label=" + ((char) targetLabel) + " " + toHex(targetLabel));
+ // }
+
+ validIndexPrefix = currentFrame.prefix;
+ //validIndexPrefix = targetUpto;
+
+ currentFrame.scanToFloorFrame(target);
+
+ if (!currentFrame.hasTerms) {
+ termExists = false;
+ term.bytes[targetUpto] = (byte) targetLabel;
+ term.length = 1+targetUpto;
+ // if (DEBUG) {
+ // System.out.println(" FAST NOT_FOUND term=" + brToString(term));
+ // }
+ return false;
+ }
+
+ currentFrame.loadBlock();
+
+ final SeekStatus result = currentFrame.scanToTerm(target, true);
+ if (result == SeekStatus.FOUND) {
+ // if (DEBUG) {
+ // System.out.println(" return FOUND term=" + term.utf8ToString() + " " + term);
+ // }
+ return true;
+ } else {
+ // if (DEBUG) {
+ // System.out.println(" got " + result + "; return NOT_FOUND term=" + brToString(term));
+ // }
+ return false;
+ }
+ } else {
+ // Follow this arc
+ arc = nextArc;
+ term.bytes[targetUpto] = (byte) targetLabel;
+ // Aggregate output as we go:
+ assert arc.output != null;
+ if (arc.output != NO_OUTPUT) {
+ output = fstOutputs.add(output, arc.output);
+ }
+
+ // if (DEBUG) {
+ // System.out.println(" index: follow label=" + toHex(target.bytes[target.offset + targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);
+ // }
+ targetUpto++;
+
+ if (arc.isFinal()) {
+ //if (DEBUG) System.out.println(" arc is final!");
+ currentFrame = pushFrame(arc, fstOutputs.add(output, arc.nextFinalOutput), targetUpto);
+ //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms);
+ }
+ }
+ }
+
+ //validIndexPrefix = targetUpto;
+ validIndexPrefix = currentFrame.prefix;
+
+ currentFrame.scanToFloorFrame(target);
+
+ // Target term is entirely contained in the index:
+ if (!currentFrame.hasTerms) {
+ termExists = false;
+ term.length = targetUpto;
+ // if (DEBUG) {
+ // System.out.println(" FAST NOT_FOUND term=" + brToString(term));
+ // }
+ return false;
+ }
+
+ currentFrame.loadBlock();
+
+ final SeekStatus result = currentFrame.scanToTerm(target, true);
+ if (result == SeekStatus.FOUND) {
+ // if (DEBUG) {
+ // System.out.println(" return FOUND term=" + term.utf8ToString() + " " + term);
+ // }
+ return true;
+ } else {
+ // if (DEBUG) {
+ // System.out.println(" got result " + result + "; return NOT_FOUND term=" + term.utf8ToString());
+ // }
+
+ return false;
+ }
+ }
+
+ @Override
+ public SeekStatus seekCeil(final BytesRef target) throws IOException {
+ if (index == null) {
+ throw new IllegalStateException("terms index was not loaded");
+ }
+
+ if (term.bytes.length <= target.length) {
+ term.bytes = ArrayUtil.grow(term.bytes, 1+target.length);
+ }
+
+ assert clearEOF();
+
+ //if (DEBUG) {
+ //System.out.println("\nBTTR.seekCeil seg=" + segment + " target=" + fieldInfo.name + ":" + target.utf8ToString() + " " + target + " current=" + brToString(term) + " (exists?=" + termExists + ") validIndexPrefix= " + validIndexPrefix);
+ //printSeekState();
+ //}
+
+ FST.Arc<BytesRef> arc;
+ int targetUpto;
+ BytesRef output;
+
+ targetBeforeCurrentLength = currentFrame.ord;
+
+ if (currentFrame != staticFrame) {
+
+ // We are already seek'd; find the common
+ // prefix of new seek term vs current term and
+ // re-use the corresponding seek state. For
+ // example, if app first seeks to foobar, then
+ // seeks to foobaz, we can re-use the seek state
+ // for the first 5 bytes.
+
+ //if (DEBUG) {
+ //System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);
+ //}
+
+ arc = arcs[0];
+ assert arc.isFinal();
+ output = arc.output;
+ targetUpto = 0;
+
+ Frame lastFrame = stack[0];
+ assert validIndexPrefix <= term.length;
+
+ final int targetLimit = Math.min(target.length, validIndexPrefix);
+
+ int cmp = 0;
+
+ // TOOD: we should write our vLong backwards (MSB
+ // first) to get better sharing from the FST
+
+ // First compare up to valid seek frames:
+ while (targetUpto < targetLimit) {
+ cmp = (term.bytes[targetUpto]&0xFF) - (target.bytes[target.offset + targetUpto]&0xFF);
+ //if (DEBUG) {
+ //System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) + " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")" + " arc.output=" + arc.output + " output=" + output);
+ //}
+ if (cmp != 0) {
+ break;
+ }
+ arc = arcs[1+targetUpto];
+ assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF);
+ // TOOD: we could save the outputs in local
+ // byte[][] instead of making new objs ever
+ // seek; but, often the FST doesn't have any
+ // shared bytes (but this could change if we
+ // reverse vLong byte order)
+ if (arc.output != NO_OUTPUT) {
+ output = fstOutputs.add(output, arc.output);
+ }
+ if (arc.isFinal()) {
+ lastFrame = stack[1+lastFrame.ord];
+ }
+ targetUpto++;
+ }
+
+
+ if (cmp == 0) {
+ final int targetUptoMid = targetUpto;
+ // Second compare the rest of the term, but
+ // don't save arc/output/frame:
+ final int targetLimit2 = Math.min(target.length, term.length);
+ while (targetUpto < targetLimit2) {
+ cmp = (term.bytes[targetUpto]&0xFF) - (target.bytes[target.offset + targetUpto]&0xFF);
+ //if (DEBUG) {
+ //System.out.println(" cycle2 targetUpto=" + targetUpto + " (vs limit=" + targetLimit + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) + " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")");
+ //}
+ if (cmp != 0) {
+ break;
+ }
+ targetUpto++;
+ }
+
+ if (cmp == 0) {
+ cmp = term.length - target.length;
+ }
+ targetUpto = targetUptoMid;
+ }
+
+ if (cmp < 0) {
+ // Common case: target term is after current
+ // term, ie, app is seeking multiple terms
+ // in sorted order
+ //if (DEBUG) {
+ //System.out.println(" target is after current (shares prefixLen=" + targetUpto + "); clear frame.scanned ord=" + lastFrame.ord);
+ //}
+ currentFrame = lastFrame;
+
+ } else if (cmp > 0) {
+ // Uncommon case: target term
+ // is before current term; this means we can
+ // keep the currentFrame but we must rewind it
+ // (so we scan from the start)
+ targetBeforeCurrentLength = 0;
+ //if (DEBUG) {
+ //System.out.println(" target is before current (shares prefixLen=" + targetUpto + "); rewind frame ord=" + lastFrame.ord);
+ //}
+ currentFrame = lastFrame;
+ currentFrame.rewind();
+ } else {
+ // Target is exactly the same as current term
+ assert term.length == target.length;
+ if (termExists) {
+ //if (DEBUG) {
+ //System.out.println(" target is same as current; return FOUND");
+ //}
+ return SeekStatus.FOUND;
+ } else {
+ //if (DEBUG) {
+ //System.out.println(" target is same as current but term doesn't exist");
+ //}
+ }
+ }
+
+ } else {
+
+ targetBeforeCurrentLength = -1;
+ arc = index.getFirstArc(arcs[0]);
+
+ // Empty string prefix must have an output (block) in the index!
+ assert arc.isFinal();
+ assert arc.output != null;
+
+ //if (DEBUG) {
+ //System.out.println(" no seek state; push root frame");
+ //}
+
+ output = arc.output;
+
+ currentFrame = staticFrame;
+
+ //term.length = 0;
+ targetUpto = 0;
+ currentFrame = pushFrame(arc, fstOutputs.add(output, arc.nextFinalOutput), 0);
+ }
+
+ //if (DEBUG) {
+ //System.out.println(" start index loop targetUpto=" + targetUpto + " output=" + output + " currentFrame.ord+1=" + currentFrame.ord + " targetBeforeCurrentLength=" + targetBeforeCurrentLength);
+ //}
+
+ while (targetUpto < target.length) {
+
+ final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;
+
+ final FST.Arc<BytesRef> nextArc = index.findTargetArc(targetLabel, arc, getArc(1+targetUpto), fstReader);
+
+ if (nextArc == null) {
+
+ // Index is exhausted
+ // if (DEBUG) {
+ // System.out.println(" index: index exhausted label=" + ((char) targetLabel) + " " + toHex(targetLabel));
+ // }
+
+ validIndexPrefix = currentFrame.prefix;
+ //validIndexPrefix = targetUpto;
+
+ currentFrame.scanToFloorFrame(target);
+
+ currentFrame.loadBlock();
+
+ final SeekStatus result = currentFrame.scanToTerm(target, false);
+ if (result == SeekStatus.END) {
+ term.copyBytes(target);
+ termExists = false;
+
+ if (next() != null) {
+ //if (DEBUG) {
+ //System.out.println(" return NOT_FOUND term=" + brToString(term) + " " + term);
+ //}
+ return SeekStatus.NOT_FOUND;
+ } else {
+ //if (DEBUG) {
+ //System.out.println(" return END");
+ //}
+ return SeekStatus.END;
+ }
+ } else {
+ //if (DEBUG) {
+ //System.out.println(" return " + result + " term=" + brToString(term) + " " + term);
+ //}
+ return result;
+ }
+ } else {
+ // Follow this arc
+ term.bytes[targetUpto] = (byte) targetLabel;
+ arc = nextArc;
+ // Aggregate output as we go:
+ assert arc.output != null;
+ if (arc.output != NO_OUTPUT) {
+ output = fstOutputs.add(output, arc.output);
+ }
+
+ //if (DEBUG) {
+ //System.out.println(" index: follow label=" + toHex(target.bytes[target.offset + targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);
+ //}
+ targetUpto++;
+
+ if (arc.isFinal()) {
+ //if (DEBUG) System.out.println(" arc is final!");
+ currentFrame = pushFrame(arc, fstOutputs.add(output, arc.nextFinalOutput), targetUpto);
+ //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms);
+ }
+ }
+ }
+
+ //validIndexPrefix = targetUpto;
+ validIndexPrefix = currentFrame.prefix;
+
+ currentFrame.scanToFloorFrame(target);
+
+ currentFrame.loadBlock();
+
+ final SeekStatus result = currentFrame.scanToTerm(target, false);
+
+ if (result == SeekStatus.END) {
+ term.copyBytes(target);
+ termExists = false;
+ if (next() != null) {
+ //if (DEBUG) {
+ //System.out.println(" return NOT_FOUND term=" + term.utf8ToString() + " " + term);
+ //}
+ return SeekStatus.NOT_FOUND;
+ } else {
+ //if (DEBUG) {
+ //System.out.println(" return END");
+ //}
+ return SeekStatus.END;
+ }
+ } else {
+ return result;
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private void printSeekState(PrintStream out) throws IOException {
+ if (currentFrame == staticFrame) {
+ out.println(" no prior seek");
+ } else {
+ out.println(" prior seek state:");
+ int ord = 0;
+ boolean isSeekFrame = true;
+ while(true) {
+ Frame f = getFrame(ord);
+ assert f != null;
+ final BytesRef prefix = new BytesRef(term.bytes, 0, f.prefix);
+ if (f.nextEnt == -1) {
+ out.println(" frame " + (isSeekFrame ? "(seek)" : "(next)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<<BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) + (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FL [...]
+ } else {
+ out.println(" frame " + (isSeekFrame ? "(seek, loaded)" : "(next, loaded)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + " nextEnt=" + f.nextEnt + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<<BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) + ( [...]
+ }
+ if (index != null) {
+ assert !isSeekFrame || f.arc != null: "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc;
+ if (f.prefix > 0 && isSeekFrame && f.arc.label != (term.bytes[f.prefix-1]&0xFF)) {
+ out.println(" broken seek state: arc.label=" + (char) f.arc.label + " vs term byte=" + (char) (term.bytes[f.prefix-1]&0xFF));
+ throw new RuntimeException("seek state is broken");
+ }
+ BytesRef output = Util.get(index, prefix);
+ if (output == null) {
+ out.println(" broken seek state: prefix is not final in index");
+ throw new RuntimeException("seek state is broken");
+ } else if (isSeekFrame && !f.isFloor) {
+ final ByteArrayDataInput reader = new ByteArrayDataInput(output.bytes, output.offset, output.length);
+ final long codeOrig = reader.readVLong();
+ final long code = (f.fp << BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) | (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) | (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR:0);
+ if (codeOrig != code) {
+ out.println(" broken seek state: output code=" + codeOrig + " doesn't match frame code=" + code);
+ throw new RuntimeException("seek state is broken");
+ }
+ }
+ }
+ if (f == currentFrame) {
+ break;
+ }
+ if (f.prefix == validIndexPrefix) {
+ isSeekFrame = false;
+ }
+ ord++;
+ }
+ }
+ }
+
+ /* Decodes only the term bytes of the next term. If caller then asks for
+ metadata, ie docFreq, totalTermFreq or pulls a D/&PEnum, we then (lazily)
+ decode all metadata up to the current term. */
+ @Override
+ public BytesRef next() throws IOException {
+
+ if (in == null) {
+ // Fresh TermsEnum; seek to first term:
+ final FST.Arc<BytesRef> arc;
+ if (index != null) {
+ arc = index.getFirstArc(arcs[0]);
+ // Empty string prefix must have an output in the index!
+ assert arc.isFinal();
+ } else {
+ arc = null;
+ }
+ currentFrame = pushFrame(arc, rootCode, 0);
+ currentFrame.loadBlock();
+ }
+
+ targetBeforeCurrentLength = currentFrame.ord;
+
+ assert !eof;
+ //if (DEBUG) {
+ //System.out.println("\nBTTR.next seg=" + segment + " term=" + brToString(term) + " termExists?=" + termExists + " field=" + fieldInfo.name + " termBlockOrd=" + currentFrame.state.termBlockOrd + " validIndexPrefix=" + validIndexPrefix);
+ //printSeekState();
+ //}
+
+ if (currentFrame == staticFrame) {
+ // If seek was previously called and the term was
+ // cached, or seek(TermState) was called, usually
+ // caller is just going to pull a D/&PEnum or get
+ // docFreq, etc. But, if they then call next(),
+ // this method catches up all internal state so next()
+ // works properly:
+ //if (DEBUG) System.out.println(" re-seek to pending term=" + term.utf8ToString() + " " + term);
+ final boolean result = seekExact(term);
+ assert result;
+ }
+
+ // Pop finished blocks
+ while (currentFrame.nextEnt == currentFrame.entCount) {
+ if (!currentFrame.isLastInFloor) {
+ currentFrame.loadNextFloorBlock();
+ } else {
+ //if (DEBUG) System.out.println(" pop frame");
+ if (currentFrame.ord == 0) {
+ //if (DEBUG) System.out.println(" return null");
+ assert setEOF();
+ term.length = 0;
+ validIndexPrefix = 0;
+ currentFrame.rewind();
+ termExists = false;
+ return null;
+ }
+ final long lastFP = currentFrame.fpOrig;
+ currentFrame = stack[currentFrame.ord-1];
+
+ if (currentFrame.nextEnt == -1 || currentFrame.lastSubFP != lastFP) {
+ // We popped into a frame that's not loaded
+ // yet or not scan'd to the right entry
+ currentFrame.scanToFloorFrame(term);
+ currentFrame.loadBlock();
+ currentFrame.scanToSubBlock(lastFP);
+ }
+
+ // Note that the seek state (last seek) has been
+ // invalidated beyond this depth
+ validIndexPrefix = Math.min(validIndexPrefix, currentFrame.prefix);
+ //if (DEBUG) {
+ //System.out.println(" reset validIndexPrefix=" + validIndexPrefix);
+ //}
+ }
+ }
+
+ while(true) {
+ if (currentFrame.next()) {
+ // Push to new block:
+ //if (DEBUG) System.out.println(" push frame");
+ currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length);
+ // This is a "next" frame -- even if it's
+ // floor'd we must pretend it isn't so we don't
+ // try to scan to the right floor frame:
+ currentFrame.isFloor = false;
+ //currentFrame.hasTerms = true;
+ currentFrame.loadBlock();
+ } else {
+ //if (DEBUG) System.out.println(" return term=" + term.utf8ToString() + " " + term + " currentFrame.ord=" + currentFrame.ord);
+ return term;
+ }
+ }
+ }
+
+ @Override
+ public BytesRef term() {
+ assert !eof;
+ return term;
+ }
+
+ @Override
+ public int docFreq() throws IOException {
+ assert !eof;
+ //if (DEBUG) System.out.println("BTR.docFreq");
+ currentFrame.decodeMetaData();
+ //if (DEBUG) System.out.println(" return " + currentFrame.state.docFreq);
+ return currentFrame.state.docFreq;
+ }
+
+ @Override
+ public long totalTermFreq() throws IOException {
+ assert !eof;
+ currentFrame.decodeMetaData();
+ return currentFrame.state.totalTermFreq;
+ }
+
+ @Override
+ public DocsEnum docs(Bits skipDocs, DocsEnum reuse, int flags) throws IOException {
+ assert !eof;
+ //if (DEBUG) {
+ //System.out.println("BTTR.docs seg=" + segment);
+ //}
+ currentFrame.decodeMetaData();
+ //if (DEBUG) {
+ //System.out.println(" state=" + currentFrame.state);
+ //}
+ return postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse, flags);
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+ // Positions were not indexed:
+ return null;
+ }
+
+ assert !eof;
+ currentFrame.decodeMetaData();
+ return postingsReader.docsAndPositions(fieldInfo, currentFrame.state, skipDocs, reuse, flags);
+ }
+
+ @Override
+ public void seekExact(BytesRef target, TermState otherState) {
+ // if (DEBUG) {
+ // System.out.println("BTTR.seekExact termState seg=" + segment + " target=" + target.utf8ToString() + " " + target + " state=" + otherState);
+ // }
+ assert clearEOF();
+ if (target.compareTo(term) != 0 || !termExists) {
+ assert otherState != null && otherState instanceof BlockTermState;
+ currentFrame = staticFrame;
+ currentFrame.state.copyFrom(otherState);
+ term.copyBytes(target);
+ currentFrame.metaDataUpto = currentFrame.getTermBlockOrd();
+ assert currentFrame.metaDataUpto > 0;
+ validIndexPrefix = 0;
+ } else {
+ // if (DEBUG) {
+ // System.out.println(" skip seek: already on target state=" + currentFrame.state);
+ // }
+ }
+ }
+
+ @Override
+ public TermState termState() throws IOException {
+ assert !eof;
+ currentFrame.decodeMetaData();
+ TermState ts = currentFrame.state.clone();
+ //if (DEBUG) System.out.println("BTTR.termState seg=" + segment + " state=" + ts);
+ return ts;
+ }
+
+ @Override
+ public void seekExact(long ord) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long ord() {
+ throw new UnsupportedOperationException();
+ }
+
+ // Not static -- references term, postingsReader,
+ // fieldInfo, in
+ private final class Frame {
+ // Our index in stack[]:
+ final int ord;
+
+ boolean hasTerms;
+ boolean hasTermsOrig;
+ boolean isFloor;
+
+ FST.Arc<BytesRef> arc;
+
+ // File pointer where this block was loaded from
+ long fp;
+ long fpOrig;
+ long fpEnd;
+
+ byte[] suffixBytes = new byte[128];
+ final ByteArrayDataInput suffixesReader = new ByteArrayDataInput();
+
+ byte[] statBytes = new byte[64];
+ final ByteArrayDataInput statsReader = new ByteArrayDataInput();
+
+ byte[] floorData = new byte[32];
+ final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();
+
+ // Length of prefix shared by all terms in this block
+ int prefix;
+
+ // Number of entries (term or sub-block) in this block
+ int entCount;
+
+ // Which term we will next read, or -1 if the block
+ // isn't loaded yet
+ int nextEnt;
+
+ // True if this block is either not a floor block,
+ // or, it's the last sub-block of a floor block
+ boolean isLastInFloor;
+
+ // True if all entries are terms
+ boolean isLeafBlock;
+
+ long lastSubFP;
+
+ int nextFloorLabel;
+ int numFollowFloorBlocks;
+
+ // Next term to decode metaData; we decode metaData
+ // lazily so that scanning to find the matching term is
+ // fast and only if you find a match and app wants the
+ // stats or docs/positions enums, will we decode the
+ // metaData
+ int metaDataUpto;
+
+ final BlockTermState state;
+
+ // metadata buffer, holding monotonic values
+ public long[] longs;
+ // metadata buffer, holding general values
+ public byte[] bytes;
+ ByteArrayDataInput bytesReader;
+
+ public Frame(int ord) throws IOException {
+ this.ord = ord;
+ this.state = postingsReader.newTermState();
+ this.state.totalTermFreq = -1;
+ this.longs = new long[longsSize];
+ }
+
+ public void setFloorData(ByteArrayDataInput in, BytesRef source) {
+ final int numBytes = source.length - (in.getPosition() - source.offset);
+ if (numBytes > floorData.length) {
+ floorData = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ System.arraycopy(source.bytes, source.offset+in.getPosition(), floorData, 0, numBytes);
+ floorDataReader.reset(floorData, 0, numBytes);
+ numFollowFloorBlocks = floorDataReader.readVInt();
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ //if (DEBUG) {
+ //System.out.println(" setFloorData fpOrig=" + fpOrig + " bytes=" + new BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));
+ //}
+ }
+
+ public int getTermBlockOrd() {
+ return isLeafBlock ? nextEnt : state.termBlockOrd;
+ }
+
+ void loadNextFloorBlock() throws IOException {
+ //if (DEBUG) {
+ //System.out.println(" loadNextFloorBlock fp=" + fp + " fpEnd=" + fpEnd);
+ //}
+ assert arc == null || isFloor: "arc=" + arc + " isFloor=" + isFloor;
+ fp = fpEnd;
+ nextEnt = -1;
+ loadBlock();
+ }
+
+ /* Does initial decode of next block of terms; this
+ doesn't actually decode the docFreq, totalTermFreq,
+ postings details (frq/prx offset, etc.) metadata;
+ it just loads them as byte[] blobs which are then
+ decoded on-demand if the metadata is ever requested
+ for any term in this block. This enables terms-only
+ intensive consumes (eg certain MTQs, respelling) to
+ not pay the price of decoding metadata they won't
+ use. */
+ void loadBlock() throws IOException {
+
+ // Clone the IndexInput lazily, so that consumers
+ // that just pull a TermsEnum to
+ // seekExact(TermState) don't pay this cost:
+ initIndexInput();
+
+ if (nextEnt != -1) {
+ // Already loaded
+ return;
+ }
+ //System.out.println("blc=" + blockLoadCount);
+
+ in.seek(fp);
+ int code = in.readVInt();
+ entCount = code >>> 1;
+ assert entCount > 0;
+ isLastInFloor = (code & 1) != 0;
+ assert arc == null || (isLastInFloor || isFloor);
+
+ // TODO: if suffixes were stored in random-access
+ // array structure, then we could do binary search
+ // instead of linear scan to find target term; eg
+ // we could have simple array of offsets
+
+ // term suffixes:
+ code = in.readVInt();
+ isLeafBlock = (code & 1) != 0;
+ int numBytes = code >>> 1;
+ if (suffixBytes.length < numBytes) {
+ suffixBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ in.readBytes(suffixBytes, 0, numBytes);
+ suffixesReader.reset(suffixBytes, 0, numBytes);
+
+ /*if (DEBUG) {
+ if (arc == null) {
+ System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);
+ } else {
+ System.out.println(" loadBlock (seek) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);
+ }
+ }*/
+
+ // stats
+ numBytes = in.readVInt();
+ if (statBytes.length < numBytes) {
+ statBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ in.readBytes(statBytes, 0, numBytes);
+ statsReader.reset(statBytes, 0, numBytes);
+ metaDataUpto = 0;
+
+ state.termBlockOrd = 0;
+ nextEnt = 0;
+ lastSubFP = -1;
+
+ // TODO: we could skip this if !hasTerms; but
+ // that's rare so won't help much
+ // metadata
+ numBytes = in.readVInt();
+ if (bytes == null) {
+ bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ bytesReader = new ByteArrayDataInput();
+ } else if (bytes.length < numBytes) {
+ bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
+ }
+ in.readBytes(bytes, 0, numBytes);
+ bytesReader.reset(bytes, 0, numBytes);
+
+
+ // Sub-blocks of a single floor block are always
+ // written one after another -- tail recurse:
+ fpEnd = in.getFilePointer();
+ // if (DEBUG) {
+ // System.out.println(" fpEnd=" + fpEnd);
+ // }
+ }
+
+ void rewind() {
+
+ // Force reload:
+ fp = fpOrig;
+ nextEnt = -1;
+ hasTerms = hasTermsOrig;
+ if (isFloor) {
+ floorDataReader.rewind();
+ numFollowFloorBlocks = floorDataReader.readVInt();
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ }
+
+ /*
+ //System.out.println("rewind");
+ // Keeps the block loaded, but rewinds its state:
+ if (nextEnt > 0 || fp != fpOrig) {
+ if (DEBUG) {
+ System.out.println(" rewind frame ord=" + ord + " fpOrig=" + fpOrig + " fp=" + fp + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " nextEnt=" + nextEnt + " prefixLen=" + prefix);
+ }
+ if (fp != fpOrig) {
+ fp = fpOrig;
+ nextEnt = -1;
+ } else {
+ nextEnt = 0;
+ }
+ hasTerms = hasTermsOrig;
+ if (isFloor) {
+ floorDataReader.rewind();
+ numFollowFloorBlocks = floorDataReader.readVInt();
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ }
+ assert suffixBytes != null;
+ suffixesReader.rewind();
+ assert statBytes != null;
+ statsReader.rewind();
+ metaDataUpto = 0;
+ state.termBlockOrd = 0;
+ // TODO: skip this if !hasTerms? Then postings
+ // impl wouldn't have to write useless 0 byte
+ postingsReader.resetTermsBlock(fieldInfo, state);
+ lastSubFP = -1;
+ } else if (DEBUG) {
+ System.out.println(" skip rewind fp=" + fp + " fpOrig=" + fpOrig + " nextEnt=" + nextEnt + " ord=" + ord);
+ }
+ */
+ }
+
+ public boolean next() {
+ return isLeafBlock ? nextLeaf() : nextNonLeaf();
+ }
+
+ // Decodes next entry; returns true if it's a sub-block
+ public boolean nextLeaf() {
+ //if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt + " entCount=" + entCount);
+ assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
+ nextEnt++;
+ suffix = suffixesReader.readVInt();
+ startBytePos = suffixesReader.getPosition();
+ term.length = prefix + suffix;
+ if (term.bytes.length < term.length) {
+ term.grow(term.length);
+ }
+ suffixesReader.readBytes(term.bytes, prefix, suffix);
+ // A normal term
+ termExists = true;
+ return false;
+ }
+
+ public boolean nextNonLeaf() {
+ //if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt + " entCount=" + entCount);
+ assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
+ nextEnt++;
+ final int code = suffixesReader.readVInt();
+ suffix = code >>> 1;
+ startBytePos = suffixesReader.getPosition();
+ term.length = prefix + suffix;
+ if (term.bytes.length < term.length) {
+ term.grow(term.length);
+ }
+ suffixesReader.readBytes(term.bytes, prefix, suffix);
+ if ((code & 1) == 0) {
+ // A normal term
+ termExists = true;
+ subCode = 0;
+ state.termBlockOrd++;
+ return false;
+ } else {
+ // A sub-block; make sub-FP absolute:
+ termExists = false;
+ subCode = suffixesReader.readVLong();
+ lastSubFP = fp - subCode;
+ //if (DEBUG) {
+ //System.out.println(" lastSubFP=" + lastSubFP);
+ //}
+ return true;
+ }
+ }
+
+ // TODO: make this array'd so we can do bin search?
+ // likely not worth it? need to measure how many
+ // floor blocks we "typically" get
+ public void scanToFloorFrame(BytesRef target) {
+
+ if (!isFloor || target.length <= prefix) {
+ // if (DEBUG) {
+ // System.out.println(" scanToFloorFrame skip: isFloor=" + isFloor + " target.length=" + target.length + " vs prefix=" + prefix);
+ // }
+ return;
+ }
+
+ final int targetLabel = target.bytes[target.offset + prefix] & 0xFF;
+
+ // if (DEBUG) {
+ // System.out.println(" scanToFloorFrame fpOrig=" + fpOrig + " targetLabel=" + toHex(targetLabel) + " vs nextFloorLabel=" + toHex(nextFloorLabel) + " numFollowFloorBlocks=" + numFollowFloorBlocks);
+ // }
+
+ if (targetLabel < nextFloorLabel) {
+ // if (DEBUG) {
+ // System.out.println(" already on correct block");
+ // }
+ return;
+ }
+
+ assert numFollowFloorBlocks != 0;
+
+ long newFP = fpOrig;
+ while (true) {
+ final long code = floorDataReader.readVLong();
+ newFP = fpOrig + (code >>> 1);
+ hasTerms = (code & 1) != 0;
+ // if (DEBUG) {
+ // System.out.println(" label=" + toHex(nextFloorLabel) + " fp=" + newFP + " hasTerms?=" + hasTerms + " numFollowFloor=" + numFollowFloorBlocks);
+ // }
+
+ isLastInFloor = numFollowFloorBlocks == 1;
+ numFollowFloorBlocks--;
+
+ if (isLastInFloor) {
+ nextFloorLabel = 256;
+ // if (DEBUG) {
+ // System.out.println(" stop! last block nextFloorLabel=" + toHex(nextFloorLabel));
+ // }
+ break;
+ } else {
+ nextFloorLabel = floorDataReader.readByte() & 0xff;
+ if (targetLabel < nextFloorLabel) {
+ // if (DEBUG) {
+ // System.out.println(" stop! nextFloorLabel=" + toHex(nextFloorLabel));
+ // }
+ break;
+ }
+ }
+ }
+
+ if (newFP != fp) {
+ // Force re-load of the block:
+ // if (DEBUG) {
+ // System.out.println(" force switch to fp=" + newFP + " oldFP=" + fp);
+ // }
+ nextEnt = -1;
+ fp = newFP;
+ } else {
+ // if (DEBUG) {
+ // System.out.println(" stay on same fp=" + newFP);
+ // }
+ }
+ }
+
+ public void decodeMetaData() throws IOException {
+
+ //if (DEBUG) System.out.println("\nBTTR.decodeMetadata seg=" + segment + " mdUpto=" + metaDataUpto + " vs termBlockOrd=" + state.termBlockOrd);
+
+ // lazily catch up on metadata decode:
+ final int limit = getTermBlockOrd();
+ boolean absolute = metaDataUpto == 0;
+ assert limit > 0;
+
+ // TODO: better API would be "jump straight to term=N"???
+ while (metaDataUpto < limit) {
+
+ // TODO: we could make "tiers" of metadata, ie,
+ // decode docFreq/totalTF but don't decode postings
+ // metadata; this way caller could get
+ // docFreq/totalTF w/o paying decode cost for
+ // postings
+
+ // TODO: if docFreq were bulk decoded we could
+ // just skipN here:
+
+ // stats
+ state.docFreq = statsReader.readVInt();
+ //if (DEBUG) System.out.println(" dF=" + state.docFreq);
+ if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
+ state.totalTermFreq = state.docFreq + statsReader.readVLong();
+ //if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq);
+ }
+ // metadata
+ for (int i = 0; i < longsSize; i++) {
+ longs[i] = bytesReader.readVLong();
+ }
+ postingsReader.decodeTerm(longs, bytesReader, fieldInfo, state, absolute);
+
+ metaDataUpto++;
+ absolute = false;
+ }
+ state.termBlockOrd = metaDataUpto;
+ }
+
+ // Used only by assert
+ private boolean prefixMatches(BytesRef target) {
+ for(int bytePos=0;bytePos<prefix;bytePos++) {
+ if (target.bytes[target.offset + bytePos] != term.bytes[bytePos]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ // Scans to sub-block that has this target fp; only
+ // called by next(); NOTE: does not set
+ // startBytePos/suffix as a side effect
+ public void scanToSubBlock(long subFP) {
+ assert !isLeafBlock;
+ //if (DEBUG) System.out.println(" scanToSubBlock fp=" + fp + " subFP=" + subFP + " entCount=" + entCount + " lastSubFP=" + lastSubFP);
+ //assert nextEnt == 0;
+ if (lastSubFP == subFP) {
+ //if (DEBUG) System.out.println(" already positioned");
+ return;
+ }
+ assert subFP < fp : "fp=" + fp + " subFP=" + subFP;
+ final long targetSubCode = fp - subFP;
+ //if (DEBUG) System.out.println(" targetSubCode=" + targetSubCode);
+ while(true) {
+ assert nextEnt < entCount;
+ nextEnt++;
+ final int code = suffixesReader.readVInt();
+ suffixesReader.skipBytes(isLeafBlock ? code : code >>> 1);
+ //if (DEBUG) System.out.println(" " + nextEnt + " (of " + entCount + ") ent isSubBlock=" + ((code&1)==1));
+ if ((code & 1) != 0) {
+ final long subCode = suffixesReader.readVLong();
+ //if (DEBUG) System.out.println(" subCode=" + subCode);
+ if (targetSubCode == subCode) {
+ //if (DEBUG) System.out.println(" match!");
+ lastSubFP = subFP;
+ return;
+ }
+ } else {
+ state.termBlockOrd++;
+ }
+ }
+ }
+
+ // NOTE: sets startBytePos/suffix as a side effect
+ public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException {
+ return isLeafBlock ? scanToTermLeaf(target, exactOnly) : scanToTermNonLeaf(target, exactOnly);
+ }
+
+ private int startBytePos;
+ private int suffix;
+ private long subCode;
+
+ // Target's prefix matches this block's prefix; we
+ // scan the entries check if the suffix matches.
+ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOException {
+
+ // if (DEBUG) System.out.println(" scanToTermLeaf: block fp=" + fp + " prefix=" + prefix + " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + brToString(term));
+
+ assert nextEnt != -1;
+
+ termExists = true;
+ subCode = 0;
+
+ if (nextEnt == entCount) {
+ if (exactOnly) {
+ fillTerm();
+ }
+ return SeekStatus.END;
+ }
+
+ assert prefixMatches(target);
+
+ // Loop over each entry (term or sub-block) in this block:
+ //nextTerm: while(nextEnt < entCount) {
+ nextTerm: while (true) {
+ nextEnt++;
+
+ suffix = suffixesReader.readVInt();
+
+ // if (DEBUG) {
+ // BytesRef suffixBytesRef = new BytesRef();
+ // suffixBytesRef.bytes = suffixBytes;
+ // suffixBytesRef.offset = suffixesReader.getPosition();
+ // suffixBytesRef.length = suffix;
+ // System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix=" + brToString(suffixBytesRef));
+ // }
+
+ final int termLen = prefix + suffix;
+ startBytePos = suffixesReader.getPosition();
+ suffixesReader.skipBytes(suffix);
+
+ final int targetLimit = target.offset + (target.length < termLen ? target.length : termLen);
+ int targetPos = target.offset + prefix;
+
+ // Loop over bytes in the suffix, comparing to
+ // the target
+ int bytePos = startBytePos;
+ while(true) {
+ final int cmp;
+ final boolean stop;
+ if (targetPos < targetLimit) {
+ cmp = (suffixBytes[bytePos++]&0xFF) - (target.bytes[targetPos++]&0xFF);
+ stop = false;
+ } else {
+ assert targetPos == targetLimit;
+ cmp = termLen - target.length;
+ stop = true;
+ }
+
+ if (cmp < 0) {
+ // Current entry is still before the target;
+ // keep scanning
+
+ if (nextEnt == entCount) {
+ if (exactOnly) {
+ fillTerm();
+ }
+ // We are done scanning this block
+ break nextTerm;
+ } else {
+ continue nextTerm;
+ }
+ } else if (cmp > 0) {
+
+ // Done! Current entry is after target --
+ // return NOT_FOUND:
+ fillTerm();
+
+ if (!exactOnly && !termExists) {
+ // We are on a sub-block, and caller wants
+ // us to position to the next term after
+ // the target, so we must recurse into the
+ // sub-frame(s):
+ currentFrame = pushFrame(null, currentFrame.lastSubFP, termLen);
+ currentFrame.loadBlock();
+ while (currentFrame.next()) {
+ currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length);
+ currentFrame.loadBlock();
+ }
+ }
+
+ //if (DEBUG) System.out.println(" not found");
+ return SeekStatus.NOT_FOUND;
+ } else if (stop) {
+ // Exact match!
+
+ // This cannot be a sub-block because we
+ // would have followed the index to this
+ // sub-block from the start:
+
+ assert termExists;
+ fillTerm();
+ //if (DEBUG) System.out.println(" found!");
+ return SeekStatus.FOUND;
+ }
+ }
+ }
+
+ // It is possible (and OK) that terms index pointed us
+ // at this block, but, we scanned the entire block and
+ // did not find the term to position to. This happens
+ // when the target is after the last term in the block
+ // (but, before the next term in the index). EG
+ // target could be foozzz, and terms index pointed us
+ // to the foo* block, but the last term in this block
+ // was fooz (and, eg, first term in the next block will
+ // bee fop).
+ //if (DEBUG) System.out.println(" block end");
+ if (exactOnly) {
+ fillTerm();
+ }
+
+ // TODO: not consistent that in the
+ // not-exact case we don't next() into the next
+ // frame here
+ return SeekStatus.END;
+ }
+
+ // Target's prefix matches this block's prefix; we
+ // scan the entries check if the suffix matches.
+ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException {
+
+ //if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix + " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + brToString(term));
+
+ assert nextEnt != -1;
+
+ if (nextEnt == entCount) {
+ if (exactOnly) {
+ fillTerm();
+ termExists = subCode == 0;
+ }
+ return SeekStatus.END;
+ }
+
+ assert prefixMatches(target);
+
+ // Loop over each entry (term or sub-block) in this block:
+ //nextTerm: while(nextEnt < entCount) {
+ nextTerm: while (true) {
+ nextEnt++;
+
+ final int code = suffixesReader.readVInt();
+ suffix = code >>> 1;
+ // if (DEBUG) {
+ // BytesRef suffixBytesRef = new BytesRef();
+ // suffixBytesRef.bytes = suffixBytes;
+ // suffixBytesRef.offset = suffixesReader.getPosition();
+ // suffixBytesRef.length = suffix;
+ // System.out.println(" cycle: " + ((code&1)==1 ? "sub-block" : "term") + " " + (nextEnt-1) + " (of " + entCount + ") suffix=" + brToString(suffixBytesRef));
+ // }
+
+ termExists = (code & 1) == 0;
+ final int termLen = prefix + suffix;
+ startBytePos = suffixesReader.getPosition();
+ suffixesReader.skipBytes(suffix);
+ if (termExists) {
+ state.termBlockOrd++;
+ subCode = 0;
+ } else {
+ subCode = suffixesReader.readVLong();
+ lastSubFP = fp - subCode;
+ }
+
+ final int targetLimit = target.offset + (target.length < termLen ? target.length : termLen);
+ int targetPos = target.offset + prefix;
+
+ // Loop over bytes in the suffix, comparing to
+ // the target
+ int bytePos = startBytePos;
+ while(true) {
+ final int cmp;
+ final boolean stop;
+ if (targetPos < targetLimit) {
+ cmp = (suffixBytes[bytePos++]&0xFF) - (target.bytes[targetPos++]&0xFF);
+ stop = false;
+ } else {
+ assert targetPos == targetLimit;
+ cmp = termLen - target.length;
+ stop = true;
+ }
+
+ if (cmp < 0) {
+ // Current entry is still before the target;
+ // keep scanning
+
+ if (nextEnt == entCount) {
+ if (exactOnly) {
+ fillTerm();
+ //termExists = true;
+ }
+ // We are done scanning this block
+ break nextTerm;
+ } else {
+ continue nextTerm;
+ }
+ } else if (cmp > 0) {
+
+ // Done! Current entry is after target --
+ // return NOT_FOUND:
+ fillTerm();
+
+ if (!exactOnly && !termExists) {
+ // We are on a sub-block, and caller wants
+ // us to position to the next term after
+ // the target, so we must recurse into the
+ // sub-frame(s):
+ currentFrame = pushFrame(null, currentFrame.lastSubFP, termLen);
+ currentFrame.loadBlock();
+ while (currentFrame.next()) {
+ currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length);
+ currentFrame.loadBlock();
+ }
+ }
+
+ //if (DEBUG) System.out.println(" not found");
+ return SeekStatus.NOT_FOUND;
+ } else if (stop) {
+ // Exact match!
+
+ // This cannot be a sub-block because we
+ // would have followed the index to this
+ // sub-block from the start:
+
+ assert termExists;
+ fillTerm();
+ //if (DEBUG) System.out.println(" found!");
+ return SeekStatus.FOUND;
+ }
+ }
+ }
+
+ // It is possible (and OK) that terms index pointed us
+ // at this block, but, we scanned the entire block and
+ // did not find the term to position to. This happens
+ // when the target is after the last term in the block
+ // (but, before the next term in the index). EG
+ // target could be foozzz, and terms index pointed us
+ // to the foo* block, but the last term in this block
+ // was fooz (and, eg, first term in the next block will
+ // bee fop).
+ //if (DEBUG) System.out.println(" block end");
+ if (exactOnly) {
+ fillTerm();
+ }
+
+ // TODO: not consistent that in the
+ // not-exact case we don't next() into the next
+ // frame here
+ return SeekStatus.END;
+ }
+
+ private void fillTerm() {
+ final int termLength = prefix + suffix;
+ term.length = prefix + suffix;
+ if (term.bytes.length < termLength) {
+ term.grow(termLength);
+ }
+ System.arraycopy(suffixBytes, startBytePos, term.bytes, prefix, suffix);
+ }
+ }
+ }
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ long sizeInByes = ((postingsReader!=null) ? postingsReader.ramBytesUsed() : 0);
+ for(FieldReader reader : fields.values()) {
+ sizeInByes += reader.ramBytesUsed();
+ }
+ return sizeInByes;
+ }
+}
diff --git a/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java b/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
new file mode 100644
index 0000000000..5842ed9396
--- /dev/null
+++ b/oak-lucene/src/main/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
@@ -0,0 +1,1151 @@
+/*
+ * COPIED FROM APACHE LUCENE 4.7.2
+ *
+ * Git URL: git@github.com:apache/lucene.git, tag: releases/lucene-solr/4.7.2, path: lucene/core/src/java
+ *
+ * (see https://issues.apache.org/jira/browse/OAK-10786 for details)
+ */
+
+package org.apache.lucene.codecs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMOutputStream;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.ByteSequenceOutputs;
+import org.apache.lucene.util.fst.BytesRefFSTEnum;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.NoOutputs;
+import org.apache.lucene.util.fst.Util;
+import org.apache.lucene.util.packed.PackedInts;
+
+/*
+ TODO:
+
+ - Currently there is a one-to-one mapping of indexed
+ term to term block, but we could decouple the two, ie,
+ put more terms into the index than there are blocks.
+ The index would take up more RAM but then it'd be able
+ to avoid seeking more often and could make PK/FuzzyQ
+ faster if the additional indexed terms could store
+ the offset into the terms block.
+
+ - The blocks are not written in true depth-first
+ order, meaning if you just next() the file pointer will
+ sometimes jump backwards. For example, block foo* will
+ be written before block f* because it finished before.
+ This could possibly hurt performance if the terms dict is
+ not hot, since OSs anticipate sequential file access. We
+ could fix the writer to re-order the blocks as a 2nd
+ pass.
+
+ - Each block encodes the term suffixes packed
+ sequentially using a separate vInt per term, which is
+ 1) wasteful and 2) slow (must linear scan to find a
+ particular suffix). We should instead 1) make
+ random-access array so we can directly access the Nth
+ suffix, and 2) bulk-encode this array using bulk int[]
+ codecs; then at search time we can binary search when
+ we seek a particular term.
+*/
+
+/**
+ * Block-based terms index and dictionary writer.
+ * <p>
+ * Writes terms dict and index, block-encoding (column
+ * stride) each term's metadata for each set of terms
+ * between two index terms.
+ * <p>
+ * Files:
+ * <ul>
+ * <li><tt>.tim</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
+ * <li><tt>.tip</tt>: <a href="#Termindex">Term Index</a></li>
+ * </ul>
+ * <p>
+ * <a name="Termdictionary" id="Termdictionary"></a>
+ * <h3>Term Dictionary</h3>
+ *
+ * <p>The .tim file contains the list of terms in each
+ * field along with per-term statistics (such as docfreq)
+ * and per-term metadata (typically pointers to the postings list
+ * for that term in the inverted index).
+ * </p>
+ *
+ * <p>The .tim is arranged in blocks: with blocks containing
+ * a variable number of entries (by default 25-48), where
+ * each entry is either a term or a reference to a
+ * sub-block.</p>
+ *
+ * <p>NOTE: The term dictionary can plug into different postings implementations:
+ * the postings writer/reader are actually responsible for encoding
+ * and decoding the Postings Metadata and Term Metadata sections.</p>
+ *
+ * <ul>
+ * <li>TermsDict (.tim) --> Header, <i>PostingsHeader</i>, NodeBlock<sup>NumBlocks</sup>,
+ * FieldSummary, DirOffset</li>
+ * <li>NodeBlock --> (OuterNode | InnerNode)</li>
+ * <li>OuterNode --> EntryCount, SuffixLength, Byte<sup>SuffixLength</sup>, StatsLength, < TermStats ><sup>EntryCount</sup>, MetaLength, <<i>TermMetadata</i>><sup>EntryCount</sup></li>
+ * <li>InnerNode --> EntryCount, SuffixLength[,Sub?], Byte<sup>SuffixLength</sup>, StatsLength, < TermStats ? ><sup>EntryCount</sup>, MetaLength, <<i>TermMetadata ? </i>><sup>EntryCount</sup></li>
+ * <li>TermStats --> DocFreq, TotalTermFreq </li>
+ * <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, RootCodeLength, Byte<sup>RootCodeLength</sup>,
+ * SumTotalTermFreq?, SumDocFreq, DocCount><sup>NumFields</sup></li>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
+ * <li>EntryCount,SuffixLength,StatsLength,DocFreq,MetaLength,NumFields,
+ * FieldNumber,RootCodeLength,DocCount --> {@link DataOutput#writeVInt VInt}</li>
+ * <li>TotalTermFreq,NumTerms,SumTotalTermFreq,SumDocFreq -->
+ * {@link DataOutput#writeVLong VLong}</li>
+ * </ul>
+ * <p>Notes:</p>
+ * <ul>
+ * <li>Header is a {@link CodecUtil#writeHeader CodecHeader} storing the version information
+ * for the BlockTree implementation.</li>
+ * <li>DirOffset is a pointer to the FieldSummary section.</li>
+ * <li>DocFreq is the count of documents which contain the term.</li>
+ * <li>TotalTermFreq is the total number of occurrences of the term. This is encoded
+ * as the difference between the total number of occurrences and the DocFreq.</li>
+ * <li>FieldNumber is the fields number from {@link FieldInfos}. (.fnm)</li>
+ * <li>NumTerms is the number of unique terms for the field.</li>
+ * <li>RootCode points to the root block for the field.</li>
+ * <li>SumDocFreq is the total number of postings, the number of term-document pairs across
+ * the entire field.</li>
+ * <li>DocCount is the number of documents that have at least one posting for this field.</li>
+ * <li>PostingsHeader and TermMetadata are plugged into by the specific postings implementation:
+ * these contain arbitrary per-file data (such as parameters or versioning information)
+ * and per-term data (such as pointers to inverted files).</li>
+ * <li>For inner nodes of the tree, every entry will steal one bit to mark whether it points
+ * to child nodes(sub-block). If so, the corresponding TermStats and TermMetaData are omitted </li>
+ * </ul>
+ * <a name="Termindex" id="Termindex"></a>
+ * <h3>Term Index</h3>
+ * <p>The .tip file contains an index into the term dictionary, so that it can be
+ * accessed randomly. The index is also used to determine
+ * when a given term cannot exist on disk (in the .tim file), saving a disk seek.</p>
+ * <ul>
+ * <li>TermsIndex (.tip) --> Header, FSTIndex<sup>NumFields</sup>
+ * <IndexStartFP><sup>NumFields</sup>, DirOffset</li>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
+ * <li>IndexStartFP --> {@link DataOutput#writeVLong VLong}</li>
+ * <!-- TODO: better describe FST output here -->
+ * <li>FSTIndex --> {@link FST FST<byte[]>}</li>
+ * </ul>
+ * <p>Notes:</p>
+ * <ul>
+ * <li>The .tip file contains a separate FST for each
+ * field. The FST maps a term prefix to the on-disk
+ * block that holds all terms starting with that
+ * prefix. Each field's IndexStartFP points to its
+ * FST.</li>
+ * <li>DirOffset is a pointer to the start of the IndexStartFPs
+ * for all fields</li>
+ * <li>It's possible that an on-disk block would contain
+ * too many terms (more than the allowed maximum
+ * (default: 48)). When this happens, the block is
+ * sub-divided into new blocks (called "floor
+ * blocks"), and then the output in the FST for the
+ * block's prefix encodes the leading byte of each
+ * sub-block, and its file pointer.
+ * </ul>
+ *
+ * @see BlockTreeTermsReader
+ * @lucene.experimental
+ */
+
+public class BlockTreeTermsWriter extends FieldsConsumer {
+
+ /** Suggested default value for the {@code
+ * minItemsInBlock} parameter to {@link
+ * #BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}. */
+ public final static int DEFAULT_MIN_BLOCK_SIZE = 25;
+
+ /** Suggested default value for the {@code
+ * maxItemsInBlock} parameter to {@link
+ * #BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}. */
+ public final static int DEFAULT_MAX_BLOCK_SIZE = 48;
+
+ //public final static boolean DEBUG = false;
+ //private final static boolean SAVE_DOT_FILES = false;
+
+ static final int OUTPUT_FLAGS_NUM_BITS = 2;
+ static final int OUTPUT_FLAGS_MASK = 0x3;
+ static final int OUTPUT_FLAG_IS_FLOOR = 0x1;
+ static final int OUTPUT_FLAG_HAS_TERMS = 0x2;
+
+ /** Extension of terms file */
+ static final String TERMS_EXTENSION = "tim";
+ final static String TERMS_CODEC_NAME = "BLOCK_TREE_TERMS_DICT";
+
+ /** Initial terms format. */
+ public static final int TERMS_VERSION_START = 0;
+
+ /** Append-only */
+ public static final int TERMS_VERSION_APPEND_ONLY = 1;
+
+ /** Meta data as array */
+ public static final int TERMS_VERSION_META_ARRAY = 2;
+
+ /** Current terms format. */
+ public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_META_ARRAY;
+
+ /** Extension of terms index file */
+ static final String TERMS_INDEX_EXTENSION = "tip";
+ final static String TERMS_INDEX_CODEC_NAME = "BLOCK_TREE_TERMS_INDEX";
+
+ /** Initial index format. */
+ public static final int TERMS_INDEX_VERSION_START = 0;
+
+ /** Append-only */
+ public static final int TERMS_INDEX_VERSION_APPEND_ONLY = 1;
+
+ /** Meta data as array */
+ public static final int TERMS_INDEX_VERSION_META_ARRAY = 2;
+
+ /** Current index format. */
+ public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_META_ARRAY;
+
+ private final IndexOutput out;
+ private final IndexOutput indexOut;
+ final int minItemsInBlock;
+ final int maxItemsInBlock;
+
+ final PostingsWriterBase postingsWriter;
+ final FieldInfos fieldInfos;
+ FieldInfo currentField;
+
+ private static class FieldMetaData {
+ public final FieldInfo fieldInfo;
+ public final BytesRef rootCode;
+ public final long numTerms;
+ public final long indexStartFP;
+ public final long sumTotalTermFreq;
+ public final long sumDocFreq;
+ public final int docCount;
+ private final int longsSize;
+
+ public FieldMetaData(FieldInfo fieldInfo, BytesRef rootCode, long numTerms, long indexStartFP, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) {
+ assert numTerms > 0;
+ this.fieldInfo = fieldInfo;
+ assert rootCode != null: "field=" + fieldInfo.name + " numTerms=" + numTerms;
+ this.rootCode = rootCode;
+ this.indexStartFP = indexStartFP;
+ this.numTerms = numTerms;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+ this.longsSize = longsSize;
+ }
+ }
+
+ private final List<FieldMetaData> fields = new ArrayList<FieldMetaData>();
+ // private final String segment;
+
+ /** Create a new writer. The number of items (terms or
+ * sub-blocks) per block will aim to be between
+ * minItemsPerBlock and maxItemsPerBlock, though in some
+ * cases the blocks may be smaller than the min. */
+ public BlockTreeTermsWriter(
+ SegmentWriteState state,
+ PostingsWriterBase postingsWriter,
+ int minItemsInBlock,
+ int maxItemsInBlock)
+ throws IOException
+ {
+ if (minItemsInBlock <= 1) {
+ throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
+ }
+ if (maxItemsInBlock <= 0) {
+ throw new IllegalArgumentException("maxItemsInBlock must be >= 1; got " + maxItemsInBlock);
+ }
+ if (minItemsInBlock > maxItemsInBlock) {
+ throw new IllegalArgumentException("maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
+ }
+ if (2*(minItemsInBlock-1) > maxItemsInBlock) {
+ throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
+ }
+
+ final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION);
+ out = state.directory.createOutput(termsFileName, state.context);
+ boolean success = false;
+ IndexOutput indexOut = null;
+ try {
+ fieldInfos = state.fieldInfos;
+ this.minItemsInBlock = minItemsInBlock;
+ this.maxItemsInBlock = maxItemsInBlock;
+ writeHeader(out);
+
+ //DEBUG = state.segmentName.equals("_4a");
+
+ final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
+ indexOut = state.directory.createOutput(termsIndexFileName, state.context);
+ writeIndexHeader(indexOut);
+
+ currentField = null;
+ this.postingsWriter = postingsWriter;
+ // segment = state.segmentName;
+
+ // System.out.println("BTW.init seg=" + state.segmentName);
+
+ postingsWriter.init(out); // have consumer write its format/header
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(out, indexOut);
+ }
+ }
+ this.indexOut = indexOut;
+ }
+
+ /** Writes the terms file header. */
+ protected void writeHeader(IndexOutput out) throws IOException {
+ CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
+ }
+
+ /** Writes the index file header. */
+ protected void writeIndexHeader(IndexOutput out) throws IOException {
+ CodecUtil.writeHeader(out, TERMS_INDEX_CODEC_NAME, TERMS_INDEX_VERSION_CURRENT);
+ }
+
+ /** Writes the terms file trailer. */
+ protected void writeTrailer(IndexOutput out, long dirStart) throws IOException {
+ out.writeLong(dirStart);
+ }
+
+ /** Writes the index file trailer. */
+ protected void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException {
+ indexOut.writeLong(dirStart);
+ }
+
+ @Override
+ public TermsConsumer addField(FieldInfo field) throws IOException {
+ //DEBUG = field.name.equals("id");
+ //if (DEBUG) System.out.println("\nBTTW.addField seg=" + segment + " field=" + field.name);
+ assert currentField == null || currentField.name.compareTo(field.name) < 0;
+ currentField = field;
+ return new TermsWriter(field);
+ }
+
+ static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
+ assert fp < (1L << 62);
+ return (fp << 2) | (hasTerms ? OUTPUT_FLAG_HAS_TERMS : 0) | (isFloor ? OUTPUT_FLAG_IS_FLOOR : 0);
+ }
+
+ private static class PendingEntry {
+ public final boolean isTerm;
+
+ protected PendingEntry(boolean isTerm) {
+ this.isTerm = isTerm;
+ }
+ }
+
+ private static final class PendingTerm extends PendingEntry {
+ public final BytesRef term;
+ // stats + metadata
+ public final BlockTermState state;
+
+ public PendingTerm(BytesRef term, BlockTermState state) {
+ super(true);
+ this.term = term;
+ this.state = state;
+ }
+
+ @Override
+ public String toString() {
+ return term.utf8ToString();
+ }
+ }
+
+ private static final class PendingBlock extends PendingEntry {
+ public final BytesRef prefix;
+ public final long fp;
+ public FST<BytesRef> index;
+ public List<FST<BytesRef>> subIndices;
+ public final boolean hasTerms;
+ public final boolean isFloor;
+ public final int floorLeadByte;
+ private final IntsRef scratchIntsRef = new IntsRef();
+
+ public PendingBlock(BytesRef prefix, long fp, boolean hasTerms, boolean isFloor, int floorLeadByte, List<FST<BytesRef>> subIndices) {
+ super(false);
+ this.prefix = prefix;
+ this.fp = fp;
+ this.hasTerms = hasTerms;
+ this.isFloor = isFloor;
+ this.floorLeadByte = floorLeadByte;
+ this.subIndices = subIndices;
+ }
+
+ @Override
+ public String toString() {
+ return "BLOCK: " + prefix.utf8ToString();
+ }
+
+ public void compileIndex(List<PendingBlock> floorBlocks, RAMOutputStream scratchBytes) throws IOException {
+
+ assert (isFloor && floorBlocks != null && floorBlocks.size() != 0) || (!isFloor && floorBlocks == null): "isFloor=" + isFloor + " floorBlocks=" + floorBlocks;
+
+ assert scratchBytes.getFilePointer() == 0;
+
+ // TODO: try writing the leading vLong in MSB order
+ // (opposite of what Lucene does today), for better
+ // outputs sharing in the FST
+ scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
+ if (isFloor) {
+ scratchBytes.writeVInt(floorBlocks.size());
+ for (PendingBlock sub : floorBlocks) {
+ assert sub.floorLeadByte != -1;
+ //if (DEBUG) {
+ // System.out.println(" write floorLeadByte=" + Integer.toHexString(sub.floorLeadByte&0xff));
+ //}
+ scratchBytes.writeByte((byte) sub.floorLeadByte);
+ assert sub.fp > fp;
+ scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0));
+ }
+ }
+
+ final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+ final Builder<BytesRef> indexBuilder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1,
+ 0, 0, true, false, Integer.MAX_VALUE,
+ outputs, null, false,
+ PackedInts.COMPACT, true, 15);
+ //if (DEBUG) {
+ // System.out.println(" compile index for prefix=" + prefix);
+ //}
+ //indexBuilder.DEBUG = false;
+ final byte[] bytes = new byte[(int) scratchBytes.getFilePointer()];
+ assert bytes.length > 0;
+ scratchBytes.writeTo(bytes, 0);
+ indexBuilder.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));
+ scratchBytes.reset();
+
+ // Copy over index for all sub-blocks
+
+ if (subIndices != null) {
+ for(FST<BytesRef> subIndex : subIndices) {
+ append(indexBuilder, subIndex);
+ }
+ }
+
+ if (floorBlocks != null) {
+ for (PendingBlock sub : floorBlocks) {
+ if (sub.subIndices != null) {
+ for(FST<BytesRef> subIndex : sub.subIndices) {
+ append(indexBuilder, subIndex);
+ }
+ }
+ sub.subIndices = null;
+ }
+ }
+
+ index = indexBuilder.finish();
+ subIndices = null;
+
+ /*
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
+ Util.toDot(index, w, false, false);
+ System.out.println("SAVED to out.dot");
+ w.close();
+ */
+ }
+
+ // TODO: maybe we could add bulk-add method to
+ // Builder? Takes FST and unions it w/ current
+ // FST.
+ private void append(Builder<BytesRef> builder, FST<BytesRef> subIndex) throws IOException {
+ final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<BytesRef>(subIndex);
+ BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
+ while((indexEnt = subIndexEnum.next()) != null) {
+ //if (DEBUG) {
+ // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
+ //}
+ builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
+ }
+ }
+ }
+
+ final RAMOutputStream scratchBytes = new RAMOutputStream();
+
+ class TermsWriter extends TermsConsumer {
+ private final FieldInfo fieldInfo;
+ private final int longsSize;
+ private long numTerms;
+ long sumTotalTermFreq;
+ long sumDocFreq;
+ int docCount;
+ long indexStartFP;
+
+ // Used only to partition terms into the block tree; we
+ // don't pull an FST from this builder:
+ private final NoOutputs noOutputs;
+ private final Builder<Object> blockBuilder;
+
+ // PendingTerm or PendingBlock:
+ private final List<PendingEntry> pending = new ArrayList<PendingEntry>();
+
+ // Index into pending of most recently written block
+ private int lastBlockIndex = -1;
+
+ // Re-used when segmenting a too-large block into floor
+ // blocks:
+ private int[] subBytes = new int[10];
+ private int[] subTermCounts = new int[10];
+ private int[] subTermCountSums = new int[10];
+ private int[] subSubCounts = new int[10];
+
+ // This class assigns terms to blocks "naturally", ie,
+ // according to the number of terms under a given prefix
+ // that we encounter:
+ private class FindBlocks extends Builder.FreezeTail<Object> {
+
+ @Override
+ public void freeze(final Builder.UnCompiledNode<Object>[] frontier, int prefixLenPlus1, final IntsRef lastInput) throws IOException {
+
+ //if (DEBUG) System.out.println(" freeze prefixLenPlus1=" + prefixLenPlus1);
+
+ for(int idx=lastInput.length; idx >= prefixLenPlus1; idx--) {
+ final Builder.UnCompiledNode<Object> node = frontier[idx];
+
+ long totCount = 0;
+
+ if (node.isFinal) {
+ totCount++;
+ }
+
+ for(int arcIdx=0;arcIdx<node.numArcs;arcIdx++) {
+ @SuppressWarnings("unchecked") final Builder.UnCompiledNode<Object> target = (Builder.UnCompiledNode<Object>) node.arcs[arcIdx].target;
+ totCount += target.inputCount;
+ target.clear();
+ node.arcs[arcIdx].target = null;
+ }
+ node.numArcs = 0;
+
+ if (totCount >= minItemsInBlock || idx == 0) {
+ // We are on a prefix node that has enough
+ // entries (terms or sub-blocks) under it to let
+ // us write a new block or multiple blocks (main
+ // block + follow on floor blocks):
+ //if (DEBUG) {
+ // if (totCount < minItemsInBlock && idx != 0) {
+ // System.out.println(" force block has terms");
+ // }
+ //}
+ writeBlocks(lastInput, idx, (int) totCount);
+ node.inputCount = 1;
+ } else {
+ // stragglers! carry count upwards
+ node.inputCount = totCount;
+ }
+ frontier[idx] = new Builder.UnCompiledNode<Object>(blockBuilder, idx);
+ }
+ }
+ }
+
+ // Write the top count entries on the pending stack as
+ // one or more blocks. Returns how many blocks were
+ // written. If the entry count is <= maxItemsPerBlock
+ // we just write a single block; else we break into
+ // primary (initial) block and then one or more
+ // following floor blocks:
+
+ void writeBlocks(IntsRef prevTerm, int prefixLength, int count) throws IOException {
+ if (prefixLength == 0 || count <= maxItemsInBlock) {
+ // Easy case: not floor block. Eg, prefix is "foo",
+ // and we found 30 terms/sub-blocks starting w/ that
+ // prefix, and minItemsInBlock <= 30 <=
+ // maxItemsInBlock.
+ final PendingBlock nonFloorBlock = writeBlock(prevTerm, prefixLength, prefixLength, count, count, 0, false, -1, true);
+ nonFloorBlock.compileIndex(null, scratchBytes);
+ pending.add(nonFloorBlock);
+ } else {
+ // Floor block case. Eg, prefix is "foo" but we
+ // have 100 terms/sub-blocks starting w/ that
+ // prefix. We segment the entries into a primary
+ // block and following floor blocks using the first
+ // label in the suffix to assign to floor blocks.
+
+ // TODO: we could store min & max suffix start byte
+ // in each block, to make floor blocks authoritative
+
+ //if (DEBUG) {
+ // final BytesRef prefix = new BytesRef(prefixLength);
+ // for(int m=0;m<prefixLength;m++) {
+ // prefix.bytes[m] = (byte) prevTerm.ints[m];
+ // }
+ // prefix.length = prefixLength;
+ // //System.out.println("\nWBS count=" + count + " prefix=" + prefix.utf8ToString() + " " + prefix);
+ // System.out.println("writeBlocks: prefix=" + prefix + " " + prefix + " count=" + count + " pending.size()=" + pending.size());
+ //}
+ //System.out.println("\nwbs count=" + count);
+
+ final int savLabel = prevTerm.ints[prevTerm.offset + prefixLength];
+
+ // Count up how many items fall under
+ // each unique label after the prefix.
+
+ // TODO: this is wasteful since the builder had
+ // already done this (partitioned these sub-terms
+ // according to their leading prefix byte)
+
+ final List<PendingEntry> slice = pending.subList(pending.size()-count, pending.size());
+ int lastSuffixLeadLabel = -1;
+ int termCount = 0;
+ int subCount = 0;
+ int numSubs = 0;
+
+ for(PendingEntry ent : slice) {
+
+ // First byte in the suffix of this term
+ final int suffixLeadLabel;
+ if (ent.isTerm) {
+ PendingTerm term = (PendingTerm) ent;
+ if (term.term.length == prefixLength) {
+ // Suffix is 0, ie prefix 'foo' and term is
+ // 'foo' so the term has empty string suffix
+ // in this block
+ assert lastSuffixLeadLabel == -1;
+ assert numSubs == 0;
+ suffixLeadLabel = -1;
+ } else {
+ suffixLeadLabel = term.term.bytes[term.term.offset + prefixLength] & 0xff;
+ }
+ } else {
+ PendingBlock block = (PendingBlock) ent;
+ assert block.prefix.length > prefixLength;
+ suffixLeadLabel = block.prefix.bytes[block.prefix.offset + prefixLength] & 0xff;
+ }
+
+ if (suffixLeadLabel != lastSuffixLeadLabel && (termCount + subCount) != 0) {
+ if (subBytes.length == numSubs) {
+ subBytes = ArrayUtil.grow(subBytes);
+ subTermCounts = ArrayUtil.grow(subTermCounts);
+ subSubCounts = ArrayUtil.grow(subSubCounts);
+ }
+ subBytes[numSubs] = lastSuffixLeadLabel;
+ lastSuffixLeadLabel = suffixLeadLabel;
+ subTermCounts[numSubs] = termCount;
+ subSubCounts[numSubs] = subCount;
+ /*
+ if (suffixLeadLabel == -1) {
+ System.out.println(" sub " + -1 + " termCount=" + termCount + " subCount=" + subCount);
+ } else {
+ System.out.println(" sub " + Integer.toHexString(suffixLeadLabel) + " termCount=" + termCount + " subCount=" + subCount);
+ }
+ */
+ termCount = subCount = 0;
+ numSubs++;
+ }
+
+ if (ent.isTerm) {
+ termCount++;
+ } else {
+ subCount++;
+ }
+ }
+
+ if (subBytes.length == numSubs) {
+ subBytes = ArrayUtil.grow(subBytes);
+ subTermCounts = ArrayUtil.grow(subTermCounts);
+ subSubCounts = ArrayUtil.grow(subSubCounts);
+ }
+
+ subBytes[numSubs] = lastSuffixLeadLabel;
+ subTermCounts[numSubs] = termCount;
+ subSubCounts[numSubs] = subCount;
+ numSubs++;
+ /*
+ if (lastSuffixLeadLabel == -1) {
+ System.out.println(" sub " + -1 + " termCount=" + termCount + " subCount=" + subCount);
+ } else {
+ System.out.println(" sub " + Integer.toHexString(lastSuffixLeadLabel) + " termCount=" + termCount + " subCount=" + subCount);
+ }
+ */
+
+ if (subTermCountSums.length < numSubs) {
+ subTermCountSums = ArrayUtil.grow(subTermCountSums, numSubs);
+ }
+
+ // Roll up (backwards) the termCounts; postings impl
+ // needs this to know where to pull the term slice
+ // from its pending terms stack:
+ int sum = 0;
+ for(int idx=numSubs-1;idx>=0;idx--) {
+ sum += subTermCounts[idx];
+ subTermCountSums[idx] = sum;
+ }
+
+ // TODO: make a better segmenter? It'd have to
+ // absorb the too-small end blocks backwards into
+ // the previous blocks
+
+ // Naive greedy segmentation; this is not always
+ // best (it can produce a too-small block as the
+ // last block):
+ int pendingCount = 0;
+ int startLabel = subBytes[0];
+ int curStart = count;
+ subCount = 0;
+
+ final List<PendingBlock> floorBlocks = new ArrayList<PendingBlock>();
+ PendingBlock firstBlock = null;
+
+ for(int sub=0;sub<numSubs;sub++) {
+ pendingCount += subTermCounts[sub] + subSubCounts[sub];
+ //System.out.println(" " + (subTermCounts[sub] + subSubCounts[sub]));
+ subCount++;
+
+ // Greedily make a floor block as soon as we've
+ // crossed the min count
+ if (pendingCount >= minItemsInBlock) {
+ final int curPrefixLength;
+ if (startLabel == -1) {
+ curPrefixLength = prefixLength;
+ } else {
+ curPrefixLength = 1+prefixLength;
+ // floor term:
+ prevTerm.ints[prevTerm.offset + prefixLength] = startLabel;
+ }
+ //System.out.println(" " + subCount + " subs");
+ final PendingBlock floorBlock = writeBlock(prevTerm, prefixLength, curPrefixLength, curStart, pendingCount, subTermCountSums[1+sub], true, startLabel, curStart == pendingCount);
+ if (firstBlock == null) {
+ firstBlock = floorBlock;
+ } else {
+ floorBlocks.add(floorBlock);
+ }
+ curStart -= pendingCount;
+ //System.out.println(" = " + pendingCount);
+ pendingCount = 0;
+
+ assert minItemsInBlock == 1 || subCount > 1: "minItemsInBlock=" + minItemsInBlock + " subCount=" + subCount + " sub=" + sub + " of " + numSubs + " subTermCount=" + subTermCountSums[sub] + " subSubCount=" + subSubCounts[sub] + " depth=" + prefixLength;
+ subCount = 0;
+ startLabel = subBytes[sub+1];
+
+ if (curStart == 0) {
+ break;
+ }
+
+ if (curStart <= maxItemsInBlock) {
+ // remainder is small enough to fit into a
+ // block. NOTE that this may be too small (<
+ // minItemsInBlock); need a true segmenter
+ // here
+ assert startLabel != -1;
+ assert firstBlock != null;
+ prevTerm.ints[prevTerm.offset + prefixLength] = startLabel;
+ //System.out.println(" final " + (numSubs-sub-1) + " subs");
+ /*
+ for(sub++;sub < numSubs;sub++) {
+ System.out.println(" " + (subTermCounts[sub] + subSubCounts[sub]));
+ }
+ System.out.println(" = " + curStart);
+ if (curStart < minItemsInBlock) {
+ System.out.println(" **");
+ }
+ */
+ floorBlocks.add(writeBlock(prevTerm, prefixLength, prefixLength+1, curStart, curStart, 0, true, startLabel, true));
+ break;
+ }
+ }
+ }
+
+ prevTerm.ints[prevTerm.offset + prefixLength] = savLabel;
+
+ assert firstBlock != null;
+ firstBlock.compileIndex(floorBlocks, scratchBytes);
+
+ pending.add(firstBlock);
+ //if (DEBUG) System.out.println(" done pending.size()=" + pending.size());
+ }
+ lastBlockIndex = pending.size()-1;
+ }
+
+ // for debugging
+ @SuppressWarnings("unused")
+ private String toString(BytesRef b) {
+ try {
+ return b.utf8ToString() + " " + b;
+ } catch (Throwable t) {
+ // If BytesRef isn't actually UTF8, or it's eg a
+ // prefix of UTF8 that ends mid-unicode-char, we
+ // fallback to hex:
+ return b.toString();
+ }
+ }
+
+ // Writes all entries in the pending slice as a single
+ // block:
+ private PendingBlock writeBlock(IntsRef prevTerm, int prefixLength, int indexPrefixLength, int startBackwards, int length,
+ int futureTermCount, boolean isFloor, int floorLeadByte, boolean isLastInFloor) throws IOException {
+
+ assert length > 0;
+
+ final int start = pending.size()-startBackwards;
+
+ assert start >= 0: "pending.size()=" + pending.size() + " startBackwards=" + startBackwards + " length=" + length;
+
+ final List<PendingEntry> slice = pending.subList(start, start + length);
+
+ final long startFP = out.getFilePointer();
+
+ final BytesRef prefix = new BytesRef(indexPrefixLength);
+ for(int m=0;m<indexPrefixLength;m++) {
+ prefix.bytes[m] = (byte) prevTerm.ints[m];
+ }
+ prefix.length = indexPrefixLength;
+
+ // Write block header:
+ out.writeVInt((length<<1)|(isLastInFloor ? 1:0));
+
+ // if (DEBUG) {
+ // System.out.println(" writeBlock " + (isFloor ? "(floor) " : "") + "seg=" + segment + " pending.size()=" + pending.size() + " prefixLength=" + prefixLength + " indexPrefix=" + toString(prefix) + " entCount=" + length + " startFP=" + startFP + " futureTermCount=" + futureTermCount + (isFloor ? (" floorLeadByte=" + Integer.toHexString(floorLeadByte&0xff)) : "") + " isLastInFloor=" + isLastInFloor);
+ // }
+
+ // 1st pass: pack term suffix bytes into byte[] blob
+ // TODO: cutover to bulk int codec... simple64?
+
+ final boolean isLeafBlock;
+ if (lastBlockIndex < start) {
+ // This block definitely does not contain sub-blocks:
+ isLeafBlock = true;
+ //System.out.println("no scan true isFloor=" + isFloor);
+ } else if (!isFloor) {
+ // This block definitely does contain at least one sub-block:
+ isLeafBlock = false;
+ //System.out.println("no scan false " + lastBlockIndex + " vs start=" + start + " len=" + length);
+ } else {
+ // Must scan up-front to see if there is a sub-block
+ boolean v = true;
+ //System.out.println("scan " + lastBlockIndex + " vs start=" + start + " len=" + length);
+ for (PendingEntry ent : slice) {
+ if (!ent.isTerm) {
+ v = false;
+ break;
+ }
... 164394 lines suppressed ...