You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by do...@apache.org on 2009/07/29 20:04:24 UTC
svn commit: r798995 [1/35] - in /incubator/lucene.net/trunk/C#/src:
Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/
Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/
Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net/...
Author: dougsale
Date: Wed Jul 29 18:04:12 2009
New Revision: 798995
URL: http://svn.apache.org/viewvc?rev=798995&view=rev
Log:
Initial Lucene 2.4.0 conversion
Added:
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/BufferedDeletes.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteBlockPool.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteSliceReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteSliceWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CharBlockPool.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumer.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumerPerThread.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumer.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerField.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumers.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessor.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerField.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverter.cs (with props)
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriterThreadState.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxFieldMergeState.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IntBlockPool.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocConsumer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocConsumerPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocConsumerPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocEndConsumer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocEndConsumerPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocEndConsumerPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergeDocIDRemapper.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/NormsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/NormsWriterPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/NormsWriterPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/RawPostingList.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReadOnlyMultiSegmentReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReadOnlySegmentReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReusableStringReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldWriterPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriterPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriterPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriterPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHash.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashConsumer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashConsumerPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashConsumerPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashPerField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashPerThread.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.4.0-VS2005.csproj
- copied, changed from r797580, incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.csproj
incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.4.0-VS2005.sln
- copied, changed from r797580, incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.sln
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DocIdSet.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DocIdSetIterator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Payload/PayloadSpanUtil.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/PayloadSpans.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/CheckSumIndexInput.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/CheckSumIndexOutput.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/ArrayUtil.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/BitUtil.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/Cache.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/SimpleLRUCache.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/SimpleMapCache.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/CloseableThreadLocal.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/DocIdBitSet.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSet.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSetDISI.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSetIterator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/SortedVIntList.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/UnicodeUtil.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestByteSlices.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestCloseableThreadLocal.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestCrash.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestTransactions.cs
incubator/lucene.net/trunk/C#/src/Test/Index/index.19.cfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.19.nocfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.20.cfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.20.nocfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.21.cfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.21.nocfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.22.cfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.22.nocfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.23.cfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Index/index.23.nocfs.zip (with props)
incubator/lucene.net/trunk/C#/src/Test/Search/Payloads/PayloadHelper.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestPayloadSpans.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestFilteredSearch.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSloppyPhraseQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Store/TestDirectory.cs
incubator/lucene.net/trunk/C#/src/Test/TestMergeSchedulerExternal.cs
incubator/lucene.net/trunk/C#/src/Test/Util/Cache/
incubator/lucene.net/trunk/C#/src/Test/Util/Cache/TestSimpleLRUCache.cs
incubator/lucene.net/trunk/C#/src/Test/Util/TestOpenBitSet.cs
incubator/lucene.net/trunk/C#/src/Test/Util/TestSortedVIntList.cs
Removed:
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.jj
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/TokenMgrError.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.csproj
incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.sln
incubator/lucene.net/trunk/C#/src/Test/Index/index.prelockless.cfs.zip
incubator/lucene.net/trunk/C#/src/Test/Index/index.prelockless.nocfs.zip
incubator/lucene.net/trunk/C#/src/Test/Index/index.presharedstores.cfs.zip
incubator/lucene.net/trunk/C#/src/Test/Index/index.presharedstores.nocfs.zip
incubator/lucene.net/trunk/C#/src/Test/TestWeakHashTable.cs
Modified:
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/StopFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/TeeTokenFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Token.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/TokenFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/TokenStream.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Tokenizer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/AssemblyInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/AbstractField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/DateTools.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/Document.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/Field.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/FieldSelectorResult.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/Fieldable.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/MapFieldSelector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfos.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldSortedTermVectorMapper.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNameFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNames.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexModifier.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/KeepOnlyLastCommitDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiLevelSkipListReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiSegmentReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Payload.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMergeQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermDocs.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermEnum.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermPositions.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermVector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SortedTermVectorMapper.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Term.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermBuffer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntry.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntryFreqSortedComparator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorOffsetInfo.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsWriter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net.sln
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/CharStream.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/FastCharStream.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/MultiFieldQueryParser.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/ParseException.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParser.JJ
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParser.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParserConstants.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParserTokenManager.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/Token.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/TokenMgrError.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/BooleanClause.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/BooleanQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/CachingSpanFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/CachingWrapperFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ComplexExplanation.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ConjunctionScorer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ConstantScoreQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ConstantScoreRangeQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DisjunctionMaxQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DisjunctionMaxScorer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ExtendedFieldCacheImpl.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldCache.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldCacheImpl.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldDoc.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldDocSortedHitQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldSortedHitQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Filter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FilterManager.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FilteredQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ByteFieldSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/CustomScoreQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/DocValues.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/FieldCacheSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/FieldScoreQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/FloatFieldSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/IntFieldSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/OrdFieldSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ReverseOrdFieldSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ShortFieldSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ValueSource.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ValueSourceQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FuzzyQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Hit.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/HitIterator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/HitQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Hits.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/IndexSearcher.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/MatchAllDocsQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/MultiPhraseQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/MultiTermQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ParallelMultiSearcher.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Payload/BoostingTermQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PhraseQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PhraseQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PrefixFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PrefixQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Query.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/QueryFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/QueryTermVector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/QueryWrapperFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/RangeFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/RangeQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/RemoteCachingWrapperFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ScoreDocComparator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Scorer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Searchable.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Searcher.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SimilarityDelegator.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SloppyPhraseScorer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Sort.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SortField.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SpanFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SpanFilterResult.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SpanQueryFilter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/NearSpansOrdered.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/NearSpansUnordered.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanFirstQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanNearQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanNotQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanOrQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanScorer.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanTermQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/TermSpans.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TermQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TopDocCollector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TopFieldDocCollector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TopFieldDocs.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/WildcardQuery.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/SharpZipLibAdapter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/BufferedIndexInput.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/Directory.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/FSDirectory.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/IndexInput.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/IndexOutput.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/Lock.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/LockStressTest.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/MMapDirectory.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMDirectory.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMFile.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMInputStream.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMOutputStream.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/SupportClass.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/BitVector.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Parameter.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/PriorityQueue.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/SmallFloat.cs
incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/StringHelper.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TeeSinkTokenTest.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCachingTokenFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestISOLatin1AccentFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestKeywordAnalyzer.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestLengthFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestPerFieldAnalzyerWrapper.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStandardAnalyzer.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopAnalyzer.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Analysis/TestToken.cs
incubator/lucene.net/trunk/C#/src/Test/AnalysisTest.cs
incubator/lucene.net/trunk/C#/src/Test/App.config
incubator/lucene.net/trunk/C#/src/Test/AssemblyInfo.cs
incubator/lucene.net/trunk/C#/src/Test/Document/TestBinaryDocument.cs
incubator/lucene.net/trunk/C#/src/Test/Document/TestDocument.cs
incubator/lucene.net/trunk/C#/src/Test/Index/DocHelper.cs
incubator/lucene.net/trunk/C#/src/Test/Index/MockIndexInput.cs
incubator/lucene.net/trunk/C#/src/Test/Index/Store/TestRAMDirectory.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestAddIndexesNoOptimize.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestAtomicUpdate.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestBackwardsCompatibility.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestCheckIndex.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestCompoundFile.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestConcurrentMergeScheduler.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestDoc.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestDocumentWriter.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldInfos.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldsReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestFilterIndexReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexFileDeleter.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexInput.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexModifier.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexReaderReopen.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentMerger.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermDocs.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermEnum.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing2.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestTermVectorsReader.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestTermdocPerf.cs
incubator/lucene.net/trunk/C#/src/Test/Index/TestThreadedOptimize.cs
incubator/lucene.net/trunk/C#/src/Test/IndexTest.cs
incubator/lucene.net/trunk/C#/src/Test/QueryParser/TestMultiAnalyzer.cs
incubator/lucene.net/trunk/C#/src/Test/QueryParser/TestMultiFieldQueryParser.cs
incubator/lucene.net/trunk/C#/src/Test/QueryParser/TestQueryParser.cs
incubator/lucene.net/trunk/C#/src/Test/Search/BaseTestRangeFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/CachingWrapperFilterHelper.cs
incubator/lucene.net/trunk/C#/src/Test/Search/CheckHits.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Function/FunctionTestSetup.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Function/TestCustomScoreQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Function/TestFieldScoreQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Function/TestOrdValues.cs
incubator/lucene.net/trunk/C#/src/Test/Search/MockFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Payloads/TestBoostingTermQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/QueryUtils.cs
incubator/lucene.net/trunk/C#/src/Test/Search/RemoteCachingWrapperFilterHelper.cs
incubator/lucene.net/trunk/C#/src/Test/Search/SingleDocTestFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestBasics.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestNearSpansOrdered.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestSpans.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestSpansAdvanced.cs
incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestSpansAdvanced2.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestBoolean2.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanMinShouldMatch.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanOr.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanPrefixQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanScorer.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestCachingWrapperFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestConstantScoreRangeQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestCustomSearcherSort.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestDateFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestDateSort.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestDisjunctionMaxQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestDocBoost.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestExplanations.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestExtendedFieldCache.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestFilteredQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestFuzzyQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestMatchAllDocsQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiPhraseQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiSearcher.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiSearcherRanking.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiThreadTermVectors.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestNot.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestPhrasePrefixQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestPhraseQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestPositionIncrement.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestPrefixFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestPrefixQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestRangeFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestRangeQuery.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestRemoteCachingWrapperFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestRemoteSearchable.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestScorerPerf.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSearchHitsWithDeletions.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSetNorm.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSimilarity.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSimpleExplanations.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSort.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestSpanQueryFilter.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestTermScorer.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestTermVectors.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestThreadSafe.cs
incubator/lucene.net/trunk/C#/src/Test/Search/TestWildcard.cs
incubator/lucene.net/trunk/C#/src/Test/SearchTest.cs
incubator/lucene.net/trunk/C#/src/Test/SearchTestForDuplicates.cs
incubator/lucene.net/trunk/C#/src/Test/Store/MockRAMDirectory.cs
incubator/lucene.net/trunk/C#/src/Test/Store/MockRAMInputStream.cs
incubator/lucene.net/trunk/C#/src/Test/Store/MockRAMOutputStream.cs
incubator/lucene.net/trunk/C#/src/Test/Store/TestBufferedIndexInput.cs
incubator/lucene.net/trunk/C#/src/Test/Store/TestHugeRamFile.cs
incubator/lucene.net/trunk/C#/src/Test/Store/TestLockFactory.cs
incubator/lucene.net/trunk/C#/src/Test/Store/TestWindowsMMap.cs
incubator/lucene.net/trunk/C#/src/Test/Test-VS2005.csproj
incubator/lucene.net/trunk/C#/src/Test/Test.csproj
incubator/lucene.net/trunk/C#/src/Test/Test.sln
incubator/lucene.net/trunk/C#/src/Test/TestDemo.cs
incubator/lucene.net/trunk/C#/src/Test/TestHitIterator.cs
incubator/lucene.net/trunk/C#/src/Test/TestSearch.cs
incubator/lucene.net/trunk/C#/src/Test/TestSearchForDuplicates.cs
incubator/lucene.net/trunk/C#/src/Test/TestSnapshotDeletionPolicy.cs
incubator/lucene.net/trunk/C#/src/Test/ThreadSafetyTest.cs
incubator/lucene.net/trunk/C#/src/Test/Util/LuceneTestCase.cs
incubator/lucene.net/trunk/C#/src/Test/Util/TestSmallFloat.cs
incubator/lucene.net/trunk/C#/src/Test/Util/_TestUtil.cs
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Analyzer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs Wed Jul 29 18:04:12 2009
@@ -26,17 +26,11 @@
/// Typical implementations first build a Tokenizer, which breaks the stream of
/// characters from the Reader into raw Tokens. One or more TokenFilters may
/// then be applied to the output of the Tokenizer.
- /// <p>
- /// WARNING: You must override one of the methods defined by this class in your
- /// subclass or the Analyzer will enter an infinite loop.
/// </summary>
public abstract class Analyzer
{
/// <summary>Creates a TokenStream which tokenizes all the text in the provided
- /// Reader. Default implementation forwards to tokenStream(Reader) for
- /// compatibility with older version. Override to allow Analyzer to choose
- /// strategy based on document and/or field. Must be able to handle null
- /// field name for backward compatibility.
+ /// Reader. Must be able to handle null field name for backward compatibility.
/// </summary>
public abstract TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader);
@@ -58,7 +52,7 @@
/// to retrieve previously saved TokenStreams for re-use
/// by the same thread.
/// </summary>
- protected internal virtual System.Object GetPreviousTokenStream()
+ protected internal virtual object GetPreviousTokenStream()
{
return System.Threading.Thread.GetData(tokenStreams);
}
@@ -67,7 +61,7 @@
/// to save a TokenStream for later re-use by the same
/// thread.
/// </summary>
- protected internal virtual void SetPreviousTokenStream(System.Object obj)
+ protected internal virtual void SetPreviousTokenStream(object obj)
{
System.Threading.Thread.SetData(tokenStreams, obj);
}
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/CachingTokenFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs Wed Jul 29 18:04:12 2009
@@ -38,13 +38,14 @@
{
}
- public override Token Next()
+ public override Token Next(/* in */ Token reusableToken)
{
+ System.Diagnostics.Debug.Assert(reusableToken != null);
if (cache == null)
{
// fill cache lazily
cache = new System.Collections.ArrayList();
- FillCache();
+ FillCache(reusableToken);
iterator = cache.GetEnumerator();
}
@@ -54,7 +55,8 @@
return null;
}
- return (Token) iterator.Current;
+ Token nextToken = (Token) iterator.Current;
+ return (Token) nextToken.Clone();
}
public override void Reset()
@@ -65,13 +67,10 @@
}
}
- private void FillCache()
+ private void FillCache(/* in */ Token reusableToken)
{
- Token token;
- while ((token = input.Next()) != null)
- {
- cache.Add(token);
- }
+ for (Token nextToken = input.Next(reusableToken); nextToken != null; nextToken = input.Next(reusableToken))
+ cache.Add(nextToken.Clone());
}
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/CharArraySet.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs Wed Jul 29 18:04:12 2009
@@ -194,7 +194,7 @@
return count == 0;
}
- public override bool Contains(System.Object o)
+ public override bool Contains(object o)
{
if (o is char[])
{
@@ -209,7 +209,7 @@
return false;
}
- public virtual bool Add(System.Object o)
+ public virtual bool Add(object o)
{
if (o is char[])
{
@@ -238,7 +238,7 @@
/// <summary>Returns the next String, as a Set<String> would...
/// use nextCharArray() for better efficiency.
/// </summary>
- public virtual System.Object Current
+ public virtual object Current
{
get
{
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/CharTokenizer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs Wed Jul 29 18:04:12 2009
@@ -48,19 +48,20 @@
return c;
}
- public override Token Next(Token token)
+ public override Token Next(/* in */ Token reusableToken)
{
- token.Clear();
+ System.Diagnostics.Debug.Assert(reusableToken != null);
+ reusableToken.Clear();
int length = 0;
int start = bufferIndex;
- char[] buffer = token.TermBuffer();
+ char[] buffer = reusableToken.TermBuffer();
while (true)
{
if (bufferIndex >= dataLen)
{
offset += dataLen;
- dataLen = input is Lucene.Net.Index.DocumentsWriter.ReusableStringReader ? ((Lucene.Net.Index.DocumentsWriter.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
+ dataLen = input is Lucene.Net.Index.ReusableStringReader ? ((Lucene.Net.Index.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
if (dataLen <= 0)
{
if (length > 0)
@@ -81,7 +82,7 @@
// start of token
start = offset + bufferIndex - 1;
else if (length == buffer.Length)
- buffer = token.ResizeTermBuffer(1 + length);
+ buffer = reusableToken.ResizeTermBuffer(1 + length);
buffer[length++] = Normalize(c); // buffer it, normalized
@@ -94,10 +95,10 @@
break; // return 'em
}
- token.termLength = length;
- token.startOffset = start;
- token.endOffset = start + length;
- return token;
+ reusableToken.SetTermLength(length);
+ reusableToken.SetStartOffset(start);
+ reusableToken.SetEndOffset(start + length);
+ return reusableToken;
}
public override void Reset(System.IO.TextReader input)
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs Wed Jul 29 18:04:12 2009
@@ -35,26 +35,27 @@
private char[] output = new char[256];
private int outputPos;
- public override Token Next(Token result)
+ public override Token Next(/* in */ Token reusableToken)
{
- result = input.Next(result);
- if (result != null)
+ System.Diagnostics.Debug.Assert(reusableToken != null);
+ Token nextToken = input.Next(reusableToken);
+ if (nextToken != null)
{
- char[] buffer = result.TermBuffer();
- int length = result.TermLength();
+ char[] buffer = nextToken.TermBuffer();
+ int length = nextToken.TermLength();
// If no characters actually require rewriting then we
// just return token as-is:
for (int i = 0; i < length; i++)
{
char c = buffer[i];
- if (c >= '\u00c0' && c <= '\u0178')
+ if (c >= '\u00c0' && c <= '\ufb06')
{
RemoveAccents(buffer, length);
- result.SetTermBuffer(output, 0, outputPos);
+ nextToken.SetTermBuffer(output, 0, outputPos);
break;
}
}
- return result;
+ return nextToken;
}
else
return null;
@@ -84,7 +85,7 @@
// Quick test: if it's not in range then just keep
// current character
- if (c < '\u00c0')
+ if (c < '\u00c0' || c > '\ufb06')
output[outputPos++] = c;
else
{
@@ -134,6 +135,11 @@
output[outputPos++] = 'I';
break;
+ case '\u0132': // IJ
+ output[outputPos++] = 'I';
+ output[outputPos++] = 'J';
+ break;
+
case '\u00D0': // ÃÂ
output[outputPos++] = 'D';
break;
@@ -225,6 +231,11 @@
output[outputPos++] = 'i';
break;
+ case '\u0133': // ij
+ output[outputPos++] = 'i';
+ output[outputPos++] = 'j';
+ break;
+
case '\u00F0': // ð
output[outputPos++] = 'd';
break;
@@ -278,6 +289,38 @@
output[outputPos++] = 'y';
break;
+ case '\uFB00': // ï¬
+ output[outputPos++] = 'f';
+ output[outputPos++] = 'f';
+ break;
+ case '\uFB01': // ï¬
+ output[outputPos++] = 'f';
+ output[outputPos++] = 'i';
+ break;
+ case '\uFB02': // ï¬
+ output[outputPos++] = 'f';
+ output[outputPos++] = 'l';
+ break;
+ // following 2 are commented as they can break the maxSizeNeeded (and doing *3 could be expensive)
+ // case '\uFB03': // ï¬
+ // output[outputPos++] = 'f';
+ // output[outputPos++] = 'f';
+ // output[outputPos++] = 'i';
+ // break;
+ // case '\uFB04': // ï¬
+ // output[outputPos++] = 'f';
+ // output[outputPos++] = 'f';
+ // output[outputPos++] = 'l';
+ // break;
+ case '\uFB05': // ï¬
+ output[outputPos++] = 'f';
+ output[outputPos++] = 't';
+ break;
+ case '\uFB06': // ï¬
+ output[outputPos++] = 's';
+ output[outputPos++] = 't';
+ break;
+
default:
output[outputPos++] = c;
break;
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/KeywordTokenizer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs Wed Jul 29 18:04:12 2009
@@ -37,14 +37,15 @@
this.done = false;
}
- public override Token Next(Token result)
+ public override Token Next(/* in */ Token reusableToken)
{
+ System.Diagnostics.Debug.Assert(reusableToken != null);
if (!done)
{
done = true;
int upto = 0;
- result.Clear();
- char[] buffer = result.TermBuffer();
+ reusableToken.Clear();
+ char[] buffer = reusableToken.TermBuffer();
while (true)
{
int length = input.Read(buffer, upto, buffer.Length - upto);
@@ -52,10 +53,10 @@
break;
upto += length;
if (upto == buffer.Length)
- buffer = result.ResizeTermBuffer(1 + buffer.Length);
+ buffer = reusableToken.ResizeTermBuffer(1 + buffer.Length);
}
- result.termLength = upto;
- return result;
+ reusableToken.SetTermLength(upto);
+ return reusableToken;
}
return null;
}
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/LengthFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs Wed Jul 29 18:04:12 2009
@@ -41,16 +41,17 @@
this.max = max;
}
- /// <summary> Returns the next input Token whose termText() is the right len</summary>
- public override Token Next(Token result)
+ /// <summary> Returns the next input Token whose term() is the right len</summary>
+ public override Token Next(/* in */ Token reusableToken)
{
+ System.Diagnostics.Debug.Assert(reusableToken != null);
// return the first non-stop word found
- for (Token token = input.Next(result); token != null; token = input.Next(result))
+ for (Token nextToken = input.Next(reusableToken); nextToken != null; nextToken = input.Next(reusableToken))
{
- int len = token.TermText().Length;
+ int len = nextToken.TermLength();
if (len >= min && len <= max)
{
- return token;
+ return nextToken;
}
// note: else we ignore it but should we index each part of it?
}
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/LowerCaseFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs Wed Jul 29 18:04:12 2009
@@ -31,18 +31,18 @@
{
}
- public override Token Next(Token result)
+ public override Token Next(Token reusableToken)
{
- result = input.Next(result);
- if (result != null)
+ Token nextToken = input.Next(reusableToken);
+ if (nextToken != null)
{
- char[] buffer = result.TermBuffer();
- int length = result.termLength;
+ char[] buffer = nextToken.TermBuffer();
+ int length = nextToken.TermLength();
for (int i = 0; i < length; i++)
buffer[i] = System.Char.ToLower(buffer[i]);
- return result;
+ return nextToken;
}
else
return null;
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Package.html?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html Wed Jul 29 18:04:12 2009
@@ -1,256 +1,272 @@
-<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
-<html>
-<head>
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
- <meta name="Author" content="Doug Cutting">
-</head>
-<body>
-<p>API and code to convert text into indexable/searchable tokens. Covers {@link org.apache.lucene.analysis.Analyzer} and related classes.</p>
-<h2>Parsing? Tokenization? Analysis!</h2>
-<p>
-Lucene, indexing and search library, accepts only plain text input.
-<p>
-<h2>Parsing</h2>
-<p>
-Applications that build their search capabilities upon Lucene may support documents in various formats – HTML, XML, PDF, Word – just to name a few.
-Lucene does not care about the <i>Parsing</i> of these and other document formats, and it is the responsibility of the
-application using Lucene to use an appropriate <i>Parser</i> to convert the original format into plain text before passing that plain text to Lucene.
-<p>
-<h2>Tokenization</h2>
-<p>
-Plain text passed to Lucene for indexing goes through a process generally called tokenization – namely breaking of the
-input text into small indexing elements –
-{@link org.apache.lucene.analysis.Token Tokens}.
-The way input text is broken into tokens very
-much dictates further capabilities of search upon that text.
-For instance, sentences beginnings and endings can be identified to provide for more accurate phrase
-and proximity searches (though sentence identification is not provided by Lucene).
-<p>
-In some cases simply breaking the input text into tokens is not enough – a deeper <i>Analysis</i> is needed,
-providing for several functions, including (but not limited to):
-<ul>
- <li><a href = "http://en.wikipedia.org//wiki/Stemming">Stemming</a> –
- Replacing of words by their stems.
- For instance with English stemming "bikes" is replaced by "bike";
- now query "bike" can find both documents containing "bike" and those containing "bikes".
- </li>
- <li><a href = "http://en.wikipedia.org//wiki/Stop_words">Stop Words Filtering</a> –
- Common words like "the", "and" and "a" rarely add any value to a search.
- Removing them shrinks the index size and increases performance.
- It may also reduce some "noise" and actually improve search quality.
- </li>
- <li><a href = "http://en.wikipedia.org//wiki/Text_normalization">Text Normalization</a> –
- Stripping accents and other character markings can make for better searching.
- </li>
- <li><a href = "http://en.wikipedia.org//wiki/Synonym">Synonym Expansion</a> –
- Adding in synonyms at the same token position as the current word can mean better
- matching when users search with words in the synonym set.
- </li>
-</ul>
-<p>
-<h2>Core Analysis</h2>
-<p>
- The analysis package provides the mechanism to convert Strings and Readers into tokens that can be indexed by Lucene. There
- are three main classes in the package from which all analysis processes are derived. These are:
- <ul>
- <li>{@link org.apache.lucene.analysis.Analyzer} – An Analyzer is responsible for building a {@link org.apache.lucene.analysis.TokenStream} which can be consumed
- by the indexing and searching processes. See below for more information on implementing your own Analyzer.</li>
- <li>{@link org.apache.lucene.analysis.Tokenizer} – A Tokenizer is a {@link org.apache.lucene.analysis.TokenStream} and is responsible for breaking
- up incoming text into {@link org.apache.lucene.analysis.Token}s. In most cases, an Analyzer will use a Tokenizer as the first step in
- the analysis process.</li>
- <li>{@link org.apache.lucene.analysis.TokenFilter} – A TokenFilter is also a {@link org.apache.lucene.analysis.TokenStream} and is responsible
- for modifying {@link org.apache.lucene.analysis.Token}s that have been created by the Tokenizer. Common modifications performed by a
- TokenFilter are: deletion, stemming, synonym injection, and down casing. Not all Analyzers require TokenFilters</li>
- </ul>
-</p>
-<h2>Hints, Tips and Traps</h2>
-<p>
- The synergy between {@link org.apache.lucene.analysis.Analyzer} and {@link org.apache.lucene.analysis.Tokenizer}
- is sometimes confusing. To ease on this confusion, some clarifications:
- <ul>
- <li>The {@link org.apache.lucene.analysis.Analyzer} is responsible for the entire task of
- <u>creating</u> tokens out of the input text, while the {@link org.apache.lucene.analysis.Tokenizer}
- is only responsible for <u>breaking</u> the input text into tokens. Very likely, tokens created
- by the {@link org.apache.lucene.analysis.Tokenizer} would be modified or even omitted
- by the {@link org.apache.lucene.analysis.Analyzer} (via one or more
- {@link org.apache.lucene.analysis.TokenFilter}s) before being returned.
- </li>
- <li>{@link org.apache.lucene.analysis.Tokenizer} is a {@link org.apache.lucene.analysis.TokenStream},
- but {@link org.apache.lucene.analysis.Analyzer} is not.
- </li>
- <li>{@link org.apache.lucene.analysis.Analyzer} is "field aware", but
- {@link org.apache.lucene.analysis.Tokenizer} is not.
- </li>
- </ul>
-</p>
-<p>
- Lucene Java provides a number of analysis capabilities, the most commonly used one being the {@link
- org.apache.lucene.analysis.standard.StandardAnalyzer}. Many applications will have a long and industrious life with nothing more
- than the StandardAnalyzer. However, there are a few other classes/packages that are worth mentioning:
- <ol>
- <li>{@link org.apache.lucene.analysis.PerFieldAnalyzerWrapper} – Most Analyzers perform the same operation on all
- {@link org.apache.lucene.document.Field}s. The PerFieldAnalyzerWrapper can be used to associate a different Analyzer with different
- {@link org.apache.lucene.document.Field}s.</li>
- <li>The contrib/analyzers library located at the root of the Lucene distribution has a number of different Analyzer implementations to solve a variety
- of different problems related to searching. Many of the Analyzers are designed to analyze non-English languages.</li>
- <li>The {@link org.apache.lucene.analysis.snowball contrib/snowball library}
- located at the root of the Lucene distribution has Analyzer and TokenFilter
- implementations for a variety of Snowball stemmers.
- See <a href = "http://snowball.tartarus.org">http://snowball.tartarus.org</a>
- for more information on Snowball stemmers.</li>
- <li>There are a variety of Tokenizer and TokenFilter implementations in this package. Take a look around, chances are someone has implemented what you need.</li>
- </ol>
-</p>
-<p>
- Analysis is one of the main causes of performance degradation during indexing. Simply put, the more you analyze the slower the indexing (in most cases).
- Perhaps your application would be just fine using the simple {@link org.apache.lucene.analysis.WhitespaceTokenizer} combined with a
- {@link org.apache.lucene.analysis.StopFilter}. The contrib/benchmark library can be useful for testing out the speed of the analysis process.
-</p>
-<h2>Invoking the Analyzer</h2>
-<p>
- Applications usually do not invoke analysis – Lucene does it for them:
- <ul>
- <li>At indexing, as a consequence of
- {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document) addDocument(doc)},
- the Analyzer in effect for indexing is invoked for each indexed field of the added document.
- </li>
- <li>At search, as a consequence of
- {@link org.apache.lucene.queryParser.QueryParser#parse(java.lang.String) QueryParser.parse(queryText)},
- the QueryParser may invoke the Analyzer in effect.
- Note that for some queries analysis does not take place, e.g. wildcard queries.
- </li>
- </ul>
- However an application might invoke Analysis of any text for testing or for any other purpose, something like:
- <PRE>
- Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
- TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
- Token t = ts.next();
- while (t!=null) {
- System.out.println("token: "+t));
- t = ts.next();
- }
- </PRE>
-</p>
-<h2>Indexing Analysis vs. Search Analysis</h2>
-<p>
- Selecting the "correct" analyzer is crucial
- for search quality, and can also affect indexing and search performance.
- The "correct" analyzer differs between applications.
- Lucene java's wiki page
- <a href = "http://wiki.apache.org//lucene-java/AnalysisParalysis">AnalysisParalysis</a>
- provides some data on "analyzing your analyzer".
- Here are some rules of thumb:
- <ol>
- <li>Test test test... (did we say test?)</li>
- <li>Beware of over analysis – might hurt indexing performance.</li>
- <li>Start with same analyzer for indexing and search, otherwise searches would not find what they are supposed to...</li>
- <li>In some cases a different analyzer is required for indexing and search, for instance:
- <ul>
- <li>Certain searches require more stop words to be filtered. (I.e. more than those that were filtered at indexing.)</li>
- <li>Query expansion by synonyms, acronyms, auto spell correction, etc.</li>
- </ul>
- This might sometimes require a modified analyzer – see the next section on how to do that.
- </li>
- </ol>
-</p>
-<h2>Implementing your own Analyzer</h2>
-<p>Creating your own Analyzer is straightforward. It usually involves either wrapping an existing Tokenizer and set of TokenFilters to create a new Analyzer
-or creating both the Analyzer and a Tokenizer or TokenFilter. Before pursuing this approach, you may find it worthwhile
-to explore the contrib/analyzers library and/or ask on the java-user@lucene.apache.org mailing list first to see if what you need already exists.
-If you are still committed to creating your own Analyzer or TokenStream derivation (Tokenizer or TokenFilter) have a look at
-the source code of any one of the many samples located in this package.
-</p>
-<p>
- The following sections discuss some aspects of implementing your own analyzer.
-</p>
-<h3>Field Section Boundaries</h2>
-<p>
- When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)}
- is called multiple times for the same field name, we could say that each such call creates a new
- section for that field in that document.
- In fact, a separate call to
- {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)}
- would take place for each of these so called "sections".
- However, the default Analyzer behavior is to treat all these sections as one large section.
- This allows phrase search and proximity search to seamlessly cross
- boundaries between these "sections".
- In other words, if a certain field "f" is added like this:
- <PRE>
- document.add(new Field("f","first ends",...);
- document.add(new Field("f","starts two",...);
- indexWriter.addDocument(document);
- </PRE>
- Then, a phrase search for "ends starts" would find that document.
- Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections",
- simply by overriding
- {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
- <PRE>
- Analyzer myAnalyzer = new StandardAnalyzer() {
- public int getPositionIncrementGap(String fieldName) {
- return 10;
- }
- };
- </PRE>
-</p>
-<h3>Token Position Increments</h2>
-<p>
- By default, all tokens created by Analyzers and Tokenizers have a
- {@link org.apache.lucene.analysis.Token#getPositionIncrement() position increment} of one.
- This means that the position stored for that token in the index would be one more than
- that of the previous token.
- Recall that phrase and proximity searches rely on position info.
-</p>
-<p>
- If the selected analyzer filters the stop words "is" and "the", then for a document
- containing the string "blue is the sky", only the tokens "blue", "sky" are indexed,
- with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky"
- would find that document, because the same analyzer filters the same stop words from
- that query. But also the phrase query "blue sky" would find that document.
-</p>
-<p>
- If this behavior does not fit the application needs,
- a modified analyzer can be used, that would increment further the positions of
- tokens following a removed stop word, using
- {@link org.apache.lucene.analysis.Token#setPositionIncrement(int)}.
- This can be done with something like:
- <PRE>
- public TokenStream tokenStream(final String fieldName, Reader reader) {
- final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
- TokenStream res = new TokenStream() {
- public Token next() throws IOException {
- int extraIncrement = 0;
- while (true) {
- Token t = ts.next();
- if (t!=null) {
- if (stopWords.contains(t.termText())) {
- extraIncrement++; // filter this word
- continue;
- }
- if (extraIncrement>0) {
- t.setPositionIncrement(t.getPositionIncrement()+extraIncrement);
- }
- }
- return t;
- }
- }
- };
- return res;
- }
- </PRE>
- Now, with this modified analyzer, the phrase query "blue sky" would find that document.
- But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
- where both w1 and w2 are stop words would match that document.
-</p>
-<p>
- Few more use cases for modifying position increments are:
- <ol>
- <li>Inhibiting phrase and proximity matches in sentence boundaries – for this, a tokenizer that
- identifies a new sentence can add 1 to the position increment of the first token of the new sentence.</li>
- <li>Injecting synonyms – here, synonyms of a token should be added after that token,
- and their position increment should be set to 0.
- As result, all synonyms of a token would be considered to appear in exactly the
- same position as that token, and so would they be seen by phrase and proximity searches.</li>
- </ol>
-</p>
-</body>
-</html>
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+ <meta name="Author" content="Doug Cutting">
+</head>
+<body>
+<p>API and code to convert text into indexable/searchable tokens. Covers {@link org.apache.lucene.analysis.Analyzer} and related classes.</p>
+<h2>Parsing? Tokenization? Analysis!</h2>
+<p>
+Lucene, indexing and search library, accepts only plain text input.
+<p>
+<h2>Parsing</h2>
+<p>
+Applications that build their search capabilities upon Lucene may support documents in various formats – HTML, XML, PDF, Word – just to name a few.
+Lucene does not care about the <i>Parsing</i> of these and other document formats, and it is the responsibility of the
+application using Lucene to use an appropriate <i>Parser</i> to convert the original format into plain text before passing that plain text to Lucene.
+<p>
+<h2>Tokenization</h2>
+<p>
+Plain text passed to Lucene for indexing goes through a process generally called tokenization – namely breaking of the
+input text into small indexing elements –
+{@link org.apache.lucene.analysis.Token Tokens}.
+The way input text is broken into tokens very
+much dictates further capabilities of search upon that text.
+For instance, sentences beginnings and endings can be identified to provide for more accurate phrase
+and proximity searches (though sentence identification is not provided by Lucene).
+<p>
+In some cases simply breaking the input text into tokens is not enough – a deeper <i>Analysis</i> is needed,
+providing for several functions, including (but not limited to):
+<ul>
+ <li><a href="http://en.wikipedia.org/wiki/Stemming">Stemming</a> –
+ Replacing of words by their stems.
+ For instance with English stemming "bikes" is replaced by "bike";
+ now query "bike" can find both documents containing "bike" and those containing "bikes".
+ </li>
+ <li><a href="http://en.wikipedia.org/wiki/Stop_words">Stop Words Filtering</a> –
+ Common words like "the", "and" and "a" rarely add any value to a search.
+ Removing them shrinks the index size and increases performance.
+ It may also reduce some "noise" and actually improve search quality.
+ </li>
+ <li><a href="http://en.wikipedia.org/wiki/Text_normalization">Text Normalization</a> –
+ Stripping accents and other character markings can make for better searching.
+ </li>
+ <li><a href="http://en.wikipedia.org/wiki/Synonym">Synonym Expansion</a> –
+ Adding in synonyms at the same token position as the current word can mean better
+ matching when users search with words in the synonym set.
+ </li>
+</ul>
+<p>
+<h2>Core Analysis</h2>
+<p>
+ The analysis package provides the mechanism to convert Strings and Readers into tokens that can be indexed by Lucene. There
+ are three main classes in the package from which all analysis processes are derived. These are:
+ <ul>
+ <li>{@link org.apache.lucene.analysis.Analyzer} – An Analyzer is responsible for building a {@link org.apache.lucene.analysis.TokenStream} which can be consumed
+ by the indexing and searching processes. See below for more information on implementing your own Analyzer.</li>
+ <li>{@link org.apache.lucene.analysis.Tokenizer} – A Tokenizer is a {@link org.apache.lucene.analysis.TokenStream} and is responsible for breaking
+ up incoming text into {@link org.apache.lucene.analysis.Token}s. In most cases, an Analyzer will use a Tokenizer as the first step in
+ the analysis process.</li>
+ <li>{@link org.apache.lucene.analysis.TokenFilter} – A TokenFilter is also a {@link org.apache.lucene.analysis.TokenStream} and is responsible
+ for modifying {@link org.apache.lucene.analysis.Token}s that have been created by the Tokenizer. Common modifications performed by a
+ TokenFilter are: deletion, stemming, synonym injection, and down casing. Not all Analyzers require TokenFilters</li>
+ </ul>
+</p>
+<h2>Hints, Tips and Traps</h2>
+<p>
+ The synergy between {@link org.apache.lucene.analysis.Analyzer} and {@link org.apache.lucene.analysis.Tokenizer}
+ is sometimes confusing. To ease on this confusion, some clarifications:
+ <ul>
+ <li>The {@link org.apache.lucene.analysis.Analyzer} is responsible for the entire task of
+ <u>creating</u> tokens out of the input text, while the {@link org.apache.lucene.analysis.Tokenizer}
+ is only responsible for <u>breaking</u> the input text into tokens. Very likely, tokens created
+ by the {@link org.apache.lucene.analysis.Tokenizer} would be modified or even omitted
+ by the {@link org.apache.lucene.analysis.Analyzer} (via one or more
+ {@link org.apache.lucene.analysis.TokenFilter}s) before being returned.
+ </li>
+ <li>{@link org.apache.lucene.analysis.Tokenizer} is a {@link org.apache.lucene.analysis.TokenStream},
+ but {@link org.apache.lucene.analysis.Analyzer} is not.
+ </li>
+ <li>{@link org.apache.lucene.analysis.Analyzer} is "field aware", but
+ {@link org.apache.lucene.analysis.Tokenizer} is not.
+ </li>
+ </ul>
+</p>
+<p>
+ Lucene Java provides a number of analysis capabilities, the most commonly used one being the {@link
+ org.apache.lucene.analysis.standard.StandardAnalyzer}. Many applications will have a long and industrious life with nothing more
+ than the StandardAnalyzer. However, there are a few other classes/packages that are worth mentioning:
+ <ol>
+ <li>{@link org.apache.lucene.analysis.PerFieldAnalyzerWrapper} – Most Analyzers perform the same operation on all
+ {@link org.apache.lucene.document.Field}s. The PerFieldAnalyzerWrapper can be used to associate a different Analyzer with different
+ {@link org.apache.lucene.document.Field}s.</li>
+ <li>The contrib/analyzers library located at the root of the Lucene distribution has a number of different Analyzer implementations to solve a variety
+ of different problems related to searching. Many of the Analyzers are designed to analyze non-English languages.</li>
+ <li>The contrib/snowball library
+ located at the root of the Lucene distribution has Analyzer and TokenFilter
+ implementations for a variety of Snowball stemmers.
+ See <a href="http://snowball.tartarus.org">http://snowball.tartarus.org</a>
+ for more information on Snowball stemmers.</li>
+ <li>There are a variety of Tokenizer and TokenFilter implementations in this package. Take a look around, chances are someone has implemented what you need.</li>
+ </ol>
+</p>
+<p>
+ Analysis is one of the main causes of performance degradation during indexing. Simply put, the more you analyze the slower the indexing (in most cases).
+ Perhaps your application would be just fine using the simple {@link org.apache.lucene.analysis.WhitespaceTokenizer} combined with a
+ {@link org.apache.lucene.analysis.StopFilter}. The contrib/benchmark library can be useful for testing out the speed of the analysis process.
+</p>
+<h2>Invoking the Analyzer</h2>
+<p>
+ Applications usually do not invoke analysis – Lucene does it for them:
+ <ul>
+ <li>At indexing, as a consequence of
+ {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document) addDocument(doc)},
+ the Analyzer in effect for indexing is invoked for each indexed field of the added document.
+ </li>
+ <li>At search, as a consequence of
+ {@link org.apache.lucene.queryParser.QueryParser#parse(java.lang.String) QueryParser.parse(queryText)},
+ the QueryParser may invoke the Analyzer in effect.
+ Note that for some queries analysis does not take place, e.g. wildcard queries.
+ </li>
+ </ul>
+ However an application might invoke Analysis of any text for testing or for any other purpose, something like:
+ <PRE>
+ Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
+ TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
+ Token t = ts.next();
+ while (t!=null) {
+ System.out.println("token: "+t));
+ t = ts.next();
+ }
+ </PRE>
+</p>
+<h2>Indexing Analysis vs. Search Analysis</h2>
+<p>
+ Selecting the "correct" analyzer is crucial
+ for search quality, and can also affect indexing and search performance.
+ The "correct" analyzer differs between applications.
+ Lucene java's wiki page
+ <a href="http://wiki.apache.org/lucene-java/AnalysisParalysis">AnalysisParalysis</a>
+ provides some data on "analyzing your analyzer".
+ Here are some rules of thumb:
+ <ol>
+ <li>Test test test... (did we say test?)</li>
+ <li>Beware of over analysis – might hurt indexing performance.</li>
+ <li>Start with same analyzer for indexing and search, otherwise searches would not find what they are supposed to...</li>
+ <li>In some cases a different analyzer is required for indexing and search, for instance:
+ <ul>
+ <li>Certain searches require more stop words to be filtered. (I.e. more than those that were filtered at indexing.)</li>
+ <li>Query expansion by synonyms, acronyms, auto spell correction, etc.</li>
+ </ul>
+ This might sometimes require a modified analyzer – see the next section on how to do that.
+ </li>
+ </ol>
+</p>
+<h2>Implementing your own Analyzer</h2>
+<p>Creating your own Analyzer is straightforward. It usually involves either wrapping an existing Tokenizer and set of TokenFilters to create a new Analyzer
+or creating both the Analyzer and a Tokenizer or TokenFilter. Before pursuing this approach, you may find it worthwhile
+to explore the contrib/analyzers library and/or ask on the java-user@lucene.apache.org mailing list first to see if what you need already exists.
+If you are still committed to creating your own Analyzer or TokenStream derivation (Tokenizer or TokenFilter) have a look at
+the source code of any one of the many samples located in this package.
+</p>
+<p>
+ The following sections discuss some aspects of implementing your own analyzer.
+</p>
+<h3>Field Section Boundaries</h2>
+<p>
+ When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)}
+ is called multiple times for the same field name, we could say that each such call creates a new
+ section for that field in that document.
+ In fact, a separate call to
+ {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)}
+ would take place for each of these so called "sections".
+ However, the default Analyzer behavior is to treat all these sections as one large section.
+ This allows phrase search and proximity search to seamlessly cross
+ boundaries between these "sections".
+ In other words, if a certain field "f" is added like this:
+ <PRE>
+ document.add(new Field("f","first ends",...);
+ document.add(new Field("f","starts two",...);
+ indexWriter.addDocument(document);
+ </PRE>
+ Then, a phrase search for "ends starts" would find that document.
+ Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections",
+ simply by overriding
+ {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
+ <PRE>
+ Analyzer myAnalyzer = new StandardAnalyzer() {
+ public int getPositionIncrementGap(String fieldName) {
+ return 10;
+ }
+ };
+ </PRE>
+</p>
+<h3>Token Position Increments</h2>
+<p>
+ By default, all tokens created by Analyzers and Tokenizers have a
+ {@link org.apache.lucene.analysis.Token#getPositionIncrement() position increment} of one.
+ This means that the position stored for that token in the index would be one more than
+ that of the previous token.
+ Recall that phrase and proximity searches rely on position info.
+</p>
+<p>
+ If the selected analyzer filters the stop words "is" and "the", then for a document
+ containing the string "blue is the sky", only the tokens "blue", "sky" are indexed,
+ with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky"
+ would find that document, because the same analyzer filters the same stop words from
+ that query. But also the phrase query "blue sky" would find that document.
+</p>
+<p>
+ If this behavior does not fit the application needs,
+ a modified analyzer can be used, that would increment further the positions of
+ tokens following a removed stop word, using
+ {@link org.apache.lucene.analysis.Token#setPositionIncrement(int)}.
+ This can be done with something like:
+ <PRE>
+ public TokenStream tokenStream(final String fieldName, Reader reader) {
+ final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
+ TokenStream res = new TokenStream() {
+ public Token next() throws IOException {
+ int extraIncrement = 0;
+ while (true) {
+ Token t = ts.next();
+ if (t!=null) {
+ if (stopWords.contains(t.termText())) {
+ extraIncrement++; // filter this word
+ continue;
+ }
+ if (extraIncrement>0) {
+ t.setPositionIncrement(t.getPositionIncrement()+extraIncrement);
+ }
+ }
+ return t;
+ }
+ }
+ };
+ return res;
+ }
+ </PRE>
+ Now, with this modified analyzer, the phrase query "blue sky" would find that document.
+ But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
+ where both w1 and w2 are stop words would match that document.
+</p>
+<p>
+ Few more use cases for modifying position increments are:
+ <ol>
+ <li>Inhibiting phrase and proximity matches in sentence boundaries – for this, a tokenizer that
+ identifies a new sentence can add 1 to the position increment of the first token of the new sentence.</li>
+ <li>Injecting synonyms – here, synonyms of a token should be added after that token,
+ and their position increment should be set to 0.
+ As result, all synonyms of a token would be considered to appear in exactly the
+ same position as that token, and so would they be seen by phrase and proximity searches.</li>
+ </ol>
+</p>
+</body>
+</html>
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/PorterStemFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs Wed Jul 29 18:04:12 2009
@@ -47,17 +47,17 @@
stemmer = new PorterStemmer();
}
- public override Token Next(Token result)
+ public override Token Next(/* in */ Token reusableToken)
{
- result = input.Next(result);
- if (result != null)
- {
- if (stemmer.Stem(result.TermBuffer(), 0, result.termLength))
- result.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
- return result;
- }
- else
+ System.Diagnostics.Debug.Assert(reusableToken != null);
+ Token nextToken = input.Next(reusableToken);
+ if (nextToken == null)
return null;
+
+ if (stemmer.Stem(nextToken.TermBuffer(), 0, nextToken.TermLength()))
+ nextToken.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
+
+ return nextToken;
}
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/SinkTokenizer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs Wed Jul 29 18:04:12 2009
@@ -55,7 +55,8 @@
/// WARNING: Adding tokens to this list requires the {@link #Reset()} method to be called in order for them
/// to be made available. Also, this Tokenizer does nothing to protect against {@link java.util.ConcurrentModificationException}s
/// in the case of adds happening while {@link #Next(Lucene.Net.Analysis.Token)} is being called.
- ///
+ /// <p/>
+ /// WARNING: Since this SinkTokenizer can be reset and the cached tokens made available again, do not modify them. Modify clones instead.
/// </summary>
/// <returns> A List of {@link Lucene.Net.Analysis.Token}s
/// </returns>
@@ -68,11 +69,17 @@
/// <returns> The next {@link Lucene.Net.Analysis.Token} in the Sink.
/// </returns>
/// <throws> IOException </throws>
- public override Token Next()
+ public override Token Next(/* in */ Token reusableToken)
{
+ System.Diagnostics.Debug.Assert(reusableToken != null);
if (iter == null)
iter = lst.GetEnumerator();
- return iter.MoveNext() ? (Token) iter.Current : null;
+ if (iter.MoveNext())
+ {
+ Token nextToken = (Token) iter.Current;
+ return (Token) nextToken.Clone();
+ }
+ return null;
}
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs Wed Jul 29 18:04:12 2009
@@ -30,6 +30,50 @@
/// </version>
public class StandardAnalyzer : Analyzer
{
+ /// <summary>An array containing some common English words that are usually not
+ /// useful for searching.
+ /// </summary>
+ public static readonly System.String[] STOP_WORDS;
+
+ private static bool defaultReplaceInvalidAcronym;
+
+ static StandardAnalyzer()
+ {
+ STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
+
+ // Default to true (fixed the bug), unless the system property is set
+ string v = SupportClass.AppSettings.Get("Lucene.Net.Analysis.Standard.StandardAnalyzer.replaceInvalidAcronym", "true");
+ if (v.Equals("true"))
+ defaultReplaceInvalidAcronym = true;
+ else
+ defaultReplaceInvalidAcronym = false;
+ }
+
+ /// <summary>
+ /// see https://issues.apache.org/jira/browse/LUCENE-1068
+ /// </summary>
+ /// <returns>true if new instances of StandardTokenizer will replace mischaracterized acronyms</returns>
+ /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+ public static bool GetDefaultReplaceInvalidAcronym()
+ {
+ return defaultReplaceInvalidAcronym;
+ }
+
+ /// <summary>
+ /// set to true to have new instances of StandardTokenizer
+ /// replace mischaracerized acronyms by default. Set to
+ /// false to preserve the previous (before 2.4) buggy behavior.
+ /// Alternatively, set the system property
+ /// Lucene.Net.Analysis.Standard.StandardAnalyzer.replaceInvalidAcronym
+ /// to false.
+ /// </summary>
+ /// <param name="replaceInvalidAcronym"/>
+ /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+ public static void SetDefaultReplaceInvalidAcronym(bool replaceInvalidAcronym)
+ {
+ defaultReplaceInvalidAcronym = replaceInvalidAcronym;
+ }
+
private System.Collections.Hashtable stopSet;
/// <summary> Specifies whether deprecated acronyms should be replaced with HOST type.
@@ -40,13 +84,8 @@
///
/// See https://issues.apache.org/jira/browse/LUCENE-1068
/// </deprecated>
- private bool replaceInvalidAcronym = false;
-
- /// <summary>An array containing some common English words that are usually not
- /// useful for searching.
- /// </summary>
- public static readonly System.String[] STOP_WORDS;
-
+ private bool replaceInvalidAcronym = defaultReplaceInvalidAcronym;
+
/// <summary>Builds an analyzer with the default stop words ({@link #STOP_WORDS}). </summary>
public StandardAnalyzer() : this(STOP_WORDS)
{
@@ -219,7 +258,8 @@
///
/// See https://issues.apache.org/jira/browse/LUCENE-1068
/// </returns>
- public virtual bool IsReplaceInvalidAcronym()
+ /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+ public virtual bool IsReplaceInvalidAcronym()
{
return replaceInvalidAcronym;
}
@@ -229,13 +269,10 @@
///
/// See https://issues.apache.org/jira/browse/LUCENE-1068
/// </param>
- public virtual void SetReplaceInvalidAcronym(bool replaceInvalidAcronym)
+ /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+ public virtual void SetReplaceInvalidAcronym(bool replaceInvalidAcronym)
{
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
- static StandardAnalyzer()
- {
- STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
- }
}
}
\ No newline at end of file