You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/07/08 10:09:37 UTC
svn commit: r1144189 [1/5] - in /lucene/dev/branches/LUCENE2793: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/ lucene/...
Author: simonw
Date: Fri Jul 8 08:09:32 2011
New Revision: 1144189
URL: http://svn.apache.org/viewvc?rev=1144189&view=rev
Log:
merged with trunk
Added:
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java
- copied unchanged from r1144179, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/util/TermContext.java
- copied unchanged from r1144179, lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TermContext.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
- copied unchanged from r1144179, lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
lucene/dev/branches/LUCENE2793/solr/src/test-files/docs1.xml
- copied unchanged from r1144179, lucene/dev/trunk/solr/src/test-files/docs1.xml
lucene/dev/branches/LUCENE2793/solr/src/test-files/docs2.xml
- copied unchanged from r1144179, lucene/dev/trunk/solr/src/test-files/docs2.xml
Removed:
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java
Modified:
lucene/dev/branches/LUCENE2793/ (props changed)
lucene/dev/branches/LUCENE2793/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/LUCENE2793/lucene/ (props changed)
lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt
lucene/dev/branches/LUCENE2793/lucene/MIGRATE.txt
lucene/dev/branches/LUCENE2793/lucene/common-build.xml
lucene/dev/branches/LUCENE2793/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/branches/LUCENE2793/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/LUCENE2793/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/AbstractField.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/Fieldable.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/FieldsReader.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/values/Bytes.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Explanation.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/PhraseScorer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Similarity.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/TermQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/TermScorer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/Weight.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java
lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java
lucene/dev/branches/LUCENE2793/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestSetNorm.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/branches/LUCENE2793/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
lucene/dev/branches/LUCENE2793/modules/join/src/java/org/apache/lucene/search/join/BlockJoinQuery.java
lucene/dev/branches/LUCENE2793/modules/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java
lucene/dev/branches/LUCENE2793/modules/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java
lucene/dev/branches/LUCENE2793/modules/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
lucene/dev/branches/LUCENE2793/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
lucene/dev/branches/LUCENE2793/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
lucene/dev/branches/LUCENE2793/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
lucene/dev/branches/LUCENE2793/solr/ (props changed)
lucene/dev/branches/LUCENE2793/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/LUCENE2793/solr/LICENSE.txt (props changed)
lucene/dev/branches/LUCENE2793/solr/NOTICE.txt (props changed)
lucene/dev/branches/LUCENE2793/solr/README.txt (props changed)
lucene/dev/branches/LUCENE2793/solr/build.xml (props changed)
lucene/dev/branches/LUCENE2793/solr/client/ (props changed)
lucene/dev/branches/LUCENE2793/solr/common-build.xml (contents, props changed)
lucene/dev/branches/LUCENE2793/solr/contrib/ (props changed)
lucene/dev/branches/LUCENE2793/solr/example/ (props changed)
lucene/dev/branches/LUCENE2793/solr/lib/ (props changed)
lucene/dev/branches/LUCENE2793/solr/site/ (props changed)
lucene/dev/branches/LUCENE2793/solr/site/features.pdf
lucene/dev/branches/LUCENE2793/solr/site/index.html
lucene/dev/branches/LUCENE2793/solr/site/index.pdf
lucene/dev/branches/LUCENE2793/solr/site/issue_tracking.pdf
lucene/dev/branches/LUCENE2793/solr/site/linkmap.pdf
lucene/dev/branches/LUCENE2793/solr/site/mailing_lists.pdf
lucene/dev/branches/LUCENE2793/solr/site/tutorial.pdf
lucene/dev/branches/LUCENE2793/solr/site/version_control.pdf
lucene/dev/branches/LUCENE2793/solr/src/ (props changed)
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/Grouping.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/JoinQParserPlugin.java
lucene/dev/branches/LUCENE2793/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java
lucene/dev/branches/LUCENE2793/solr/src/site/src/documentation/content/xdocs/index.xml
lucene/dev/branches/LUCENE2793/solr/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/client/solrj/embedded/TestSolrProperties.java
lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/core/MockCodecProviderFactory.java
lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/request/SimpleFacetsTest.java
lucene/dev/branches/LUCENE2793/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
lucene/dev/branches/LUCENE2793/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/LUCENE2793/lucene/CHANGES.txt Fri Jul 8 08:09:32 2011
@@ -156,6 +156,12 @@ Changes in backwards compatibility polic
the queries module and can be found at o.a.l.queries.function. See MIGRATE.txt
for more information (Chris Male)
+* LUCENE-2392: Decoupled vector space scoring from Query/Weight/Scorer. If you
+ extended Similarity directly before, you should extend TFIDFSimilarity instead.
+ Similarity is now a lower-level API to implement other scoring algorithms.
+ See MIGRATE.txt for more details.
+ (David Nemeskey, Simon Willnauer, Mike Mccandless, Robert Muir)
+
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
Modified: lucene/dev/branches/LUCENE2793/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/MIGRATE.txt?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/MIGRATE.txt (original)
+++ lucene/dev/branches/LUCENE2793/lucene/MIGRATE.txt Fri Jul 8 08:09:32 2011
@@ -382,3 +382,13 @@ LUCENE-1458, LUCENE-2111: Flexible Index
- o.a.l.search.function.ShortFieldSource -> o.a.l.queries.function.valuesource.ShortFieldSource
- o.a.l.search.function.ValueSource -> o.a.l.queries.function.ValueSource
- o.a.l.search.function.ValueSourceQuery -> o.a.l.queries.function.FunctionQuery
+
+* LUCENE-2392: Enable flexible scoring:
+
+ The existing "Similarity" api is now TFIDFSimilarity, if you were extending
+ Similarity before, you should likely extend this instead.
+
+ Weight.normalize no longer takes a norm value that incorporates the top-level
+ boost from outer queries such as BooleanQuery, instead it takes 2 parameters,
+ the outer boost (topLevelBoost) and the norm. Weight.sumOfSquaredWeights has
+ been renamed to Weight.getValueForNormalization().
Modified: lucene/dev/branches/LUCENE2793/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/common-build.xml?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/common-build.xml (original)
+++ lucene/dev/branches/LUCENE2793/lucene/common-build.xml Fri Jul 8 08:09:32 2011
@@ -331,7 +331,7 @@
<artifact:deploy file="${build.dir}/${maven.project.build.finalName}.jar">
<artifact-attachments/>
<remoteRepository url="${m2.repository.url}">
- <authentication username="${m2.repository.username}" privateKey="${m2.repository.private.key}"/>
+ <authentication username="${m2.repository.username}" privateKey="${m2.repository.private.key}" password="${m2.repository.password}"/>
</remoteRepository>
<pom refid="maven.project"/>
</artifact:deploy>
@@ -351,7 +351,7 @@
<artifact:pom id="maven.project" file="${maven.build.dir}/@{pom.xml}" />
<artifact:deploy file="@{jar.file}">
<remoteRepository url="${m2.repository.url}">
- <authentication username="${m2.repository.username}" privateKey="${m2.repository.private.key}"/>
+ <authentication username="${m2.repository.username}" privateKey="${m2.repository.private.key}" password="${m2.repository.password}"/>
</remoteRepository>
<pom refid="maven.project"/>
</artifact:deploy>
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Fri Jul 8 08:09:32 2011
@@ -240,8 +240,7 @@ public class InstantiatedIndexWriter imp
final FieldInvertState invertState = new FieldInvertState();
invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost());
invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
- final float norm = similarityProvider.get(fieldName).computeNorm(invertState);
- normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).encodeNormValue(norm);
+ normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).computeNorm(invertState);
} else {
System.currentTimeMillis();
}
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Jul 8 08:09:32 2011
@@ -51,7 +51,6 @@ import org.apache.lucene.index.TermFreqV
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorMapper;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
@@ -1202,19 +1201,18 @@ public class MemoryIndex {
int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
float boost = info != null ? info.getBoost() : 1.0f;
FieldInvertState invertState = new FieldInvertState(0, numTokens, numOverlapTokens, 0, boost);
- float n = fieldSim.computeNorm(invertState);
- byte norm = fieldSim.encodeNormValue(n);
+ byte norm = fieldSim.computeNorm(invertState);
norms = new byte[] {norm};
// cache it for future reuse
cachedNorms = norms;
cachedFieldName = fieldName;
cachedSimilarity = sim;
- if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" + norm + ":" + numTokens);
+ if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldName + ":" + norm + ":" + numTokens);
}
return norms;
}
-
+
@Override
protected void doSetNorm(int doc, String fieldName, byte value) {
throw new UnsupportedOperationException();
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Fri Jul 8 08:09:32 2011
@@ -147,7 +147,7 @@ public class FieldNormModifier {
for (int d = 0; d < termCounts.length; d++) {
if (liveDocs == null || liveDocs.get(d)) {
invertState.setLength(termCounts[d]);
- subReader.setNorm(d, field, fieldSim.encodeNormValue(fieldSim.computeNorm(invertState)));
+ subReader.setNorm(d, field, fieldSim.computeNorm(invertState));
}
}
}
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java Fri Jul 8 08:09:32 2011
@@ -25,7 +25,6 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
@@ -58,7 +57,7 @@ public class AppendingCodec extends Code
public static String CODEC_NAME = "Appending";
public AppendingCodec() {
- name = CODEC_NAME;
+ super(CODEC_NAME);
}
@Override
@@ -138,22 +137,22 @@ public class AppendingCodec extends Code
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
StandardCodec.getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, state.context);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
}
}
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Fri Jul 8 08:09:32 2011
@@ -106,7 +106,7 @@ public class SweetSpotSimilarity extends
* discountOverlaps is true by default or true for this
* specific field. */
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTokens;
if (discountOverlaps)
@@ -114,7 +114,7 @@ public class SweetSpotSimilarity extends
else
numTokens = state.getLength();
- return state.getBoost() * computeLengthNorm(numTokens);
+ return encodeNormValue(state.getBoost() * computeLengthNorm(numTokens));
}
/**
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java Fri Jul 8 08:09:32 2011
@@ -70,7 +70,7 @@ public class WindowsDirectory extends FS
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
- return new WindowsIndexInput(new File(getDirectory(), name), DEFAULT_BUFFERSIZE);
+ return new WindowsIndexInput(new File(getDirectory(), name), Math.max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE));
}
protected static class WindowsIndexInput extends BufferedIndexInput {
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Fri Jul 8 08:09:32 2011
@@ -49,8 +49,8 @@ public class TestFieldNormModifier exten
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Fri Jul 8 08:09:32 2011
@@ -21,6 +21,7 @@ package org.apache.lucene.misc;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.index.FieldInvertState;
@@ -58,15 +59,15 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("3,10: spot i="+i,
1.0f,
- s.computeNorm(invertState),
+ ss.decodeNormValue(s.computeNorm(invertState)),
0.0f);
}
for (int i = 10; i < 1000; i++) {
invertState.setLength(i-9);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = s.computeNorm(invertState);
+ final byte normS = s.computeNorm(invertState);
assertEquals("3,10: 10<x : i="+i,
normD,
normS,
@@ -104,14 +105,14 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
1.0f,
- sp.get("foo").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("foo").computeNorm(invertState)),
0.0f);
}
for (int i = 10; i < 1000; i++) {
invertState.setLength(i-9);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("foo").computeNorm(invertState);
+ final byte normS = sp.get("foo").computeNorm(invertState);
assertEquals("f: 3,10: 10<x : i="+i,
normD,
normS,
@@ -121,21 +122,21 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
1.0f,
- sp.get("bar").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("bar").computeNorm(invertState)),
0.0f);
}
for (int i = 6; i <=9; i++) {
invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
1.0f,
- sp.get("yak").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("yak").computeNorm(invertState)),
0.0f);
}
for (int i = 13; i < 1000; i++) {
invertState.setLength(i-12);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("bar").computeNorm(invertState);
+ final byte normS = sp.get("bar").computeNorm(invertState);
assertEquals("f: 8,13: 13<x : i="+i,
normD,
normS,
@@ -143,9 +144,9 @@ public class SweetSpotSimilarityTest ext
}
for (int i = 9; i < 1000; i++) {
invertState.setLength(i-8);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("yak").computeNorm(invertState);
+ final byte normS = sp.get("yak").computeNorm(invertState);
assertEquals("f: 6,9: 9<x : i="+i,
normD,
normS,
@@ -157,8 +158,8 @@ public class SweetSpotSimilarityTest ext
for (int i = 9; i < 1000; i++) {
invertState.setLength(i);
- final float normSS = sp.get("a").computeNorm(invertState);
- final float normS = sp.get("b").computeNorm(invertState);
+ final byte normSS = sp.get("a").computeNorm(invertState);
+ final byte normS = sp.get("b").computeNorm(invertState);
assertTrue("s: i="+i+" : a="+normSS+
" < b="+normS,
normSS < normS);
@@ -170,8 +171,8 @@ public class SweetSpotSimilarityTest ext
SweetSpotSimilarity ss = new SweetSpotSimilarity();
- Similarity d = new DefaultSimilarity();
- Similarity s = ss;
+ TFIDFSimilarity d = new DefaultSimilarity();
+ TFIDFSimilarity s = ss;
// tf equal
@@ -222,7 +223,7 @@ public class SweetSpotSimilarityTest ext
};
ss.setHyperbolicTfFactors(3.3f, 7.7f, Math.E, 5.0f);
- Similarity s = ss;
+ TFIDFSimilarity s = ss;
for (int i = 1; i <=1000; i++) {
assertTrue("MIN tf: i="+i+" : s="+s.tf(i),
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Fri Jul 8 08:09:32 2011
@@ -54,8 +54,8 @@ public class TestLengthNormModifier exte
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
@@ -175,8 +175,8 @@ public class TestLengthNormModifier exte
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Fri Jul 8 08:09:32 2011
@@ -51,7 +51,11 @@ import org.apache.lucene.util.PriorityQu
*/
public class FuzzyLikeThisQuery extends Query
{
- static Similarity sim=new DefaultSimilarity();
+ // TODO: generalize this query (at least it should not reuse this static sim!
+ // a better way might be to convert this into multitermquery rewrite methods.
+ // the rewrite method can 'average' the TermContext's term statistics (docfreq,totalTermFreq)
+ // provided to TermQuery, so that the general idea is agnostic to any scoring system...
+ static TFIDFSimilarity sim=new DefaultSimilarity();
Query rewrittenQuery=null;
ArrayList<FieldVals> fieldVals=new ArrayList<FieldVals>();
Analyzer analyzer;
Modified: lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Fri Jul 8 08:09:32 2011
@@ -44,6 +44,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
@@ -285,7 +286,7 @@ public final class MoreLikeThis {
/**
* For idf() calculations.
*/
- private Similarity similarity;// = new DefaultSimilarity();
+ private TFIDFSimilarity similarity;// = new DefaultSimilarity();
/**
* IndexReader to use
@@ -320,17 +321,17 @@ public final class MoreLikeThis {
this(ir, new DefaultSimilarity());
}
- public MoreLikeThis(IndexReader ir, Similarity sim){
+ public MoreLikeThis(IndexReader ir, TFIDFSimilarity sim){
this.ir = ir;
this.similarity = sim;
}
- public Similarity getSimilarity() {
+ public TFIDFSimilarity getSimilarity() {
return similarity;
}
- public void setSimilarity(Similarity similarity) {
+ public void setSimilarity(TFIDFSimilarity similarity) {
this.similarity = similarity;
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/AbstractField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/AbstractField.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/AbstractField.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/AbstractField.java Fri Jul 8 08:09:32 2011
@@ -81,13 +81,13 @@ public abstract class AbstractField impl
* default, in the {@link
* org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)} method, the boost value is multiplied
* by the length normalization factor and then
- * rounded by {@link org.apache.lucene.search.Similarity#encodeNormValue(float)} before it is stored in the
+ * rounded by {@link org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
* the range of that encoding.
*
* @see org.apache.lucene.document.Document#setBoost(float)
* @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.Similarity#encodeNormValue(float)
+ * @see org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)
*/
public void setBoost(float boost) {
this.boost = boost;
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/Fieldable.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/document/Fieldable.java Fri Jul 8 08:09:32 2011
@@ -48,13 +48,13 @@ public interface Fieldable {
* default, in the {@link
* org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)} method, the boost value is multiplied
* by the length normalization factor
- * and then rounded by {@link org.apache.lucene.search.Similarity#encodeNormValue(float)} before it is stored in the
+ * and then rounded by {@link org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
* the range of that encoding.
*
* @see org.apache.lucene.document.Document#setBoost(float)
* @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.Similarity#encodeNormValue(float)
+ * @see org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)
*/
void setBoost(float boost);
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Fri Jul 8 08:09:32 2011
@@ -32,7 +32,6 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.ByteBlockPool.Allocator;
import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
@@ -546,7 +545,7 @@ public class DocumentsWriterPerThread {
PerDocWriteState newPerDocWriteState(int codecId) {
assert segment != null;
- return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId);
+ return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT);
}
void setInfoStream(PrintStream infoStream) {
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/FieldsReader.java Fri Jul 8 08:09:32 2011
@@ -31,9 +31,12 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.util.IOUtils;
+import java.io.Closeable;
import java.io.IOException;
import java.io.Reader;
+import java.util.ArrayList;
/**
* Class responsible for access to stored document fields.
@@ -42,7 +45,7 @@ import java.io.Reader;
*
* @lucene.internal
*/
-public final class FieldsReader implements Cloneable {
+public final class FieldsReader implements Cloneable, Closeable {
private final static int FORMAT_SIZE = 4;
private final FieldInfos fieldInfos;
@@ -180,21 +183,11 @@ public final class FieldsReader implemen
*/
public final void close() throws IOException {
if (!closed) {
- if (fieldsStream != null) {
- fieldsStream.close();
- }
if (isOriginal) {
- if (cloneableFieldsStream != null) {
- cloneableFieldsStream.close();
- }
- if (cloneableIndexStream != null) {
- cloneableIndexStream.close();
- }
- }
- if (indexStream != null) {
- indexStream.close();
+ IOUtils.closeSafely(false, fieldsStream, indexStream, fieldsStreamTL, cloneableFieldsStream, cloneableIndexStream);
+ } else {
+ IOUtils.closeSafely(false, fieldsStream, indexStream, fieldsStreamTL);
}
- fieldsStreamTL.close();
closed = true;
}
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Jul 8 08:09:32 2011
@@ -1025,7 +1025,7 @@ public abstract class IndexReader implem
public abstract byte[] norms(String field) throws IOException;
/** Expert: Resets the normalization factor for the named field of the named
- * document. The norm represents the product of the field's {@link
+ * document. By default, The norm represents the product of the field's {@link
* org.apache.lucene.document.Fieldable#setBoost(float) boost} and its
* length normalization}. Thus, to preserve the length normalization
* values when resetting this, one should base the new value upon the old.
@@ -1034,7 +1034,8 @@ public abstract class IndexReader implem
* this method throws {@link IllegalStateException}.
*
* @see #norms(String)
- * @see Similarity#decodeNormValue(byte)
+ * @see Similarity#computeNorm(FieldInvertState)
+ * @see org.apache.lucene.search.DefaultSimilarity#decodeNormValue(byte)
* @throws StaleReaderException if the index has changed
* since this reader was opened
* @throws CorruptIndexException if the index is corrupt
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java Fri Jul 8 08:09:32 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.index.values.MultiIndexDocValues;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.index.values.MultiIndexDocValues.DocValuesIndex;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.ReaderUtil.Gather;
@@ -151,20 +152,7 @@ public class MultiPerDocValues extends P
}
public void close() throws IOException {
- final PerDocValues[] perDocValues = this.subs;
- IOException ex = null;
- for (PerDocValues values : perDocValues) {
- try {
- values.close();
- } catch (IOException e) {
- if (ex == null) {
- ex = e;
- }
- }
- }
- if (ex != null) {
- throw ex;
- }
+ IOUtils.closeSafely(false, this.subs);
}
@Override
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java Fri Jul 8 08:09:32 2011
@@ -72,8 +72,7 @@ final class NormsWriterPerField extends
assert norms.length == upto;
norms = ArrayUtil.grow(norms, 1+upto);
}
- final float norm = similarity.computeNorm(fieldState);
- norms[upto] = similarity.encodeNormValue(norm);
+ norms[upto] = similarity.computeNorm(fieldState);
docIDs[upto] = docState.docID;
upto++;
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java Fri Jul 8 08:09:32 2011
@@ -20,6 +20,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
/**
* Encapsulates all necessary state to initiate a {@link PerDocConsumer} and
@@ -35,10 +36,11 @@ public class PerDocWriteState {
public final AtomicLong bytesUsed;
public final SegmentCodecs segmentCodecs;
public final int codecId;
+ public final IOContext context;
PerDocWriteState(PrintStream infoStream, Directory directory,
String segmentName, FieldInfos fieldInfos, AtomicLong bytesUsed,
- int codecId) {
+ int codecId, IOContext context) {
this.infoStream = infoStream;
this.directory = directory;
this.segmentName = segmentName;
@@ -46,6 +48,7 @@ public class PerDocWriteState {
this.segmentCodecs = fieldInfos.buildSegmentCodecs(false);
this.codecId = codecId;
this.bytesUsed = bytesUsed;
+ this.context = context;
}
PerDocWriteState(SegmentWriteState state) {
@@ -56,6 +59,7 @@ public class PerDocWriteState {
fieldInfos = state.fieldInfos;
codecId = state.codecId;
bytesUsed = new AtomicLong(0);
+ context = state.context;
}
PerDocWriteState(PerDocWriteState state, int codecId) {
@@ -66,5 +70,6 @@ public class PerDocWriteState {
this.segmentCodecs = state.segmentCodecs;
this.codecId = codecId;
this.bytesUsed = state.bytesUsed;
+ this.context = state.context;
}
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java Fri Jul 8 08:09:32 2011
@@ -52,7 +52,7 @@ final class PerFieldCodecWrapper extends
private final SegmentCodecs segmentCodecs;
PerFieldCodecWrapper(SegmentCodecs segmentCodecs) {
- name = "PerField";
+ super("PerField");
this.segmentCodecs = segmentCodecs;
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Fri Jul 8 08:09:32 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.codecs.Pe
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.IOUtils;
/** Holds core readers that are shared (unchanged) when
* SegmentReader is cloned or reopened */
@@ -120,33 +121,9 @@ final class SegmentCoreReaders {
}
synchronized void decRef() throws IOException {
-
if (ref.decrementAndGet() == 0) {
-
- if (fields != null) {
- fields.close();
- }
-
- if (perDocProducer != null) {
- perDocProducer.close();
- }
-
- if (termVectorsReaderOrig != null) {
- termVectorsReaderOrig.close();
- }
-
- if (fieldsReaderOrig != null) {
- fieldsReaderOrig.close();
- }
-
- if (cfsReader != null) {
- cfsReader.close();
- }
-
- if (storeCFSReader != null) {
- storeCFSReader.close();
- }
-
+ IOUtils.closeSafely(false, fields, perDocProducer, termVectorsReaderOrig,
+ fieldsReaderOrig, cfsReader, storeCFSReader);
// Now, notify any ReaderFinished listeners:
if (owner != null) {
owner.notifyReaderFinishedListeners();
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Fri Jul 8 08:09:32 2011
@@ -30,7 +30,6 @@ import java.util.Set;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
-import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -248,7 +247,7 @@ public final class SegmentInfo implement
}
final Directory dirToTest;
if (isCompoundFile) {
- dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE );
+ dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE);
} else {
dirToTest = dir;
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Jul 8 08:09:32 2011
@@ -119,6 +119,7 @@ final class SegmentMerger {
mergedDocs = mergeFields();
mergeTerms();
+ mergePerDoc();
mergeNorms();
if (fieldInfos.hasVectors())
@@ -482,18 +483,11 @@ final class SegmentMerger {
// the new segment:
int docBase = 0;
-
+
final List<Fields> fields = new ArrayList<Fields>();
-
final List<ReaderUtil.Slice> slices = new ArrayList<ReaderUtil.Slice>();
final List<Bits> bits = new ArrayList<Bits>();
final List<Integer> bitsStarts = new ArrayList<Integer>();
-
- // TODO: move this into its own method - this merges currently only docvalues
- final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();
- final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
- final List<Bits> perDocBits = new ArrayList<Bits>();
- final List<Integer> perDocBitsStarts = new ArrayList<Integer>();
for(IndexReader r : readers) {
final Fields f = r.fields();
@@ -504,18 +498,10 @@ final class SegmentMerger {
bits.add(r.getLiveDocs());
bitsStarts.add(docBase);
}
- final PerDocValues producer = r.perDocValues();
- if (producer != null) {
- perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size()));
- perDocProducers.add(producer);
- perDocBits.add(r.getLiveDocs());
- perDocBitsStarts.add(docBase);
- }
docBase += maxDoc;
}
bitsStarts.add(docBase);
- perDocBitsStarts.add(docBase);
// we may gather more readers than mergeState.readerCount
mergeState = new MergeState();
@@ -581,19 +567,45 @@ final class SegmentMerger {
} finally {
consumer.close();
}
+ }
+
+ private void mergePerDoc() throws IOException {
+ final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();
+ final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
+ final List<Bits> perDocBits = new ArrayList<Bits>();
+ final List<Integer> perDocBitsStarts = new ArrayList<Integer>();
+ int docBase = 0;
+ for (IndexReader r : readers) {
+ final int maxDoc = r.maxDoc();
+ final PerDocValues producer = r.perDocValues();
+ if (producer != null) {
+ perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, perDocProducers
+ .size()));
+ perDocProducers.add(producer);
+ perDocBits.add(r.getLiveDocs());
+ perDocBitsStarts.add(docBase);
+ }
+ docBase += maxDoc;
+ }
+ perDocBitsStarts.add(docBase);
if (!perDocSlices.isEmpty()) {
- mergeState.multiLiveDocs = new MultiBits(perDocBits, perDocBitsStarts, true);
+ mergeState.multiLiveDocs = new MultiBits(perDocBits, perDocBitsStarts,
+ true);
final PerDocConsumer docsConsumer = codec
.docsConsumer(new PerDocWriteState(segmentWriteState));
+ boolean success = false;
try {
- final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(perDocProducers
- .toArray(PerDocValues.EMPTY_ARRAY), perDocSlices
- .toArray(ReaderUtil.Slice.EMPTY_ARRAY));
+ final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(
+ perDocProducers.toArray(PerDocValues.EMPTY_ARRAY),
+ perDocSlices.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
docsConsumer.merge(mergeState, multiPerDocValues);
+ success = true;
} finally {
- docsConsumer.close();
+ IOUtils.closeSafely(!success, docsConsumer);
}
}
+ /* don't close the perDocProducers here since they are private segment producers
+ * and will be closed once the SegmentReader goes out of scope */
}
private MergeState mergeState;
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java Fri Jul 8 08:09:32 2011
@@ -25,11 +25,13 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
-class TermVectorsReader implements Cloneable {
+class TermVectorsReader implements Cloneable, Closeable {
// NOTE: if you make a new format, it must be larger than
// the current format
@@ -190,14 +192,8 @@ class TermVectorsReader implements Clone
return format;
}
- void close() throws IOException {
- // make all effort to close up. Keep the first exception
- // and throw it as a new one.
- IOException keep = null;
- if (tvx != null) try { tvx.close(); } catch (IOException e) { keep = e; }
- if (tvd != null) try { tvd.close(); } catch (IOException e) { if (keep == null) keep = e; }
- if (tvf != null) try { tvf.close(); } catch (IOException e) { if (keep == null) keep = e; }
- if (keep != null) throw (IOException) keep.fillInStackTrace();
+ public void close() throws IOException {
+ IOUtils.closeSafely(false, tvx, tvd, tvf);
}
/**
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Fri Jul 8 08:09:32 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index.codecs;
*/
import java.io.IOException;
+import java.util.Comparator;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
@@ -25,13 +26,21 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
/** @lucene.experimental */
public abstract class Codec {
public static final Codec[] EMPTY = new Codec[0];
/** Unique name that's used to retrieve this codec when
* reading the index */
- public String name;
+ public final String name;
+ private boolean dvUseCompoundFile = true;
+ private Comparator<BytesRef> docValuesSortComparator = BytesRef
+ .getUTF8SortedAsUnicodeComparator();
+
+ protected Codec(String name) {
+ this.name = name;
+ }
/** Writes a new segment */
public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException;
@@ -68,7 +77,48 @@ public abstract class Codec {
/** Records all file extensions this codec uses */
public abstract void getExtensions(Set<String> extensions);
+
+
+ /**
+ * If set to <code>true</code> this codec will use a compound file for
+ * IndexDocValues, otherwise each IndexDocValues field will create up to 2
+ * files per segment.
+ * <p>
+ * NOTE: The default values is <code>true</code>.
+ */
+ public void setDocValuesUseCFS(boolean docValuesUseCFS) {
+ this.dvUseCompoundFile = docValuesUseCFS;
+ }
+
+ /**
+ * Returns <code>true</code> iff compound file should be used for
+ * IndexDocValues, otherwise <code>false</code>.
+ *
+ * @see #setDocValuesUseCFS(boolean)
+ * @return <code>true</code> iff compound file should be used for
+ * IndexDocValues, otherwise <code>false</code>.
+ */
+ public boolean getDocValuesUseCFS() {
+ return dvUseCompoundFile;
+ }
+
+ /**
+ * Sets the {@link BytesRef} comparator for sorted IndexDocValue variants. The
+ * default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}. *
+ */
+ public void setDocValuesSortComparator(
+ Comparator<BytesRef> docValuesSortComparator) {
+ this.docValuesSortComparator = docValuesSortComparator;
+ }
+ /**
+ * Returns the {@link BytesRef} comparator for sorted IndexDocValue variants.
+ * The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
+ */
+ public Comparator<BytesRef> getDocValuesSortComparator() {
+ return docValuesSortComparator;
+ }
+
@Override
public String toString() {
return name;
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java Fri Jul 8 08:09:32 2011
@@ -44,7 +44,7 @@ public class CoreCodecProvider extends C
public CoreCodecProvider() {
register(new StandardCodec());
register(new PreFlexCodec());
- register(new PulsingCodec(1));
+ register(new PulsingCodec());
register(new SimpleTextCodec());
register(new MemoryCodec());
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Fri Jul 8 08:09:32 2011
@@ -32,79 +32,106 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
+/**
+ *
+ * @lucene.experimental
+ */
public class DefaultDocValuesConsumer extends PerDocConsumer {
private final String segmentName;
private final int codecId;
private final Directory directory;
private final AtomicLong bytesUsed;
private final Comparator<BytesRef> comparator;
-
- public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator) {
+ private boolean useCompoundFile;
+ private final IOContext context;
+
+ public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator, boolean useCompoundFile) throws IOException {
this.segmentName = state.segmentName;
this.codecId = state.codecId;
this.bytesUsed = state.bytesUsed;
- this.directory = state.directory;
+ this.context = state.context;
+ //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files?
+ this.directory = useCompoundFile ? state.directory.createCompoundOutput(
+ IndexFileNames.segmentFileName(segmentName, codecId,
+ IndexFileNames.COMPOUND_FILE_EXTENSION), context) : state.directory;
this.comparator = comparator;
+ this.useCompoundFile = useCompoundFile;
}
-
+
public void close() throws IOException {
+ if (useCompoundFile) {
+ this.directory.close();
+ }
}
@Override
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
return Writer.create(field.getDocValues(),
docValuesId(segmentName, codecId, field.number),
- // TODO can we have a compound file per segment and codec for
- // docvalues?
- directory, comparator, bytesUsed, IOContext.DEFAULT);
+ directory, comparator, bytesUsed, context);
}
@SuppressWarnings("fallthrough")
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
- Set<String> files) throws IOException {
+ Set<String> files, boolean useCompoundFile) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
String filename = docValuesId(segmentInfo.name, codecId,
fieldInfo.number);
- switch (fieldInfo.getDocValues()) {
- case BYTES_FIXED_DEREF:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_STRAIGHT:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- // until here all types use an index
- case BYTES_FIXED_STRAIGHT:
- case FLOAT_32:
- case FLOAT_64:
- case VAR_INTS:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- break;
-
- default:
- assert false;
+ if (useCompoundFile) {
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ return;
+ } else {
+ switch (fieldInfo.getDocValues()) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ Writer.INDEX_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ Writer.INDEX_EXTENSION));
+ // until here all types use an index
+ case BYTES_FIXED_STRAIGHT:
+ case FLOAT_32:
+ case FLOAT_64:
+ case VAR_INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ Writer.DATA_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ Writer.DATA_EXTENSION));
+ break;
+
+ default:
+ assert false;
+ }
}
}
}
}
+
static String docValuesId(String segmentsName, int codecID, int fieldId) {
return segmentsName + "_" + codecID + "-" + fieldId;
}
-
- public static void getDocValuesExtensions(Set<String> extensions) {
- extensions.add(Writer.DATA_EXTENSION);
- extensions.add(Writer.INDEX_EXTENSION);
+
+ public static void getDocValuesExtensions(Set<String> extensions, boolean useCompoundFile) {
+ if (useCompoundFile) {
+ extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
+ extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
+ } else {
+ extensions.add(Writer.DATA_EXTENSION);
+ extensions.add(Writer.INDEX_EXTENSION);
+ }
}
+
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Fri Jul 8 08:09:32 2011
@@ -16,12 +16,16 @@ package org.apache.lucene.index.codecs;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Comparator;
import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.values.Bytes;
import org.apache.lucene.index.values.IndexDocValues;
@@ -30,6 +34,8 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/**
* Abstract base class for FieldsProducer implementations supporting
@@ -40,8 +46,12 @@ import org.apache.lucene.store.IOContext
public class DefaultDocValuesProducer extends PerDocValues {
protected final TreeMap<String, IndexDocValues> docValues;
+ private final boolean useCompoundFile;
+ private final Closeable cfs;
+ private final Comparator<BytesRef> sortComparator;
/**
+ *
* Creates a new {@link DefaultDocValuesProducer} instance and loads all
* {@link IndexDocValues} instances for this segment and codec.
*
@@ -53,12 +63,27 @@ public class DefaultDocValuesProducer ex
* the {@link FieldInfos}
* @param codecId
* the codec ID
+ * @param useCompoundFile
+ * if <code>true</code> this producer opens a compound file to read
+ * IndexDocValues fields, otherwise each field defines its own set of
+ * files.
+ * @param sortComparator
+ * defines the sort order for sorted IndexDocValues variants
* @throws IOException
* if an {@link IOException} occurs
*/
- public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
- FieldInfos fieldInfo, int codecId, IOContext context) throws IOException {
- docValues = load(fieldInfo, si.name, si.docCount, dir, codecId, context);
+ public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
+ FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
+ this.useCompoundFile = useCompoundFile;
+ this.sortComparator = sortComparator;
+ final Directory directory;
+ if (useCompoundFile) {
+ cfs = directory = dir.openCompoundInput(IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context);
+ } else {
+ cfs = null;
+ directory = dir;
+ }
+ docValues = load(fieldInfo, si.name, si.docCount, directory, codecId, context);
}
/**
@@ -86,14 +111,14 @@ public class DefaultDocValuesProducer ex
final String id = DefaultDocValuesConsumer.docValuesId(segment,
codecId, fieldInfo.number);
values.put(field,
- loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context));
+ loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator, context));
}
}
success = true;
} finally {
if (!success) {
// if we fail we must close all opened resources if there are any
- closeDocValues(values.values());
+ closeInternal(values.values());
}
}
return values;
@@ -113,6 +138,7 @@ public class DefaultDocValuesProducer ex
* the unique file ID within the segment
* @param type
* the type to load
+ * @param sortComparator byte comparator used by sorted variants
* @return a {@link IndexDocValues} instance for the given type
* @throws IOException
* if an {@link IOException} occurs
@@ -120,7 +146,7 @@ public class DefaultDocValuesProducer ex
* if the given {@link ValueType} is not supported
*/
protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
- ValueType type, IOContext context) throws IOException {
+ ValueType type, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
switch (type) {
case FIXED_INTS_16:
case FIXED_INTS_32:
@@ -133,39 +159,36 @@ public class DefaultDocValuesProducer ex
case FLOAT_64:
return Floats.getValues(dir, id, docCount, context);
case BYTES_FIXED_STRAIGHT:
- return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, context);
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator, context);
case BYTES_FIXED_DEREF:
- return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, context);
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator, context);
case BYTES_FIXED_SORTED:
- return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, context);
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator, context);
case BYTES_VAR_STRAIGHT:
- return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, context);
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator, context);
case BYTES_VAR_DEREF:
- return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, context);
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator, context);
case BYTES_VAR_SORTED:
- return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, context);
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator, context);
default:
throw new IllegalStateException("unrecognized index values mode " + type);
}
}
public void close() throws IOException {
- closeDocValues(docValues.values());
+ closeInternal(docValues.values());
}
- private void closeDocValues(final Collection<IndexDocValues> values)
- throws IOException {
- IOException ex = null;
- for (IndexDocValues docValues : values) {
- try {
- docValues.close();
- } catch (IOException e) {
- ex = e;
- }
- }
- if (ex != null) {
- throw ex;
- }
+ private void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
+ final Collection<? extends Closeable> toClose;
+ if (useCompoundFile) {
+ final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
+ list.add(cfs);
+ toClose = list;
+ } else {
+ toClose = closeables;
+ }
+ IOUtils.closeSafely(false, toClose);
}
@Override
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java Fri Jul 8 08:09:32 2011
@@ -77,9 +77,9 @@ import org.apache.lucene.util.fst.FST;
* @lucene.experimental */
public class MemoryCodec extends Codec {
-
+
public MemoryCodec() {
- name = "Memory";
+ super("Memory");
}
private static final boolean VERBOSE = false;
@@ -779,22 +779,22 @@ public class MemoryCodec extends Codec {
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(EXTENSION);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, IOContext.READONCE);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), IOContext.READONCE);
}
}
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Fri Jul 8 08:09:32 2011
@@ -55,7 +55,7 @@ public class PreFlexCodec extends Codec
public static final String PROX_EXTENSION = "prx";
public PreFlexCodec() {
- name = "PreFlex";
+ super("PreFlex");
}
@Override
Modified: lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1144189&r1=1144188&r2=1144189&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Fri Jul 8 08:09:32 2011
@@ -43,7 +43,6 @@ import org.apache.lucene.index.codecs.Te
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/** This codec "inlines" the postings for terms that have
@@ -58,10 +57,19 @@ public class PulsingCodec extends Codec
private final int freqCutoff;
+ /**
+ * Creates a {@link PulsingCodec} with <tt>freqCutoff = 1</tt>
+ *
+ * @see PulsingCodec#PulsingCodec(int)
+ */
+ public PulsingCodec() {
+ this(1);
+ }
+
/** Terms with freq <= freqCutoff are inlined into terms
* dict. */
public PulsingCodec(int freqCutoff) {
- name = "Pulsing";
+ super("Pulsing");
this.freqCutoff = freqCutoff;
}
@@ -157,22 +165,22 @@ public class PulsingCodec extends Codec
StandardPostingsReader.files(dir, segmentInfo, id, files);
BlockTermsReader.files(dir, segmentInfo, id, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, id, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
StandardCodec.getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, state.context);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
}
}