You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/01/14 14:57:07 UTC
svn commit: r1059010 [1/4] - in /lucene/dev/branches/bulkpostings: ./ lucene/
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
lucene/contrib/memory/sr...
Author: rmuir
Date: Fri Jan 14 13:57:03 2011
New Revision: 1059010
URL: http://svn.apache.org/viewvc?rev=1059010&view=rev
Log:
merge trunk (1058366:1058939)
Added:
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
- copied unchanged from r1058939, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
- copied unchanged from r1058939, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
Removed:
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrIndexReader.java
Modified:
lucene/dev/branches/bulkpostings/ (props changed)
lucene/dev/branches/bulkpostings/lucene/ (props changed)
lucene/dev/branches/bulkpostings/lucene/CHANGES.txt
lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/Fields.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiNorms.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriter.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/ParallelReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Similarity.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSource.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/QueryUtils.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSetNorm.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestTermScorer.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/JustCompileSearchSpans.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestFieldScoreQuery.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestOrdValues.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/function/TestValueSource.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
lucene/dev/branches/bulkpostings/solr/ (props changed)
lucene/dev/branches/bulkpostings/solr/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java
lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/ReplicationHandler.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/SnapPuller.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/HighlightComponent.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/request/UnInvertedField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/RandomSortField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableDoubleField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableFloatField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableIntField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableLongField.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/StrFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/Grouping.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrFilter.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/BoostedQuery.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ByteFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ConstValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleConstValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DualFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FileFloatSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FloatFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FunctionQuery.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/IntFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/JoinDocFreqValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/LinearFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/LiteralValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/LongFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/MaxFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/MultiFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/NormValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/OrdFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/QueryValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/RangeMapFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ReciprocalFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ScaleFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ShortFieldSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/SimpleFloatFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/StringIndexDocValues.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/TFValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/VectorValueSource.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/spelling/suggest/Suggester.java
lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/update/UpdateHandler.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/BaseDistributedSearchTestCase.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/client/solrj/response/TermsResponseTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/core/RAMDirectoryFactoryTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/core/TestConfig.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/core/TestQuerySenderListener.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/handler/TestCSVLoader.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/search/TestIndexSearcher.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java
lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
lucene/dev/branches/bulkpostings/solr/src/webapp/src/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java
Modified: lucene/dev/branches/bulkpostings/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/CHANGES.txt?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/lucene/CHANGES.txt Fri Jan 14 13:57:03 2011
@@ -134,6 +134,9 @@ Changes in backwards compatibility polic
* LUCENE-2839: Scorer#score(Collector,int,int) is now public because it is
called from other classes and part of public API. (Uwe Schindler)
+
+* LUCENE-2865: Weight#scorer(AtomicReaderContext, boolean, boolean) now accepts
+ a ScorerContext struct instead of booleans.(Simon Willnauer)
Changes in Runtime Behavior
@@ -152,6 +155,17 @@ Changes in Runtime Behavior
without any changes to the index will not cause any index version increment.
(Simon Willnauer, Mike McCandless)
+* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
+ omitNorms(true) for field "a" for 1000 documents, but then add a document with
+ omitNorms(false) for field "a", all documents for field "a" will have no norms.
+ Previously, Lucene would fill the first 1000 documents with "fake norms" from
+ Similarity.getDefault(). (Robert Muir, Mike Mccandless)
+
+* LUCENE-2846: When some documents contain field "a", and others do not, the
+ documents that don't have the field get a norm byte value of 0. Previously, Lucene
+ would populate "fake norms" with Similarity.getDefault() for these documents.
+ (Robert Muir, Mike Mccandless)
+
API Changes
* LUCENE-2302, LUCENE-1458, LUCENE-2111, LUCENE-2514: Terms are no longer
@@ -193,6 +207,14 @@ API Changes
Collector#setNextReader & FieldComparator#setNextReader now expect an
AtomicReaderContext instead of an IndexReader. (Simon Willnauer)
+* LUCENE-2846: Remove the deprecated IndexReader.setNorm(int, String, float).
+ This method was only syntactic sugar for setNorm(int, String, byte), but
+ using the global Similarity.getDefault().encodeNormValue. Use the byte-based
+ method instead to ensure that the norm is encoded with your Similarity.
+ Also removed norms(String, byte[], int), which was only used by MultiReader
+ for building top-level norms. If you really need a top-level norms, use
+ MultiNorms or SlowMultiReaderWrapper. (Robert Muir, Mike Mccandless)
+
New features
* LUCENE-2604: Added RegexpQuery support to QueryParser. Regular expressions
@@ -715,6 +737,10 @@ New features
is no longer needed and discouraged for that use case. Directly wrapping
Query improves performance, as out-of-order collection is now supported.
(Uwe Schindler)
+
+* LUCENE-2864: Add getMaxTermFrequency (maximum within-document TF) to
+ FieldInvertState so that it can be used in Similarity.computeNorm.
+ (Robert Muir)
Optimizations
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java Fri Jan 14 13:57:03 2011
@@ -334,15 +334,6 @@ public class InstantiatedIndexReader ext
}
@Override
- public void norms(String field, byte[] bytes, int offset) throws IOException {
- byte[] norms = getIndex().getNormsByFieldNameAndDocumentNumber().get(field);
- if (norms == null) {
- return;
- }
- System.arraycopy(norms, 0, bytes, offset, norms.length);
- }
-
- @Override
protected void doSetNorm(int doc, String field, byte value) throws IOException {
if (uncommittedNormsByFieldNameAndDocumentNumber == null) {
uncommittedNormsByFieldNameAndDocumentNumber = new HashMap<String,List<NormUpdate>>(getIndex().getNormsByFieldNameAndDocumentNumber().size());
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Fri Jan 14 13:57:03 2011
@@ -201,9 +201,9 @@ public class InstantiatedIndexWriter imp
byte[] oldNorms = index.getNormsByFieldNameAndDocumentNumber().get(field);
if (oldNorms != null) {
System.arraycopy(oldNorms, 0, norms, 0, oldNorms.length);
- Arrays.fill(norms, oldNorms.length, norms.length, similarity.encodeNormValue(1.0f));
+ Arrays.fill(norms, oldNorms.length, norms.length, (byte) 0);
} else {
- Arrays.fill(norms, 0, norms.length, similarity.encodeNormValue(1.0f));
+ Arrays.fill(norms, 0, norms.length, (byte) 0);
}
normsByFieldNameAndDocumentNumber.put(field, norms);
fieldNames.remove(field);
@@ -211,7 +211,7 @@ public class InstantiatedIndexWriter imp
for (String field : fieldNames) {
//System.out.println(field);
byte[] norms = new byte[index.getDocumentsByNumber().length + termDocumentInformationFactoryByDocument.size()];
- Arrays.fill(norms, 0, norms.length, similarity.encodeNormValue(1.0f));
+ Arrays.fill(norms, 0, norms.length, (byte) 0);
normsByFieldNameAndDocumentNumber.put(field, norms);
}
fieldNames.clear();
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java Fri Jan 14 13:57:03 2011
@@ -71,12 +71,6 @@ public class TestEmptyIndex extends Luce
byte[] norms = MultiNorms.norms(r, "foo");
if (norms != null) {
assertEquals(0, norms.length);
- norms = new byte[10];
- Arrays.fill(norms, (byte)10);
- r.norms("foo", norms, 10);
- for (byte b : norms) {
- assertEquals((byte)10, b);
- }
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Fri Jan 14 13:57:03 2011
@@ -358,35 +358,6 @@ public class TestIndicesEquals extends L
for (int i = 0; i < aprioriNorms.length; i++) {
assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]);
}
-
- // test norms as used by multireader
-
- aprioriNorms = new byte[aprioriReader.maxDoc()];
- MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 0);
-
- testNorms = new byte[testReader.maxDoc()];
- MultiNorms.norms(testReader, (String) field, testNorms, 0);
-
- assertEquals(aprioriNorms.length, testNorms.length);
-
- for (int i = 0; i < aprioriNorms.length; i++) {
- assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]);
- }
-
-
- // test norms as used by multireader
-
- aprioriNorms = new byte[aprioriReader.maxDoc() + 10];
- MultiNorms.norms(aprioriReader, (String) field, aprioriNorms, 10);
-
- testNorms = new byte[testReader.maxDoc() + 10];
- MultiNorms.norms(testReader, (String) field, testNorms, 10);
-
- assertEquals(aprioriNorms.length, testNorms.length);
-
- for (int i = 0; i < aprioriNorms.length; i++) {
- assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]);
- }
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Jan 14 13:57:03 2011
@@ -1365,13 +1365,6 @@ public class MemoryIndex implements Seri
}
@Override
- public void norms(String fieldName, byte[] bytes, int offset) {
- if (DEBUG) System.err.println("MemoryIndexReader.norms*: " + fieldName);
- byte[] norms = norms(fieldName);
- System.arraycopy(norms, 0, bytes, offset, norms.length);
- }
-
- @Override
protected void doSetNorm(int doc, String fieldName, byte value) {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Fri Jan 14 13:57:03 2011
@@ -372,7 +372,7 @@ class BufferedDeletes {
Query query = entry.getKey();
int limit = entry.getValue().intValue();
Weight weight = query.weight(searcher);
- Scorer scorer = weight.scorer(readerContext, true, false);
+ Scorer scorer = weight.scorer(readerContext, Weight.ScorerContext.def());
if (scorer != null) {
while(true) {
int doc = scorer.nextDoc();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/CheckIndex.java Fri Jan 14 13:57:03 2011
@@ -548,10 +548,10 @@ public class CheckIndex {
if (infoStream != null) {
infoStream.print(" test: field norms.........");
}
- final byte[] b = new byte[reader.maxDoc()];
+ byte[] b;
for (final String fieldName : fieldNames) {
if (reader.hasNorms(fieldName)) {
- reader.norms(fieldName, b, 0);
+ b = reader.norms(fieldName);
++status.totFields;
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Fri Jan 14 13:57:03 2011
@@ -602,12 +602,6 @@ class DirectoryReader extends IndexReade
}
@Override
- public synchronized void norms(String field, byte[] result, int offset)
- throws IOException {
- throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level norms");
- }
-
- @Override
protected void doSetNorm(int n, String field, byte value)
throws CorruptIndexException, IOException {
int i = readerIndex(n); // find segment num
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfo.java Fri Jan 14 13:57:03 2011
@@ -52,7 +52,7 @@ public final class FieldInfo {
this.storeOffsetWithTermVector = false;
this.storePositionWithTermVector = false;
this.storePayloads = false;
- this.omitNorms = true;
+ this.omitNorms = false;
this.omitTermFreqAndPositions = false;
}
}
@@ -82,7 +82,7 @@ public final class FieldInfo {
this.storePayloads = true;
}
if (this.omitNorms != omitNorms) {
- this.omitNorms = false; // once norms are stored, always store
+ this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
}
if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) {
this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInfos.java Fri Jan 14 13:57:03 2011
@@ -278,14 +278,21 @@ public final class FieldInfos {
}
public boolean hasVectors() {
- boolean hasVectors = false;
for (int i = 0; i < size(); i++) {
if (fieldInfo(i).storeTermVector) {
- hasVectors = true;
- break;
+ return true;
}
}
- return hasVectors;
+ return false;
+ }
+
+ public boolean hasNorms() {
+ for (int i = 0; i < size(); i++) {
+ if (!fieldInfo(i).omitNorms) {
+ return true;
+ }
+ }
+ return false;
}
public void write(Directory d, String name) throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FieldInvertState.java Fri Jan 14 13:57:03 2011
@@ -30,6 +30,7 @@ public final class FieldInvertState {
int length;
int numOverlap;
int offset;
+ int maxTermFrequency;
float boost;
AttributeSource attributeSource;
@@ -53,6 +54,7 @@ public final class FieldInvertState {
length = 0;
numOverlap = 0;
offset = 0;
+ maxTermFrequency = 0;
boost = docBoost;
attributeSource = null;
}
@@ -110,6 +112,15 @@ public final class FieldInvertState {
public void setBoost(float boost) {
this.boost = boost;
}
+
+ /**
+ * Get the maximum term-frequency encountered for any term in the field. A
+ * field containing "the quick brown fox jumps over the lazy dog" would have
+ * a value of 2, because "the" appears twice.
+ */
+ public int getMaxTermFrequency() {
+ return maxTermFrequency;
+ }
public AttributeSource getAttributeSource() {
return attributeSource;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/Fields.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/Fields.java Fri Jan 14 13:57:03 2011
@@ -28,7 +28,7 @@ public abstract class Fields {
* names. This will not return null. */
public abstract FieldsEnum iterator() throws IOException;
- /** Get the {@link Terms} for this field. This may return
+ /** Get the {@link Terms} for this field. This will return
* null if the field does not exist. */
public abstract Terms terms(String field) throws IOException;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Fri Jan 14 13:57:03 2011
@@ -353,12 +353,6 @@ public class FilterIndexReader extends I
}
@Override
- public void norms(String f, byte[] bytes, int offset) throws IOException {
- ensureOpen();
- in.norms(f, bytes, offset);
- }
-
- @Override
protected void doSetNorm(int d, String f, byte b) throws CorruptIndexException, IOException {
in.setNorm(d, f, b);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Fri Jan 14 13:57:03 2011
@@ -125,6 +125,7 @@ final class FreqProxTermsWriterPerField
postings.docFreqs[termID] = 1;
writeProx(termID, fieldState.position);
}
+ fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
}
@Override
@@ -158,11 +159,12 @@ final class FreqProxTermsWriterPerField
termsHashPerField.writeVInt(0, postings.docFreqs[termID]);
}
postings.docFreqs[termID] = 1;
+ fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1;
postings.lastDocIDs[termID] = docState.docID;
writeProx(termID, fieldState.position);
} else {
- postings.docFreqs[termID]++;
+ fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.docFreqs[termID]);
writeProx(termID, fieldState.position-postings.lastPositions[termID]);
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Jan 14 13:57:03 2011
@@ -931,14 +931,6 @@ public abstract class IndexReader implem
*/
public abstract byte[] norms(String field) throws IOException;
- /** Reads the byte-encoded normalization factor for the named field of every
- * document. This is used by the search code to score documents.
- *
- * @see org.apache.lucene.document.Field#setBoost(float)
- */
- public abstract void norms(String field, byte[] bytes, int offset)
- throws IOException;
-
/** Expert: Resets the normalization factor for the named field of the named
* document. The norm represents the product of the field's {@link
* org.apache.lucene.document.Fieldable#setBoost(float) boost} and its {@link Similarity#lengthNorm(String,
@@ -970,26 +962,6 @@ public abstract class IndexReader implem
protected abstract void doSetNorm(int doc, String field, byte value)
throws CorruptIndexException, IOException;
- /** Expert: Resets the normalization factor for the named field of the named
- * document.
- *
- * @see #norms(String)
- * @see Similarity#decodeNormValue(byte)
- *
- * @throws StaleReaderException if the index has changed
- * since this reader was opened
- * @throws CorruptIndexException if the index is corrupt
- * @throws LockObtainFailedException if another writer
- * has this index open (<code>write.lock</code> could not
- * be obtained)
- * @throws IOException if there is a low-level IO error
- */
- public void setNorm(int doc, String field, float value)
- throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
- ensureOpen();
- setNorm(doc, field, Similarity.getDefault().encodeNormValue(value));
- }
-
/** Flex API: returns {@link Fields} for this reader.
* This method may return null if the reader has no
* postings.
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiNorms.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiNorms.java Fri Jan 14 13:57:03 2011
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.ReaderUtil;
/**
@@ -61,26 +60,24 @@ public class MultiNorms {
ReaderUtil.gatherSubReaders(leaves, r);
int end = 0;
for (IndexReader leaf : leaves) {
+ Fields fields = leaf.fields();
+ boolean hasField = (fields != null && fields.terms(field) != null);
+
int start = end;
- leaf.norms(field, norms, start);
+ byte leafNorms[] = leaf.norms(field);
+ if (leafNorms == null) {
+ if (hasField) { // omitted norms
+ return null;
+ }
+ // doesn't have field, fill bytes
+ leafNorms = new byte[leaf.maxDoc()];
+ Arrays.fill(leafNorms, (byte) 0);
+ }
+
+ System.arraycopy(leafNorms, 0, norms, start, leafNorms.length);
end += leaf.maxDoc();
}
return norms;
}
}
-
- /**
- * Warning: this is heavy! Do not use in a loop, or implement norms()
- * in your own reader with this (you should likely cache the result).
- */
- public static void norms(IndexReader r, String field, byte[] bytes, int offset)
- throws IOException {
- // TODO: optimize more maybe
- byte[] norms = norms(r, field);
- if (norms == null) {
- Arrays.fill(bytes, offset, bytes.length, Similarity.getDefault().encodeNormValue(1.0f));
- } else {
- System.arraycopy(norms, 0, bytes, offset, r.maxDoc());
- }
- }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/MultiReader.java Fri Jan 14 13:57:03 2011
@@ -305,12 +305,6 @@ public class MultiReader extends IndexRe
}
@Override
- public synchronized void norms(String field, byte[] result, int offset)
- throws IOException {
- throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level norms");
- }
-
- @Override
protected void doSetNorm(int n, String field, byte value)
throws CorruptIndexException, IOException {
int i = readerIndex(n); // find segment num
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriter.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriter.java Fri Jan 14 13:57:03 2011
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.ArrayList;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.search.Similarity;
// TODO FI: norms could actually be stored as doc store
@@ -37,7 +36,6 @@ import org.apache.lucene.search.Similari
final class NormsWriter extends InvertedDocEndConsumer {
- private final byte defaultNorm = Similarity.getDefault().encodeNormValue(1.0f);
private FieldInfos fieldInfos;
@Override
public InvertedDocEndConsumerPerThread addThread(DocInverterPerThread docInverterPerThread) {
@@ -62,6 +60,10 @@ final class NormsWriter extends Inverted
final Map<FieldInfo,List<NormsWriterPerField>> byField = new HashMap<FieldInfo,List<NormsWriterPerField>>();
+ if (!fieldInfos.hasNorms()) {
+ return;
+ }
+
// Typically, each thread will have encountered the same
// field. So first we collate by field, ie, all
// per-thread field instances that correspond to the
@@ -137,7 +139,7 @@ final class NormsWriter extends Inverted
// Fill hole
for(;upto<minDocID;upto++)
- normsOut.writeByte(defaultNorm);
+ normsOut.writeByte((byte) 0);
normsOut.writeByte(fields[minLoc].norms[uptos[minLoc]]);
(uptos[minLoc])++;
@@ -155,12 +157,12 @@ final class NormsWriter extends Inverted
// Fill final hole with defaultNorm
for(;upto<state.numDocs;upto++)
- normsOut.writeByte(defaultNorm);
+ normsOut.writeByte((byte) 0);
} else if (fieldInfo.isIndexed && !fieldInfo.omitNorms) {
normCount++;
// Fill entire field with default norm:
for(;upto<state.numDocs;upto++)
- normsOut.writeByte(defaultNorm);
+ normsOut.writeByte((byte) 0);
}
assert 4+normCount*state.numDocs == normsOut.getFilePointer() : ".nrm file size mismatch: expected=" + (4+normCount*state.numDocs) + " actual=" + normsOut.getFilePointer();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java Fri Jan 14 13:57:03 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.search.Similarity;
/** Taps into DocInverter, as an InvertedDocEndConsumer,
* which is called at the end of inverting each field. We
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/ParallelReader.java Fri Jan 14 13:57:03 2011
@@ -21,11 +21,8 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
@@ -439,6 +436,8 @@ public class ParallelReader extends Inde
return bytes;
if (!hasNorms(field))
return null;
+ if (normsCache.containsKey(field)) // cached omitNorms, not missing key
+ return null;
bytes = MultiNorms.norms(reader, field);
normsCache.put(field, bytes);
@@ -446,23 +445,6 @@ public class ParallelReader extends Inde
}
@Override
- public synchronized void norms(String field, byte[] result, int offset)
- throws IOException {
- // TODO: maybe optimize
- ensureOpen();
- IndexReader reader = fieldToReader.get(field);
- if (reader==null)
- return;
-
- byte[] norms = norms(field);
- if (norms == null) {
- Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f));
- } else {
- System.arraycopy(norms, 0, result, offset, maxDoc());
- }
- }
-
- @Override
protected void doSetNorm(int n, String field, byte value)
throws CorruptIndexException, IOException {
IndexReader reader = fieldToReader.get(field);
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Jan 14 13:57:03 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -570,13 +571,6 @@ final class SegmentMerger {
}
private void mergeNorms() throws IOException {
- // get needed buffer size by finding the largest segment
- int bufferSize = 0;
- for (IndexReader reader : readers) {
- bufferSize = Math.max(bufferSize, reader.maxDoc());
- }
-
- byte[] normBuffer = null;
IndexOutput output = null;
try {
for (int i = 0, numFieldInfos = fieldInfos.size(); i < numFieldInfos; i++) {
@@ -586,12 +580,15 @@ final class SegmentMerger {
output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION));
output.writeBytes(NORMS_HEADER,NORMS_HEADER.length);
}
- if (normBuffer == null) {
- normBuffer = new byte[bufferSize];
- }
for (IndexReader reader : readers) {
final int maxDoc = reader.maxDoc();
- reader.norms(fi.name, normBuffer, 0);
+ byte normBuffer[] = reader.norms(fi.name);
+ if (normBuffer == null) {
+ // Can be null if this segment doesn't have
+ // any docs with this field
+ normBuffer = new byte[maxDoc];
+ Arrays.fill(normBuffer, (byte)0);
+ }
if (!reader.hasDeletions()) {
//optimized case for segments without deleted docs
output.writeBytes(normBuffer, maxDoc);
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Jan 14 13:57:03 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -31,7 +30,6 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -991,22 +989,6 @@ public class SegmentReader extends Index
norm.copyOnWrite()[doc] = value; // set the value
}
- /** Read norms into a pre-allocated array. */
- @Override
- public synchronized void norms(String field, byte[] bytes, int offset)
- throws IOException {
-
- ensureOpen();
- Norm norm = norms.get(field);
- if (norm == null) {
- Arrays.fill(bytes, offset, bytes.length, Similarity.getDefault().encodeNormValue(1.0f));
- return;
- }
-
- norm.bytes(bytes, offset, maxDoc());
- }
-
-
private void openNorms(Directory cfsDir, int readBufferSize) throws IOException {
long nextNormSeek = SegmentMerger.NORMS_HEADER.length; //skip header (header unused for now)
int maxDoc = maxDoc();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Fri Jan 14 13:57:03 2011
@@ -18,19 +18,14 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Arrays;
import java.util.HashMap;
-import java.util.List;
-import java.util.ArrayList;
import java.util.Map;
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ReaderUtil; // javadoc
import org.apache.lucene.index.DirectoryReader; // javadoc
import org.apache.lucene.index.MultiReader; // javadoc
-import org.apache.lucene.index.IndexReader.ReaderContext;
/**
* This class forces a composite reader (eg a {@link
@@ -93,23 +88,13 @@ public final class SlowMultiReaderWrappe
return bytes;
if (!hasNorms(field))
return null;
-
+ if (normsCache.containsKey(field)) // cached omitNorms, not missing key
+ return null;
+
bytes = MultiNorms.norms(in, field);
normsCache.put(field, bytes);
return bytes;
}
-
- @Override
- public synchronized void norms(String field, byte[] bytes, int offset) throws IOException {
- // TODO: maybe optimize
- ensureOpen();
- byte[] norms = norms(field);
- if (norms == null) {
- Arrays.fill(bytes, offset, bytes.length, Similarity.getDefault().encodeNormValue(1.0f));
- } else {
- System.arraycopy(norms, 0, bytes, offset, maxDoc());
- }
- }
@Override
public ReaderContext getTopReaderContext() {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Fri Jan 14 13:57:03 2011
@@ -226,7 +226,7 @@ public class BooleanQuery extends Query
for (Iterator<Weight> wIter = weights.iterator(); wIter.hasNext();) {
Weight w = wIter.next();
BooleanClause c = cIter.next();
- if (w.scorer(context, true, true) == null) {
+ if (w.scorer(context, ScorerContext.def().scoreDocsInOrder(true).topScorer(true)) == null) {
if (c.isRequired()) {
fail = true;
Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")");
@@ -288,7 +288,7 @@ public class BooleanQuery extends Query
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer)
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext)
throws IOException {
List<Scorer> required = new ArrayList<Scorer>();
List<Scorer> prohibited = new ArrayList<Scorer>();
@@ -296,7 +296,7 @@ public class BooleanQuery extends Query
Iterator<BooleanClause> cIter = clauses.iterator();
for (Weight w : weights) {
BooleanClause c = cIter.next();
- Scorer subScorer = w.scorer(context, true, false);
+ Scorer subScorer = w.scorer(context, ScorerContext.def());
if (subScorer == null) {
if (c.isRequired()) {
return null;
@@ -311,7 +311,7 @@ public class BooleanQuery extends Query
}
// Check if we can return a BooleanScorer
- if (!scoreDocsInOrder && topScorer && required.size() == 0 && prohibited.size() < 32) {
+ if (!scorerContext.scoreDocsInOrder && scorerContext.topScorer && required.size() == 0 && prohibited.size() < 32) {
return new BooleanScorer(this, disableCoord, similarity, minNrShouldMatch, optional, prohibited, maxCoord);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Fri Jan 14 13:57:03 2011
@@ -133,7 +133,7 @@ public class ConstantScoreQuery extends
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
final DocIdSetIterator disi;
if (filter != null) {
assert query == null;
@@ -144,7 +144,7 @@ public class ConstantScoreQuery extends
} else {
assert query != null && innerWeight != null;
disi =
- innerWeight.scorer(context, scoreDocsInOrder, topScorer);
+ innerWeight.scorer(context, scorerContext);
}
if (disi == null)
return null;
@@ -158,7 +158,7 @@ public class ConstantScoreQuery extends
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- final Scorer cs = scorer(context, true, false);
+ final Scorer cs = scorer(context, ScorerContext.def());
final boolean exists = (cs != null && cs.advance(doc) == doc);
final ComplexExplanation result = new ComplexExplanation();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java Fri Jan 14 13:57:03 2011
@@ -142,12 +142,11 @@ public class DisjunctionMaxQuery extends
/* Create the scorer used to score our associated DisjunctionMaxQuery */
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
- boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
Scorer[] scorers = new Scorer[weights.size()];
int idx = 0;
for (Weight w : weights) {
- Scorer subScorer = w.scorer(context, true, false);
+ Scorer subScorer = w.scorer(context, ScorerContext.def());
if (subScorer != null && subScorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
scorers[idx++] = subScorer;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java Fri Jan 14 13:57:03 2011
@@ -112,13 +112,13 @@ extends Query {
// return a filtering scorer
@Override
- public Scorer scorer(AtomicReaderContext indexReader, boolean scoreDocsInOrder, boolean topScorer)
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scoreContext)
throws IOException {
- final Scorer scorer = weight.scorer(indexReader, true, false);
+ final Scorer scorer = weight.scorer(context, ScorerContext.def());
if (scorer == null) {
return null;
}
- DocIdSet docIdSet = filter.getDocIdSet(indexReader);
+ DocIdSet docIdSet = filter.getDocIdSet(context);
if (docIdSet == null) {
return null;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Fri Jan 14 13:57:03 2011
@@ -36,6 +36,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Weight.ScorerContext;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory; // javadoc
import org.apache.lucene.util.ReaderUtil;
@@ -174,20 +175,34 @@ public class IndexSearcher {
}
}
}
-
- /* Ctor for concurrent sub-searchers searching only on a specific leaf of the given top-reader context
- * - instead of searching over all leaves this searcher only searches a single leaf searcher slice. Hence,
- * for scorer and filter this looks like an ordinary search in the hierarchy such that there is no difference
- * between single and multi-threaded */
- private IndexSearcher(ReaderContext topLevel, AtomicReaderContext leaf) {
+
+ /**
+ * Expert: Creates a searcher from a top-level {@link ReaderContext} with and
+ * executes searches on the given leave slice exclusively instead of searching
+ * over all leaves. This constructor should be used to run one or more leaves
+ * within a single thread. Hence, for scorer and filter this looks like an
+ * ordinary search in the hierarchy such that there is no difference between
+ * single and multi-threaded.
+ *
+ * @lucene.experimental
+ * */
+ public IndexSearcher(ReaderContext topLevel, AtomicReaderContext... leaves) {
+ assert assertLeaves(topLevel, leaves);
readerContext = topLevel;
reader = topLevel.reader;
- leafContexts = new AtomicReaderContext[] {leaf};
+ leafContexts = leaves;
executor = null;
subSearchers = null;
closeReader = false;
}
+ private boolean assertLeaves(ReaderContext topLevel, AtomicReaderContext... leaves) {
+ for (AtomicReaderContext leaf : leaves) {
+ assert ReaderUtil.getTopLevelContext(leaf) == topLevel : "leaf context is not a leaf of the given top-level context";
+ }
+ return true;
+ }
+
/** Return the {@link IndexReader} this searches. */
public IndexReader getIndexReader() {
return reader;
@@ -482,12 +497,13 @@ public class IndexSearcher {
// TODO: should we make this
// threaded...? the Collector could be sync'd?
-
+ ScorerContext scorerContext = ScorerContext.def().scoreDocsInOrder(true).topScorer(true);
// always use single thread:
if (filter == null) {
for (int i = 0; i < leafContexts.length; i++) { // search each subreader
collector.setNextReader(leafContexts[i]);
- Scorer scorer = weight.scorer(leafContexts[i], !collector.acceptsDocsOutOfOrder(), true);
+ scorerContext = scorerContext.scoreDocsInOrder(!collector.acceptsDocsOutOfOrder());
+ Scorer scorer = weight.scorer(leafContexts[i], scorerContext);
if (scorer != null) {
scorer.score(collector);
}
@@ -505,7 +521,7 @@ public class IndexSearcher {
assert filter != null;
- Scorer scorer = weight.scorer(context, true, false);
+ Scorer scorer = weight.scorer(context, ScorerContext.def());
if (scorer == null) {
return;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Fri Jan 14 13:57:03 2011
@@ -127,7 +127,7 @@ public class MatchAllDocsQuery extends Q
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
return new MatchAllScorer(context.reader, similarity, this,
normsField != null ? context.reader.norms(normsField) : null);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Fri Jan 14 13:57:03 2011
@@ -168,7 +168,7 @@ public class MultiPhraseQuery extends Qu
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
if (termArrays.size() == 0) // optimize zero-term case
return null;
final IndexReader reader = context.reader;
@@ -264,7 +264,7 @@ public class MultiPhraseQuery extends Qu
fieldExpl.setDescription("fieldWeight("+getQuery()+" in "+doc+
"), product of:");
- Scorer scorer = scorer(context, true, false);
+ Scorer scorer = scorer(context, ScorerContext.def());
if (scorer == null) {
return new Explanation(0.0f, "no matching docs");
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Jan 14 13:57:03 2011
@@ -175,7 +175,7 @@ public class PhraseQuery extends Query {
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
if (terms.size() == 0) // optimize zero-term case
return null;
final IndexReader reader = context.reader;
@@ -268,7 +268,7 @@ public class PhraseQuery extends Query {
fieldExpl.setDescription("fieldWeight("+field+":"+query+" in "+doc+
"), product of:");
- Scorer scorer = scorer(context, true, false);
+ Scorer scorer = scorer(context, ScorerContext.def());
if (scorer == null) {
return new Explanation(0.0f, "no matching docs");
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java Fri Jan 14 13:57:03 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.search.Weight.ScorerContext;
/**
* Constrains search results to only match those which also match a provided
@@ -55,7 +56,7 @@ public class QueryWrapperFilter extends
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() throws IOException {
- return weight.scorer(privateContext, true, false);
+ return weight.scorer(privateContext, ScorerContext.def());
}
@Override
public boolean isCacheable() { return false; }
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Similarity.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Similarity.java Fri Jan 14 13:57:03 2011
@@ -565,11 +565,6 @@ public abstract class Similarity impleme
}
/** Decodes a normalization factor stored in an index.
- * <p>
- * <b>WARNING: If you override this method, you should change the default
- * Similarity to your implementation with {@link Similarity#setDefault(Similarity)}.
- * Otherwise, your method may not always be called, especially if you omit norms
- * for some fields.</b>
* @see #encodeNormValue(float)
*/
public float decodeNormValue(byte b) {
@@ -662,11 +657,6 @@ public abstract class Similarity impleme
* are rounded down to the largest representable value. Positive values too
* small to represent are rounded up to the smallest positive representable
* value.
- * <p>
- * <b>WARNING: If you override this method, you should change the default
- * Similarity to your implementation with {@link Similarity#setDefault(Similarity)}.
- * Otherwise, your method may not always be called, especially if you omit norms
- * for some fields.</b>
* @see org.apache.lucene.document.Field#setBoost(float)
* @see org.apache.lucene.util.SmallFloat
*/
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java Fri Jan 14 13:57:03 2011
@@ -32,6 +32,7 @@ import org.apache.lucene.index.BulkPosti
import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.ToStringUtils;
/** A Query that matches documents containing a term.
@@ -87,10 +88,10 @@ public class TermQuery extends Query {
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
final String field = term.field();
final IndexReader reader = context.reader;
- assert assertTopReaderContext(termStates, context) : "The top-reader used to create Weight is not the same as the current reader's top-reader";
+ assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight is not the same as the current reader's top-reader";
final TermState state = termStates
.get(context.ord);
if (state == null) { // term is not present in that reader
@@ -126,14 +127,6 @@ public class TermQuery extends Query {
return terms == null || terms.docFreq(bytes) == 0;
}
- private boolean assertTopReaderContext(PerReaderTermState state, ReaderContext context) {
- while(context.parent != null) {
- context = context.parent;
- }
- return state.topReaderContext == context;
- }
-
-
@Override
public Explanation explain(AtomicReaderContext context, int doc)
throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java Fri Jan 14 13:57:03 2011
@@ -49,7 +49,7 @@ import org.apache.lucene.index.IndexRead
* <li>The query normalization factor is passed to {@link #normalize(float)}. At
* this point the weighting is complete.
* <li>A <code>Scorer</code> is constructed by
- * {@link #scorer(AtomicReaderContext,boolean,boolean)}.
+ * {@link #scorer(AtomicReaderContext, ScorerContext)}.
* </ol>
*
* @since 2.9
@@ -89,23 +89,11 @@ public abstract class Weight implements
*
* @param context
* the {@link AtomicReaderContext} for which to return the {@link Scorer}.
- * @param scoreDocsInOrder
- * specifies whether in-order scoring of documents is required. Note
- * that if set to false (i.e., out-of-order scoring is required),
- * this method can return whatever scoring mode it supports, as every
- * in-order scorer is also an out-of-order one. However, an
- * out-of-order scorer may not support {@link Scorer#nextDoc()}
- * and/or {@link Scorer#advance(int)}, therefore it is recommended to
- * request an in-order scorer if use of these methods is required.
- * @param topScorer
- * if true, {@link Scorer#score(Collector)} will be called; if false,
- * {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will
- * be called.
+ * @param scorercontext the {@link ScorerContext} holding the scores context variables
* @return a {@link Scorer} which scores documents in/out-of order.
* @throws IOException
*/
- public abstract Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
- boolean topScorer) throws IOException;
+ public abstract Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException;
/** The sum of squared weights of contained query clauses. */
public abstract float sumOfSquaredWeights() throws IOException;
@@ -123,4 +111,82 @@ public abstract class Weight implements
*/
public boolean scoresDocsOutOfOrder() { return false; }
+ /**
+ * A struct like class encapsulating a scorer's context variables.
+ * ScorerContex is a strictly immutable struct that follows a
+ * <tt>create on modification</tt> pattern. If a context variable changes
+ * through one of the modifiers like {@link #topScorer(boolean)} a new
+ * {@link ScorerContext} instance is creates. If the modifier call doesn't
+ * change the instance the method call has no effect and the same instance is
+ * returned from the modifier.
+ *
+ * @lucene.experimental
+ */
+ public static final class ScorerContext {
+
+ /**
+ * Specifies whether in-order scoring of documents is required. Note that if
+ * set to false (i.e., out-of-order scoring is required), this method can
+ * return whatever scoring mode it supports, as every in-order scorer is
+ * also an out-of-order one. However, an out-of-order scorer may not support
+ * {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)}, therefore it
+ * is recommended to request an in-order scorer if use of these methods is
+ * required.
+ */
+ public final boolean scoreDocsInOrder;
+
+ /**
+ * if <code>true</code>, {@link Scorer#score(Collector)} will be called; if
+ * false, {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will
+ * be called instead.
+ */
+ public final boolean topScorer;
+
+
+ private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false);
+
+ /**
+ * Returns a default {@link ScorerContext} template initialized with:
+ * <ul>
+ * <li>{@link #scoreDocsInOrder} = <code>true</code></li>
+ * <li>{@link #topScorer} = <code>false</code></li>
+ * </ul>
+ */
+ public static ScorerContext def() {
+ return DEFAULT_CONTEXT;
+ }
+
+ private ScorerContext(boolean scoreDocsInOrder, boolean topScorer) {
+ this.scoreDocsInOrder = scoreDocsInOrder;
+ this.topScorer = topScorer;
+ }
+
+ /**
+ * Creates and returns a copy of this context with the given value for
+ * {@link #scoreDocsInOrder} and returns a new instance of
+ * {@link ScorerContext} iff the given value differs from the
+ * {@link #scoreDocsInOrder}. Otherwise, this method has no effect and
+ * returns this instance.
+ */
+ public ScorerContext scoreDocsInOrder(boolean scoreDocsInOrder) {
+ if (this.scoreDocsInOrder == scoreDocsInOrder) {
+ return this;
+ }
+ return new ScorerContext(scoreDocsInOrder, topScorer);
+ }
+
+ /**
+ * Creates and returns a copy of this context with the given value for
+ * {@link #topScorer} and returns a new instance of
+ * {@link ScorerContext} iff the given value differs from the
+ * {@link #topScorer}. Otherwise, this method has no effect and
+ * returns this instance.
+ */
+ public ScorerContext topScorer(boolean topScorer) {
+ if (this.topScorer == topScorer) {
+ return this;
+ }
+ return new ScorerContext(scoreDocsInOrder, topScorer);
+ }
+ }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Fri Jan 14 13:57:03 2011
@@ -240,19 +240,19 @@ public class CustomScoreQuery extends Qu
}
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
// Pass true for "scoresDocsInOrder", because we
// require in-order scoring, even if caller does not,
// since we call advance on the valSrcScorers. Pass
// false for "topScorer" because we will not invoke
// score(Collector) on these scorers:
- Scorer subQueryScorer = subQueryWeight.scorer(context, true, false);
+ Scorer subQueryScorer = subQueryWeight.scorer(context, ScorerContext.def());
if (subQueryScorer == null) {
return null;
}
Scorer[] valSrcScorers = new Scorer[valSrcWeights.length];
for(int i = 0; i < valSrcScorers.length; i++) {
- valSrcScorers[i] = valSrcWeights[i].scorer(context, true, topScorer);
+ valSrcScorers[i] = valSrcWeights[i].scorer(context, scorerContext.scoreDocsInOrder(true));
}
return new CustomScorer(similarity, context.reader, this, subQueryScorer, valSrcScorers);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java Fri Jan 14 13:57:03 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search.functio
import java.io.IOException;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.FieldCache;
/**
@@ -55,8 +56,8 @@ public abstract class FieldCacheSource e
/* (non-Javadoc) @see org.apache.lucene.search.function.ValueSource#getValues(org.apache.lucene.index.IndexReader) */
@Override
- public final DocValues getValues(IndexReader reader) throws IOException {
- return getCachedFieldValues(FieldCache.DEFAULT, field, reader);
+ public final DocValues getValues(AtomicReaderContext context) throws IOException {
+ return getCachedFieldValues(FieldCache.DEFAULT, field, context.reader);
}
/* (non-Javadoc) @see org.apache.lucene.search.function.ValueSource#description() */
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java?rev=1059010&r1=1059009&r2=1059010&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java Fri Jan 14 13:57:03 2011
@@ -20,6 +20,9 @@ package org.apache.lucene.search.functio
import java.io.IOException;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.util.ReaderUtil;
@@ -44,16 +47,16 @@ public final class MultiValueSource exte
}
@Override
- public DocValues getValues(IndexReader reader) throws IOException {
-
- IndexReader[] subReaders = reader.getSequentialSubReaders();
- if (subReaders != null) {
- // This is a composite reader
- return new MultiDocValues(subReaders);
- } else {
+ public DocValues getValues(AtomicReaderContext context) throws IOException {
// Already an atomic reader -- just delegate
- return other.getValues(reader);
+ return other.getValues(context);
+ }
+
+ public DocValues getValues(ReaderContext context) throws IOException {
+ if (context.isAtomic) {
+ return getValues((AtomicReaderContext) context);
}
+ return new MultiDocValues(ReaderUtil.leaves(context));
}
@Override
@@ -78,59 +81,56 @@ public final class MultiValueSource exte
private final class MultiDocValues extends DocValues {
final DocValues[] docValues;
- final int[] docStarts;
+ final AtomicReaderContext[] leaves;
- MultiDocValues(IndexReader[] subReaders) throws IOException {
- docValues = new DocValues[subReaders.length];
- docStarts = new int[subReaders.length];
- int base = 0;
- for(int i=0;i<subReaders.length;i++) {
- docValues[i] = other.getValues(subReaders[i]);
- docStarts[i] = base;
- base += subReaders[i].maxDoc();
+ MultiDocValues(AtomicReaderContext[] leaves) throws IOException {
+ this.leaves = leaves;
+ docValues = new DocValues[leaves.length];
+ for(int i=0;i<leaves.length;i++) {
+ docValues[i] = other.getValues(leaves[i]);
}
}
@Override
public float floatVal(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].floatVal(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].floatVal(doc-leaves[n].docBase);
}
@Override
public int intVal(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].intVal(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].intVal(doc-leaves[n].docBase);
}
@Override
public long longVal(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].longVal(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].longVal(doc-leaves[n].docBase);
}
@Override
public double doubleVal(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].doubleVal(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].doubleVal(doc-leaves[n].docBase);
}
@Override
public String strVal(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].strVal(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].strVal(doc-leaves[n].docBase);
}
@Override
public String toString(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].toString(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].toString(doc-leaves[n].docBase);
}
@Override
public Explanation explain(int doc) {
- final int n = ReaderUtil.subIndex(doc, docStarts);
- return docValues[n].explain(doc-docStarts[n]);
+ final int n = ReaderUtil.subIndex(doc, leaves);
+ return docValues[n].explain(doc-leaves[n].docBase);
}
}
}