You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/01/16 15:57:17 UTC
svn commit: r1232014 [2/2] - in /lucene/dev/trunk: lucene/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/misc/src/java/org/apache/lucene/misc/
lucene/contrib/misc/src/test/org/apache/lucene/misc/
lucene/src/java/org/apac...
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Mon Jan 16 14:57:15 2012
@@ -191,9 +191,18 @@ final class SegmentMerger {
}
private void mergeFieldInfos() throws IOException {
+ mergeDocValuesAndNormsFieldInfos();
+ // write the merged infos
+ FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat()
+ .getFieldInfosWriter();
+ fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
+ }
+
+ public void mergeDocValuesAndNormsFieldInfos() throws IOException {
// mapping from all docvalues fields found to their promoted types
// this is because FieldInfos does not store the valueSize
Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
+ Map<FieldInfo,TypePromoter> normValuesTypes = new HashMap<FieldInfo,TypePromoter>();
for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader;
@@ -205,29 +214,45 @@ final class SegmentMerger {
TypePromoter previous = docValuesTypes.get(merged);
docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
}
+ if (fi.normsPresent()) {
+ TypePromoter previous = normValuesTypes.get(merged);
+ normValuesTypes.put(merged, mergeDocValuesType(previous, reader.normValues(fi.name)));
+ }
}
}
-
+ updatePromoted(normValuesTypes, true);
+ updatePromoted(docValuesTypes, false);
+ }
+
+ protected void updatePromoted(Map<FieldInfo,TypePromoter> infoAndPromoter, boolean norms) {
// update any promoted doc values types:
- for (Map.Entry<FieldInfo,TypePromoter> e : docValuesTypes.entrySet()) {
+ for (Map.Entry<FieldInfo,TypePromoter> e : infoAndPromoter.entrySet()) {
FieldInfo fi = e.getKey();
TypePromoter promoter = e.getValue();
if (promoter == null) {
- fi.resetDocValuesType(null);
+ if (norms) {
+ fi.setNormValueType(null, true);
+ } else {
+ fi.setDocValuesType(null, true);
+ }
} else {
assert promoter != TypePromoter.getIdentityPromoter();
- if (fi.getDocValuesType() != promoter.type()) {
- // reset the type if we got promoted
- fi.resetDocValuesType(promoter.type());
+ if (norms) {
+ if (fi.getNormType() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.setNormValueType(promoter.type(), true);
+ }
+ } else {
+ if (fi.getDocValuesType() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.setDocValuesType(promoter.type(), true);
+ }
}
}
}
-
- // write the merged infos
- FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
}
+
/**
*
* @return The number of documents in all of the readers
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon Jan 16 14:57:15 2012
@@ -198,7 +198,7 @@ public final class SegmentReader extends
public boolean hasNorms(String field) {
ensureOpen();
FieldInfo fi = core.fieldInfos.fieldInfo(field);
- return fi != null && fi.isIndexed && !fi.omitNorms;
+ return fi.normsPresent();
}
/** @lucene.internal */
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Mon Jan 16 14:57:15 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -122,10 +123,11 @@ public class BM25Similarity extends Simi
}
}
+
@Override
- public final byte computeNorm(FieldInvertState state) {
+ public final void computeNorm(FieldInvertState state, Norm norm) {
final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength();
- return encodeNormValue(state.getBoost(), numTerms);
+ norm.setByte(encodeNormValue(state.getBoost(), numTerms));
}
public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Mon Jan 16 14:57:15 2012
@@ -1,6 +1,7 @@
package org.apache.lucene.search.similarities;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.util.BytesRef;
/**
@@ -22,7 +23,7 @@ import org.apache.lucene.util.BytesRef;
/** Expert: Default scoring implementation. */
public class DefaultSimilarity extends TFIDFSimilarity {
-
+
/** Implemented as
* <code>state.getBoost()*lengthNorm(numTerms)</code>, where
* <code>numTerms</code> is {@link FieldInvertState#getLength()} if {@link
@@ -32,13 +33,13 @@ public class DefaultSimilarity extends T
*
* @lucene.experimental */
@Override
- public byte computeNorm(FieldInvertState state) {
+ public void computeNorm(FieldInvertState state, Norm norm) {
final int numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
+ norm.setByte(encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)))));
}
/** Implemented as <code>sqrt(freq)</code>. */
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java Mon Jan 16 14:57:15 2012
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -40,8 +41,8 @@ public class MultiSimilarity extends Sim
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return sims[0].computeNorm(state);
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ sims[0].computeNorm(state, norm);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java Mon Jan 16 14:57:15 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.document.DocVal
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader; // javadoc
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.Terms; // javadoc
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
@@ -36,7 +37,6 @@ import org.apache.lucene.search.TermStat
import org.apache.lucene.search.spans.SpanQuery; // javadoc
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SmallFloat; // javadoc
-import org.apache.lucene.util.TermContext;
/**
@@ -55,8 +55,8 @@ import org.apache.lucene.util.TermContex
* <a href="#querytime">query-time</a>.
* <p>
* <a name="indextime"/>
- * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing
- * the Similarity implementation to return a per-document byte for the field that will
+ * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState, Norm)}, allowing
+ * the Similarity implementation to set a per-document value for the field that will
* be later accessible via {@link IndexReader#normValues(String)}. Lucene makes no assumption
* about what is in this byte, but it is most useful for encoding length normalization
* information.
@@ -109,23 +109,24 @@ import org.apache.lucene.util.TermContex
* @lucene.experimental
*/
public abstract class Similarity {
+
/**
* Computes the normalization value for a field, given the accumulated
* state of term processing for this field (see {@link FieldInvertState}).
*
- * <p>Implementations should calculate a byte value based on the field
- * state and then return that value.
+ * <p>Implementations should calculate a norm value based on the field
+ * state and set that value to the given {@link Norm}.
*
* <p>Matches in longer fields are less precise, so implementations of this
- * method usually return smaller values when <code>state.getLength()</code> is large,
+ * method usually set smaller values when <code>state.getLength()</code> is large,
* and larger values when <code>state.getLength()</code> is small.
*
* @lucene.experimental
*
* @param state current processing state for this field
- * @return the calculated byte norm
+ * @param norm holds the computed norm value when this method returns
*/
- public abstract byte computeNorm(FieldInvertState state);
+ public abstract void computeNorm(FieldInvertState state, Norm norm);
/**
* Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Mon Jan 16 14:57:15 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -234,13 +235,13 @@ public abstract class SimilarityBase ext
/** Encodes the document length in the same way as {@link TFIDFSimilarity}. */
@Override
- public byte computeNorm(FieldInvertState state) {
+ public void computeNorm(FieldInvertState state, Norm norm) {
final float numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength() / state.getBoost();
- return encodeNormValue(state.getBoost(), numTerms);
+ norm.setByte(encodeNormValue(state.getBoost(), numTerms));
}
/** Decodes a normalization factor (document length) stored in an index.
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Mon Jan 16 14:57:15 2012
@@ -22,14 +22,12 @@ import java.io.IOException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.SmallFloat;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/package.html?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/package.html (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/package.html Mon Jan 16 14:57:15 2012
@@ -155,7 +155,7 @@ subclassing the Similarity, one can simp
matching term occurs. In these
cases people have overridden Similarity to return 1 from the tf() method.</p></li>
<li><p>Changing Length Normalization — By overriding
- {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state)},
+ {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state, Norm)},
it is possible to discount how the length of a field contributes
to a score. In {@link org.apache.lucene.search.similarities.DefaultSimilarity},
lengthNorm = 1 / (numTerms in field)^0.5, but if one changes this to be
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java Mon Jan 16 14:57:15 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.codecs.preflex
* limitations under the License.
*/
+import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
@@ -29,6 +30,7 @@ import org.apache.lucene.util.LuceneTest
public class PreFlexRWCodec extends Lucene3xCodec {
private final PostingsFormat postings = new PreFlexRWPostingsFormat();
private final NormsFormat norms = new PreFlexRWNormsFormat();
+ private final FieldInfosFormat fieldInfos = new PreFlexRWFieldInfosFormat();
@Override
public PostingsFormat postingsFormat() {
@@ -47,4 +49,13 @@ public class PreFlexRWCodec extends Luce
return super.normsFormat();
}
}
+
+ @Override
+ public FieldInfosFormat fieldInfosFormat() {
+ if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ return fieldInfos;
+ } else {
+ return super.fieldInfosFormat();
+ }
+ }
}
Added: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosFormat.java?rev=1232014&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosFormat.java (added)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosFormat.java Mon Jan 16 14:57:15 2012
@@ -0,0 +1,41 @@
+package org.apache.lucene.codecs.preflexrw;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.lucene.codecs.FieldInfosReader;
+import org.apache.lucene.codecs.FieldInfosWriter;
+import org.apache.lucene.codecs.lucene3x.Lucene3xFieldInfosFormat;
+
+/**
+ *
+ * @lucene.internal
+ * @lucene.experimental
+ */
+public class PreFlexRWFieldInfosFormat extends Lucene3xFieldInfosFormat {
+
+ @Override
+ public FieldInfosReader getFieldInfosReader() throws IOException {
+ return new PreFlexRWFieldInfosReader();
+ }
+
+ @Override
+ public FieldInfosWriter getFieldInfosWriter() throws IOException {
+ return new PreFlexRWFieldInfosWriter();
+ }
+}
Copied: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosReader.java (from r1229530, lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosReader.java?p2=lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosReader.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java&r1=1229530&r2=1232014&rev=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosReader.java Mon Jan 16 14:57:15 2012
@@ -1,22 +1,4 @@
-package org.apache.lucene.codecs.lucene40;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.lucene.codecs.FieldInfosReader;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexFormatTooNewException;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-
+package org.apache.lucene.codecs.preflexrw;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -33,17 +15,33 @@ import org.apache.lucene.store.IndexInpu
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.codecs.FieldInfosReader;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexFormatTooNewException;
+import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
/**
+ * @lucene.internal
* @lucene.experimental
*/
-public class Lucene40FieldInfosReader extends FieldInfosReader {
-
- static final int FORMAT_MINIMUM = Lucene40FieldInfosWriter.FORMAT_START;
+public class PreFlexRWFieldInfosReader extends FieldInfosReader {
+ static final int FORMAT_MINIMUM = PreFlexRWFieldInfosWriter.FORMAT_START;
@Override
public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosWriter.FIELD_INFOS_EXTENSION);
+ final String fileName = IndexFileNames.segmentFileName(segmentName, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION);
IndexInput input = directory.openInput(fileName, iocontext);
boolean hasVectors = false;
@@ -54,10 +52,10 @@ public class Lucene40FieldInfosReader ex
final int format = input.readVInt();
if (format > FORMAT_MINIMUM) {
- throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, Lucene40FieldInfosWriter.FORMAT_CURRENT);
+ throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT);
}
- if (format < Lucene40FieldInfosWriter.FORMAT_CURRENT) {
- throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, Lucene40FieldInfosWriter.FORMAT_CURRENT);
+ if (format < PreFlexRWFieldInfosWriter.FORMAT_CURRENT && format != PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW) {
+ throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, PreFlexRWFieldInfosWriter.FORMAT_CURRENT);
}
final int size = input.readVInt(); //read in the size
@@ -65,17 +63,17 @@ public class Lucene40FieldInfosReader ex
for (int i = 0; i < size; i++) {
String name = input.readString();
- final int fieldNumber = format <= Lucene40FieldInfosWriter.FORMAT_FLEX? input.readInt():i;
+ final int fieldNumber = format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW ? input.readInt() : i;
byte bits = input.readByte();
- boolean isIndexed = (bits & Lucene40FieldInfosWriter.IS_INDEXED) != 0;
- boolean storeTermVector = (bits & Lucene40FieldInfosWriter.STORE_TERMVECTOR) != 0;
- boolean omitNorms = (bits & Lucene40FieldInfosWriter.OMIT_NORMS) != 0;
- boolean storePayloads = (bits & Lucene40FieldInfosWriter.STORE_PAYLOADS) != 0;
+ boolean isIndexed = (bits & PreFlexRWFieldInfosWriter.IS_INDEXED) != 0;
+ boolean storeTermVector = (bits & PreFlexRWFieldInfosWriter.STORE_TERMVECTOR) != 0;
+ boolean omitNorms = (bits & PreFlexRWFieldInfosWriter.OMIT_NORMS) != 0;
+ boolean storePayloads = (bits & PreFlexRWFieldInfosWriter.STORE_PAYLOADS) != 0;
final IndexOptions indexOptions;
- if ((bits & Lucene40FieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
+ if ((bits & PreFlexRWFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
indexOptions = IndexOptions.DOCS_ONLY;
- } else if ((bits & Lucene40FieldInfosWriter.OMIT_POSITIONS) != 0) {
- if (format <= Lucene40FieldInfosWriter.FORMAT_OMIT_POSITIONS) {
+ } else if ((bits & PreFlexRWFieldInfosWriter.OMIT_POSITIONS) != 0) {
+ if (format <= PreFlexRWFieldInfosWriter.FORMAT_OMIT_POSITIONS) {
indexOptions = IndexOptions.DOCS_AND_FREQS;
} else {
throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")");
@@ -93,65 +91,20 @@ public class Lucene40FieldInfosReader ex
hasVectors |= storeTermVector;
hasProx |= isIndexed && indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY;
- DocValues.Type docValuesType = null;
- if (format <= Lucene40FieldInfosWriter.FORMAT_FLEX) {
- final byte b = input.readByte();
- switch(b) {
- case 0:
- docValuesType = null;
- break;
- case 1:
- docValuesType = DocValues.Type.VAR_INTS;
- break;
- case 2:
- docValuesType = DocValues.Type.FLOAT_32;
- break;
- case 3:
- docValuesType = DocValues.Type.FLOAT_64;
- break;
- case 4:
- docValuesType = DocValues.Type.BYTES_FIXED_STRAIGHT;
- break;
- case 5:
- docValuesType = DocValues.Type.BYTES_FIXED_DEREF;
- break;
- case 6:
- docValuesType = DocValues.Type.BYTES_VAR_STRAIGHT;
- break;
- case 7:
- docValuesType = DocValues.Type.BYTES_VAR_DEREF;
- break;
- case 8:
- docValuesType = DocValues.Type.FIXED_INTS_16;
- break;
- case 9:
- docValuesType = DocValues.Type.FIXED_INTS_32;
- break;
- case 10:
- docValuesType = DocValues.Type.FIXED_INTS_64;
- break;
- case 11:
- docValuesType = DocValues.Type.FIXED_INTS_8;
- break;
- case 12:
- docValuesType = DocValues.Type.BYTES_FIXED_SORTED;
- break;
- case 13:
- docValuesType = DocValues.Type.BYTES_VAR_SORTED;
- break;
- default:
- throw new IllegalStateException("unhandled indexValues type " + b);
- }
+ Type normType = isIndexed && !omitNorms ? Type.FIXED_INTS_8 : null;
+ if (format == PreFlexRWFieldInfosWriter.FORMAT_PREFLEX_RW && normType != null) {
+ // RW can have norms but doesn't write them
+ normType = input.readByte() != 0 ? Type.FIXED_INTS_8 : null;
}
+
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType);
+ omitNorms, storePayloads, indexOptions, null, normType);
}
if (input.getFilePointer() != input.length()) {
throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
}
-
return new FieldInfos(infos, hasFreq, hasProx, hasVectors);
} finally {
input.close();
@@ -159,6 +112,6 @@ public class Lucene40FieldInfosReader ex
}
public static void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
- files.add(IndexFileNames.segmentFileName(info.name, "", Lucene40FieldInfosWriter.FIELD_INFOS_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexRWFieldInfosWriter.FIELD_INFOS_EXTENSION));
}
}
Copied: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosWriter.java (from r1229530, lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosWriter.java?p2=lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosWriter.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java&r1=1229530&r2=1232014&rev=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldInfosWriter.java Mon Jan 16 14:57:15 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.codecs.lucene40;
+package org.apache.lucene.codecs.preflexrw;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -28,9 +28,11 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexOutput;
/**
+ * @lucene.internal
* @lucene.experimental
*/
-public class Lucene40FieldInfosWriter extends FieldInfosWriter {
+public class PreFlexRWFieldInfosWriter extends FieldInfosWriter {
+ // TODO move to test-framework preflex RW?
/** Extension of field infos */
static final String FIELD_INFOS_EXTENSION = "fnm";
@@ -39,11 +41,11 @@ public class Lucene40FieldInfosWriter ex
static final int FORMAT_START = -2;
// First used in 3.4: omit only positional information
static final int FORMAT_OMIT_POSITIONS = -3;
- // per-field codec support, records index values for fields
- static final int FORMAT_FLEX = -4;
+
+ static final int FORMAT_PREFLEX_RW = Integer.MIN_VALUE;
// whenever you add a new format, make it 1 smaller (negative version logic)!
- static final int FORMAT_CURRENT = FORMAT_FLEX;
+ static final int FORMAT_CURRENT = FORMAT_OMIT_POSITIONS;
static final byte IS_INDEXED = 0x1;
static final byte STORE_TERMVECTOR = 0x2;
@@ -57,7 +59,7 @@ public class Lucene40FieldInfosWriter ex
final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
IndexOutput output = directory.createOutput(fileName, context);
try {
- output.writeVInt(FORMAT_CURRENT);
+ output.writeVInt(FORMAT_PREFLEX_RW);
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
assert fi.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.storePayloads;
@@ -72,59 +74,18 @@ public class Lucene40FieldInfosWriter ex
bits |= OMIT_POSITIONS;
}
output.writeString(fi.name);
+ /*
+ * we need to write the field number since IW tries
+ * to stabelize the field numbers across segments so the
+ * FI ordinal is not necessarily equivalent to the field number
+ */
output.writeInt(fi.number);
output.writeByte(bits);
-
- final byte b;
-
- if (!fi.hasDocValues()) {
- b = 0;
- } else {
- switch(fi.getDocValuesType()) {
- case VAR_INTS:
- b = 1;
- break;
- case FLOAT_32:
- b = 2;
- break;
- case FLOAT_64:
- b = 3;
- break;
- case BYTES_FIXED_STRAIGHT:
- b = 4;
- break;
- case BYTES_FIXED_DEREF:
- b = 5;
- break;
- case BYTES_VAR_STRAIGHT:
- b = 6;
- break;
- case BYTES_VAR_DEREF:
- b = 7;
- break;
- case FIXED_INTS_16:
- b = 8;
- break;
- case FIXED_INTS_32:
- b = 9;
- break;
- case FIXED_INTS_64:
- b = 10;
- break;
- case FIXED_INTS_8:
- b = 11;
- break;
- case BYTES_FIXED_SORTED:
- b = 12;
- break;
- case BYTES_VAR_SORTED:
- b = 13;
- break;
- default:
- throw new IllegalStateException("unhandled indexValues type " + fi.getDocValuesType());
- }
+ if (fi.isIndexed && !fi.omitNorms) {
+ // to allow null norm types we need to indicate if norms are written
+ // only in RW case
+ output.writeByte((byte) (fi.getNormType() == null ? 0 : 1));
}
- output.writeByte(b);
}
} finally {
output.close();
Copied: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldsWriter.java (from r1231643, lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldsWriter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldsWriter.java?p2=lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldsWriter.java&p1=lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldsWriter.java&r1=1231643&r2=1232014&rev=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWFieldsWriter.java Mon Jan 16 14:57:15 2012
@@ -36,7 +36,7 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-class PreFlexFieldsWriter extends FieldsConsumer {
+class PreFlexRWFieldsWriter extends FieldsConsumer {
private final TermInfosWriter termsOut;
private final IndexOutput freqOut;
@@ -44,7 +44,7 @@ class PreFlexFieldsWriter extends Fields
private final Lucene40SkipListWriter skipListWriter;
private final int totalNumDocs;
- public PreFlexFieldsWriter(SegmentWriteState state) throws IOException {
+ public PreFlexRWFieldsWriter(SegmentWriteState state) throws IOException {
termsOut = new TermInfosWriter(state.directory,
state.segmentName,
state.fieldInfos,
@@ -88,6 +88,9 @@ class PreFlexFieldsWriter extends Fields
@Override
public TermsConsumer addField(FieldInfo field) throws IOException {
assert field.number != -1;
+ if (field.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
+ throw new IllegalArgumentException("this codec cannot index offsets");
+ }
//System.out.println("w field=" + field.name + " storePayload=" + field.storePayloads + " number=" + field.number);
return new PreFlexTermsWriter(field);
}
@@ -157,9 +160,10 @@ class PreFlexFieldsWriter extends Fields
}
@Override
- public void addPosition(int position, BytesRef payload) throws IOException {
+ public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException {
assert proxOut != null;
-
+ assert startOffset == -1;
+ assert endOffset == -1;
//System.out.println(" w pos=" + position + " payl=" + payload);
final int delta = position - lastPosition;
lastPosition = position;
Copied: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsConsumer.java (from r1231643, lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexNormsConsumer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsConsumer.java?p2=lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsConsumer.java&p1=lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexNormsConsumer.java&r1=1231643&r2=1232014&rev=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexNormsConsumer.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsConsumer.java Mon Jan 16 14:57:15 2012
@@ -22,26 +22,25 @@ import java.util.Arrays;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.PerDocConsumer;
-import org.apache.lucene.index.DocValue;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/**
* Writes and Merges Lucene 3.x norms format
* @lucene.experimental
*/
-class PreFlexNormsConsumer extends PerDocConsumer {
+class PreFlexRWNormsConsumer extends PerDocConsumer {
/** norms header placeholder */
private static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1};
@@ -62,7 +61,7 @@ class PreFlexNormsConsumer extends PerDo
private NormsWriter writer;
- public PreFlexNormsConsumer(Directory directory, String segment, IOContext context){
+ public PreFlexRWNormsConsumer(Directory directory, String segment, IOContext context){
this.directory = directory;
this.segment = segment;
this.context = context;
@@ -79,10 +78,23 @@ class PreFlexNormsConsumer extends PerDo
writer.finish();
}
}
+
+ @Override
+ protected boolean canMerge(FieldInfo info) {
+ return info.normsPresent();
+ }
+
+ @Override
+ protected Type getDocValuesType(FieldInfo info) {
+ return info.getNormType();
+ }
@Override
public DocValuesConsumer addValuesField(Type type, FieldInfo fieldInfo)
throws IOException {
+ if (type != Type.FIXED_INTS_8) {
+ throw new UnsupportedOperationException("Codec only supports single byte norm values. Type give: " + type);
+ }
return new Lucene3xNormsDocValuesConsumer(fieldInfo);
}
@@ -133,11 +145,11 @@ class PreFlexNormsConsumer extends PerDo
}
@Override
- public void add(int docID, DocValue docValue) throws IOException {
- add(docID, docValue.getBytes());
+ public void add(int docID, IndexableField docValue) throws IOException {
+ add(docID, docValue.numericValue().longValue());
}
- protected void add(int docID, BytesRef value) throws IOException {
+ protected void add(int docID, long value) {
if (docIDs.length <= upto) {
assert docIDs.length == upto;
docIDs = ArrayUtil.grow(docIDs, 1 + upto);
@@ -146,8 +158,7 @@ class PreFlexNormsConsumer extends PerDo
assert norms.length == upto;
norms = ArrayUtil.grow(norms, 1 + upto);
}
- assert value.length == 1;
- norms[upto] = value.bytes[value.offset];
+ norms[upto] = (byte) value;
docIDs[upto] = docID;
upto++;
@@ -217,7 +228,7 @@ class PreFlexNormsConsumer extends PerDo
public void merge(MergeState mergeState) throws IOException {
int numMergedDocs = 0;
for (FieldInfo fi : mergeState.fieldInfos) {
- if (fi.isIndexed && !fi.omitNorms) {
+ if (fi.normsPresent()) {
startField(fi);
int numMergedDocsForField = 0;
for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) {
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsFormat.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsFormat.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWNormsFormat.java Mon Jan 16 14:57:15 2012
@@ -21,11 +21,15 @@ import org.apache.lucene.codecs.PerDocCo
import org.apache.lucene.codecs.lucene3x.Lucene3xNormsFormat;
import org.apache.lucene.index.PerDocWriteState;
+/**
+ * @lucene.internal
+ * @lucene.experimental
+ */
public class PreFlexRWNormsFormat extends Lucene3xNormsFormat {
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new PreFlexNormsConsumer(state.directory, state.segmentName, state.context);
+ return new PreFlexRWNormsConsumer(state.directory, state.segmentName, state.context);
}
}
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWPostingsFormat.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWPostingsFormat.java Mon Jan 16 14:57:15 2012
@@ -41,7 +41,7 @@ public class PreFlexRWPostingsFormat ext
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return new PreFlexFieldsWriter(state);
+ return new PreFlexRWFieldsWriter(state);
}
@Override
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCustomNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCustomNorms.java?rev=1232014&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCustomNorms.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCustomNorms.java Mon Jan 16 14:57:15 2012
@@ -0,0 +1,206 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Before;
+
+/**
+ *
+ */
+public class TestCustomNorms extends LuceneTestCase {
+ final String floatTestField = "normsTestFloat";
+ final String exceptionTestField = "normsTestExcp";
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeFalse("cannot work with preflex codec", Codec.getDefault().getName()
+ .equals("Lucene3x"));
+ assumeFalse("cannot work with simple text codec", Codec.getDefault()
+ .getName().equals("SimpleText"));
+
+ }
+
+ public void testFloatNorms() throws IOException {
+
+ MockDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false); // can't set sim to checkindex yet
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ SimilarityProvider provider = new MySimProvider();
+ config.setSimilarityProvider(provider);
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+ final LineFileDocs docs = new LineFileDocs(random);
+ int num = atLeast(100);
+ for (int i = 0; i < num; i++) {
+ Document doc = docs.nextDoc();
+ float nextFloat = random.nextFloat();
+ Field f = new Field(floatTestField, "" + nextFloat, TextField.TYPE_STORED);
+ f.setBoost(nextFloat);
+
+ doc.add(f);
+ writer.addDocument(doc);
+ doc.removeField(floatTestField);
+ if (rarely()) {
+ writer.commit();
+ }
+ }
+ writer.commit();
+ writer.close();
+ IndexReader open = new SlowMultiReaderWrapper(IndexReader.open(dir));
+ DocValues normValues = open.normValues(floatTestField);
+ assertNotNull(normValues);
+ Source source = normValues.getSource();
+ assertTrue(source.hasArray());
+ assertEquals(Type.FLOAT_32, normValues.type());
+ float[] norms = (float[]) source.getArray();
+ for (int i = 0; i < open.maxDoc(); i++) {
+ Document document = open.document(i);
+ float expected = Float.parseFloat(document.get(floatTestField));
+ assertEquals(expected, norms[i], 0.0f);
+ }
+ open.close();
+ dir.close();
+ }
+
+ public void testExceptionOnRandomType() throws IOException {
+ MockDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false); // can't set sim to checkindex yet
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ SimilarityProvider provider = new MySimProvider();
+ config.setSimilarityProvider(provider);
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+ final LineFileDocs docs = new LineFileDocs(random);
+ int num = atLeast(100);
+ try {
+ for (int i = 0; i < num; i++) {
+ Document doc = docs.nextDoc();
+ float nextFloat = random.nextFloat();
+ Field f = new Field(exceptionTestField, "" + nextFloat,
+ TextField.TYPE_STORED);
+ f.setBoost(nextFloat);
+
+ doc.add(f);
+ writer.addDocument(doc);
+ doc.removeField(exceptionTestField);
+ if (rarely()) {
+ writer.commit();
+ }
+ }
+ fail("expected exception - incompatible types");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ writer.commit();
+ writer.close();
+ dir.close();
+
+ }
+
+ public class MySimProvider implements SimilarityProvider {
+ SimilarityProvider delegate = new DefaultSimilarityProvider();
+
+ @Override
+ public float queryNorm(float sumOfSquaredWeights) {
+
+ return delegate.queryNorm(sumOfSquaredWeights);
+ }
+
+ @Override
+ public Similarity get(String field) {
+ if (floatTestField.equals(field)) {
+ return new FloatEncodingBoostSimilarity();
+ } else if (exceptionTestField.equals(field)) {
+ return new RandomTypeSimilarity(random);
+ } else {
+ return delegate.get(field);
+ }
+ }
+
+ @Override
+ public float coord(int overlap, int maxOverlap) {
+ return delegate.coord(overlap, maxOverlap);
+ }
+ }
+
+ public static class FloatEncodingBoostSimilarity extends DefaultSimilarity {
+
+ @Override
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ float boost = state.getBoost();
+ norm.setFloat(boost);
+ }
+ }
+
+ public static class RandomTypeSimilarity extends DefaultSimilarity {
+
+ private final Random random;
+
+ public RandomTypeSimilarity(Random random) {
+ this.random = random;
+ }
+
+ @Override
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ float boost = state.getBoost();
+ int nextInt = random.nextInt(10);
+ switch (nextInt) {
+ case 0:
+ norm.setDouble((double) boost);
+ break;
+ case 1:
+ norm.setFloat(boost);
+ break;
+ case 2:
+ norm.setLong((long) boost);
+ break;
+ case 3:
+ norm.setBytes(new BytesRef(new byte[6]));
+ break;
+ case 4:
+ norm.setInt((int) boost);
+ break;
+ case 5:
+ norm.setShort((short) boost);
+ break;
+ default:
+ norm.setByte((byte) boost);
+ }
+
+ }
+ }
+
+}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Mon Jan 16 14:57:15 2012
@@ -141,7 +141,7 @@ public class TestFieldInfos extends Luce
try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(),
- random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
+ random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null);
fail("instance should be read only");
} catch (IllegalStateException e) {
// expected
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Mon Jan 16 14:57:15 2012
@@ -117,8 +117,8 @@ public class TestMaxTermFrequency extend
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue((float) state.getMaxTermFrequency());
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue((float) state.getMaxTermFrequency()));
}
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java Mon Jan 16 14:57:15 2012
@@ -18,20 +18,20 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -39,7 +39,8 @@ import org.apache.lucene.util.LuceneTest
* separate norms, addDocument, addIndexes, forceMerge.
*/
public class TestNorms extends LuceneTestCase {
-
+ final String byteTestField = "normsTestByte";
+
class CustomNormEncodingSimilarity extends DefaultSimilarity {
@Override
public byte encodeNormValue(float f) {
@@ -52,8 +53,8 @@ public class TestNorms extends LuceneTes
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue((float) state.getLength());
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue((float) state.getLength()));
}
}
@@ -93,4 +94,160 @@ public class TestNorms extends LuceneTes
reader.close();
dir.close();
}
+
+ public void testMaxByteNorms() throws IOException {
+ Directory dir = newDirectory();
+ buildIndex(dir, true);
+ IndexReader open = new SlowMultiReaderWrapper(IndexReader.open(dir));
+ DocValues normValues = open.normValues(byteTestField);
+ assertNotNull(normValues);
+ Source source = normValues.getSource();
+ assertTrue(source.hasArray());
+ assertEquals(Type.FIXED_INTS_8, normValues.type());
+ byte[] norms = (byte[]) source.getArray();
+ for (int i = 0; i < open.maxDoc(); i++) {
+ Document document = open.document(i);
+ int expected = Integer.parseInt(document.get(byteTestField));
+ assertEquals((byte)expected, norms[i]);
+ }
+ open.close();
+ dir.close();
+ }
+
+ /**
+ * this test randomly creates segments with or without norms but not omitting
+ * norms. The similarity used doesn't write a norm value if writeNorms = false is
+ * passed. This differs from omitNorm since norms are simply not written for this segment
+ * while merging fills in default values based on the Norm {@link Type}
+ */
+ public void testNormsNotPresent() throws IOException {
+ Directory dir = newDirectory();
+ boolean firstWriteNorm = random.nextBoolean();
+ buildIndex(dir, firstWriteNorm);
+
+ Directory otherDir = newDirectory();
+ boolean secondWriteNorm = random.nextBoolean();
+ buildIndex(otherDir, secondWriteNorm);
+
+ IndexReader reader = new SlowMultiReaderWrapper(IndexReader.open(otherDir));
+ FieldInfos fieldInfos = reader.getFieldInfos();
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(byteTestField);
+ assertFalse(fieldInfo.omitNorms);
+ assertTrue(fieldInfo.isIndexed);
+ if (secondWriteNorm) {
+ assertTrue(fieldInfo.normsPresent());
+ } else {
+ assertFalse(fieldInfo.normsPresent());
+ }
+
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+ writer.addIndexes(reader);
+ IndexReader mergedReader = new SlowMultiReaderWrapper(writer.getReader());
+ if (!firstWriteNorm && !secondWriteNorm) {
+ DocValues normValues = mergedReader.normValues(byteTestField);
+ assertNull(normValues);
+ FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
+ assertFalse(fi.omitNorms);
+ assertTrue(fi.isIndexed);
+ assertFalse(fi.normsPresent());
+ } else {
+ FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
+ assertFalse(fi.omitNorms);
+ assertTrue(fi.isIndexed);
+ assertTrue(fi.normsPresent());
+
+ DocValues normValues = mergedReader.normValues(byteTestField);
+ assertNotNull(normValues);
+ Source source = normValues.getSource();
+ assertTrue(source.hasArray());
+ assertEquals(Type.FIXED_INTS_8, normValues.type());
+ byte[] norms = (byte[]) source.getArray();
+ for (int i = 0; i < mergedReader.maxDoc(); i++) {
+ Document document = mergedReader.document(i);
+ int expected = Integer.parseInt(document.get(byteTestField));
+ assertEquals((byte) expected, norms[i]);
+ }
+ }
+ mergedReader.close();
+ reader.close();
+
+ writer.close();
+ dir.close();
+ otherDir.close();
+ }
+
+ public void buildIndex(Directory dir, boolean writeNorms) throws IOException,
+ CorruptIndexException {
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ SimilarityProvider provider = new MySimProvider(writeNorms);
+ config.setSimilarityProvider(provider);
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+ final LineFileDocs docs = new LineFileDocs(random);
+ int num = atLeast(100);
+ for (int i = 0; i < num; i++) {
+ Document doc = docs.nextDoc();
+ int boost = writeNorms ? 1 + random.nextInt(255) : 0;
+ Field f = new Field(byteTestField, "" + boost,
+ TextField.TYPE_STORED);
+ f.setBoost(boost);
+ doc.add(f);
+ writer.addDocument(doc);
+ doc.removeField(byteTestField);
+ if (rarely()) {
+ writer.commit();
+ }
+ }
+ writer.commit();
+ writer.close();
+ }
+
+
+ public class MySimProvider implements SimilarityProvider {
+ SimilarityProvider delegate = new DefaultSimilarityProvider();
+ private boolean writeNorms;
+ public MySimProvider(boolean writeNorms) {
+ this.writeNorms = writeNorms;
+ }
+ @Override
+ public float queryNorm(float sumOfSquaredWeights) {
+
+ return delegate.queryNorm(sumOfSquaredWeights);
+ }
+
+ @Override
+ public Similarity get(String field) {
+ if (byteTestField.equals(field)) {
+ return new ByteEncodingBoostSimilarity(writeNorms);
+ } else {
+ return delegate.get(field);
+ }
+ }
+
+ @Override
+ public float coord(int overlap, int maxOverlap) {
+ return delegate.coord(overlap, maxOverlap);
+ }
+ }
+
+
+ public static class ByteEncodingBoostSimilarity extends DefaultSimilarity {
+
+ private boolean writeNorms;
+
+ public ByteEncodingBoostSimilarity(boolean writeNorms) {
+ this.writeNorms = writeNorms;
+ }
+
+ @Override
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ if (writeNorms) {
+ int boost = (int) state.getBoost();
+ norm.setByte((byte) (0xFF & boost));
+ }
+ }
+ }
+
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Mon Jan 16 14:57:15 2012
@@ -44,8 +44,7 @@ public class TestOmitTf extends LuceneTe
public float coord(int overlap, int maxOverlap) { return 1.0f; }
public Similarity get(String field) {
return new TFIDFSimilarity() {
-
- @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); }
+ @Override public void computeNorm(FieldInvertState state, Norm norm) { norm.setByte(encodeNormValue(state.getBoost())); }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java Mon Jan 16 14:57:15 2012
@@ -102,8 +102,8 @@ public class TestUniqueTermCount extends
class TestSimilarity extends DefaultSimilarity {
@Override
- public byte computeNorm(FieldInvertState state) {
- return (byte) state.getUniqueTermCount();
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte((byte) state.getUniqueTermCount());
}
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Mon Jan 16 14:57:15 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.util.Bits;
@@ -261,7 +262,7 @@ final class JustCompileSearch {
}
@Override
- public byte computeNorm(FieldInvertState state) {
+ public void computeNorm(FieldInvertState state, Norm norm) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Mon Jan 16 14:57:15 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.document.FieldT
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.RandomIndexWriter;
@@ -66,9 +67,9 @@ public class TestDisjunctionMaxQuery ext
}
@Override
- public byte computeNorm(FieldInvertState state) {
+ public void computeNorm(FieldInvertState state, Norm norm) {
// Disable length norm
- return encodeNormValue(state.getBoost());
+ norm.setByte(encodeNormValue(state.getBoost()));
}
@Override
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java Mon Jan 16 14:57:15 2012
@@ -29,6 +29,7 @@ import org.apache.lucene.index.DocValues
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
@@ -152,8 +153,8 @@ public class TestDocValuesScoring extend
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return sim.computeNorm(state);
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ sim.computeNorm(state, norm);
}
@Override
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarity.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarity.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarity.java Mon Jan 16 14:57:15 2012
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.DefaultSimilarity;
@@ -45,7 +46,7 @@ public class TestSimilarity extends Luce
public float coord(int overlap, int maxOverlap) { return 1.0f; }
public Similarity get(String field) {
return new DefaultSimilarity() {
- @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); }
+ @Override public void computeNorm(FieldInvertState state, Norm norm) { norm.setByte(encodeNormValue(state.getBoost())); }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java Mon Jan 16 14:57:15 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.index.FieldInve
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
@@ -111,9 +112,10 @@ public class TestSimilarityProvider exte
}
private class Sim1 extends TFIDFSimilarity {
+
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue(1f);
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue(1f));
}
@Override
@@ -138,9 +140,10 @@ public class TestSimilarityProvider exte
}
private class Sim2 extends TFIDFSimilarity {
+
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue(10f);
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue(10f));
}
@Override
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Mon Jan 16 14:57:15 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -330,8 +331,8 @@ public class TestPayloadNearQuery extend
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue(state.getBoost());
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue(state.getBoost()));
}
@Override
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Mon Jan 16 14:57:15 2012
@@ -37,6 +37,7 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -324,9 +325,9 @@ public class TestPayloadTermQuery extend
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- @Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue(state.getBoost());
+ @Override
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue(state.getBoost()));
}
@Override
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1232014&r1=1232013&r2=1232014&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java Mon Jan 16 14:57:15 2012
@@ -18,6 +18,7 @@
package org.apache.solr.search.function;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.similarities.DefaultSimilarity;
@@ -346,8 +347,11 @@ public class TestFunctionQuery extends S
FieldInvertState state = new FieldInvertState();
state.setBoost(1.0f);
state.setLength(4);
+ Norm norm = new Norm();
+ similarity.computeNorm(state, norm);
+ float nrm = similarity.decodeNormValue(norm.field().numericValue().byteValue());
assertQ(req("fl","*,score","q", "{!func}norm(a_t)", "fq","id:2"),
- "//float[@name='score']='" + similarity.decodeNormValue(similarity.computeNorm(state)) + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte
+ "//float[@name='score']='" + nrm + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte
// test that ord and rord are working on a global index basis, not just
// at the segment level (since Lucene 2.9 has switched to per-segment searching)