You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/15 22:00:41 UTC
svn commit: r1433646 [2/3] - in /lucene/dev/branches/lucene4547:
lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/java/org/apache/lucene/codecs/sep/
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/
lucene/core/src/ja...
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableField.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableField.java Tue Jan 15 21:00:39 2013
@@ -55,7 +55,7 @@ public interface IndexableField extends
* a "document boost", then you must pre-multiply it across all the
* relevant fields yourself.
* <p>The boost is used to compute the norm factor for the field. By
- * default, in the {@link Similarity#computeNorm(FieldInvertState, Norm)} method,
+ * default, in the {@link Similarity#computeNorm(FieldInvertState)} method,
* the boost value is multiplied by the length normalization factor and then
* rounded by {@link DefaultSimilarity#encodeNormValue(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
@@ -65,7 +65,7 @@ public interface IndexableField extends
* indexed ({@link IndexableFieldType#indexed()} is false) or omits normalization values
* ({@link IndexableFieldType#omitNorms()} returns true).
*
- * @see Similarity#computeNorm(FieldInvertState, Norm)
+ * @see Similarity#computeNorm(FieldInvertState)
* @see DefaultSimilarity#encodeNormValue(float)
*/
public float boost();
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Tue Jan 15 21:00:39 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.analysis.Analyzer; // javadocs
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
/**
@@ -92,8 +93,8 @@ public interface IndexableFieldType {
public IndexOptions indexOptions();
/**
- * DocValues {@link DocValues.Type}: if non-null then the field's value
+ * DocValues {@link DocValuesType}: if non-null then the field's value
* will be indexed into docValues.
*/
- public DocValues.Type docValueType();
+ public DocValuesType docValueType();
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java Tue Jan 15 21:00:39 2013
@@ -20,12 +20,9 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Map;
-import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleNormsFormat;
-import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.util.IOUtils;
// TODO FI: norms could actually be stored as doc store
@@ -36,61 +33,40 @@ import org.apache.lucene.util.IOUtils;
*/
final class NormsConsumer extends InvertedDocEndConsumer {
- private final NormsFormat normsFormat;
- private PerDocConsumer consumer;
-
- public NormsConsumer(DocumentsWriterPerThread dwpt) {
- normsFormat = dwpt.codec.normsFormat();
- }
@Override
- public void abort(){
- if (consumer != null) {
- consumer.abort();
- }
- }
-
+ void abort() {}
+
@Override
public void flush(Map<String,InvertedDocEndConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException {
boolean success = false;
SimpleDVConsumer normsConsumer = null;
- boolean anythingFlushed = false;
try {
if (state.fieldInfos.hasNorms()) {
SimpleNormsFormat normsFormat = state.segmentInfo.getCodec().simpleNormsFormat();
-
- // nocommit change this to assert normsFormat != null
- if (normsFormat != null) {
- normsConsumer = normsFormat.normsConsumer(state);
- }
+ assert normsFormat != null;
+ normsConsumer = normsFormat.normsConsumer(state);
for (FieldInfo fi : state.fieldInfos) {
final NormsConsumerPerField toWrite = (NormsConsumerPerField) fieldsToFlush.get(fi.name);
// we must check the final value of omitNorms for the fieldinfo, it could have
// changed for this field since the first time we added it.
if (!fi.omitsNorms()) {
- if (toWrite != null && toWrite.initialized()) {
- anythingFlushed = true;
- final Type type = toWrite.flush(state, normsConsumer);
- assert fi.getNormType() == type;
+ if (toWrite != null && !toWrite.isEmpty()) {
+ toWrite.flush(state, normsConsumer);
+ assert fi.getNormType() == DocValuesType.NUMERIC;
} else if (fi.isIndexed()) {
- anythingFlushed = true;
assert fi.getNormType() == null: "got " + fi.getNormType() + "; field=" + fi.name;
}
}
}
- }
-
- success = true;
- if (!anythingFlushed && consumer != null) {
- consumer.abort();
- // nocommit do we also need to normsConsumer.abort!?
}
+ success = true;
} finally {
if (success) {
- IOUtils.close(consumer, normsConsumer);
+ IOUtils.close(normsConsumer);
} else {
- IOUtils.closeWhileHandlingException(consumer, normsConsumer);
+ IOUtils.closeWhileHandlingException(normsConsumer);
}
}
}
@@ -102,18 +78,7 @@ final class NormsConsumer extends Invert
void startDocument() {}
@Override
- InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField,
- FieldInfo fieldInfo) {
+ InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo) {
return new NormsConsumerPerField(docInverterPerField, fieldInfo, this);
}
-
- DocValuesConsumer newConsumer(PerDocWriteState perDocWriteState,
- FieldInfo fieldInfo, Type type) throws IOException {
- if (consumer == null) {
- consumer = normsFormat.docsConsumer(perDocWriteState);
- }
- DocValuesConsumer addValuesField = consumer.addValuesField(type, fieldInfo);
- return addValuesField;
- }
-
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java Tue Jan 15 21:00:39 2013
@@ -17,9 +17,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
-import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.similarities.Similarity;
final class NormsConsumerPerField extends InvertedDocEndConsumerPerField implements Comparable<NormsConsumerPerField> {
@@ -27,20 +25,13 @@ final class NormsConsumerPerField extend
private final DocumentsWriterPerThread.DocState docState;
private final Similarity similarity;
private final FieldInvertState fieldState;
- private DocValuesConsumer consumer;
- private final Norm norm;
- private final NormsConsumer parent;
- private Type initType;
- private final NumberDVWriter simpleNormsWriter;
+ private NumberDVWriter consumer;
public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo, NormsConsumer parent) {
this.fieldInfo = fieldInfo;
- this.parent = parent;
docState = docInverterPerField.docState;
fieldState = docInverterPerField.fieldState;
similarity = docState.similarity;
- norm = new Norm();
- simpleNormsWriter = new NumberDVWriter(fieldInfo, docState.docWriter.bytesUsed);
}
@Override
@@ -51,65 +42,28 @@ final class NormsConsumerPerField extend
@Override
void finish() throws IOException {
if (fieldInfo.isIndexed() && !fieldInfo.omitsNorms()) {
- similarity.computeNorm(fieldState, norm);
-
- if (norm.type() != null) {
- StorableField field = norm.field();
- // some similarity might not compute any norms
- DocValuesConsumer consumer = getConsumer(norm.type());
- consumer.add(docState.docID, field);
+ if (consumer == null) {
+ consumer = new NumberDVWriter(fieldInfo, docState.docWriter.bytesUsed);
}
-
- long norm = similarity.computeSimpleNorm(fieldState);
- simpleNormsWriter.addValue(docState.docID, norm);
+ consumer.addValue(docState.docID, similarity.computeNorm(fieldState));
}
}
- Type flush(SegmentWriteState state, SimpleDVConsumer normsConsumer) throws IOException {
+ void flush(SegmentWriteState state, SimpleDVConsumer normsWriter) throws IOException {
int docCount = state.segmentInfo.getDocCount();
- if (!initialized()) {
- return null; // null type - not omitted but not written
- }
- consumer.finish(docCount);
- // nocommit change to assert normsConsumer != null
- if (normsConsumer != null) {
- // nocommit we need to change the suffix? ie so norms
- // don't step on dvs? hmmm.... where does this happen
- // today ...
- simpleNormsWriter.finish(docCount);
- simpleNormsWriter.flush(state, normsConsumer);
- } else {
- // nocommit remove:
- simpleNormsWriter.reset();
- }
- return initType;
- }
-
- private DocValuesConsumer getConsumer(Type type) throws IOException {
if (consumer == null) {
- if (fieldInfo.getNormType() != null && fieldInfo.getNormType() != type) {
- throw new IllegalArgumentException("cannot change Norm type from " + fieldInfo.getNormType() + " to " + type + " for field \"" + fieldInfo.name + "\"");
- }
- if (!DocValues.isNumber(type) && !DocValues.isFloat(type)) {
- throw new IllegalArgumentException("Norm type must be numeric (got type " + type + " for field \"" + fieldInfo.name + "\"");
- }
- fieldInfo.setNormValueType(type);
- consumer = parent.newConsumer(docState.docWriter.newPerDocWriteState(""), fieldInfo, type);
- this.initType = type;
- }
- if (initType != type) {
- throw new IllegalArgumentException("NormTypes for field: " + fieldInfo.name + " doesn't match " + initType + " != " + type);
+ return; // null type - not omitted but not written
}
- return consumer;
+ consumer.finish(docCount);
+ consumer.flush(state, normsWriter);
}
- boolean initialized() {
- return consumer != null;
+ boolean isEmpty() {
+ return consumer == null;
}
@Override
void abort() {
//
}
-
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java Tue Jan 15 21:00:39 2013
@@ -262,13 +262,6 @@ public final class ParallelAtomicReader
// throw the first exception
if (ioe != null) throw ioe;
}
-
- @Override
- public DocValues docValues(String field) throws IOException {
- ensureOpen();
- AtomicReader reader = fieldToReader.get(field);
- return reader == null ? null : reader.docValues(field);
- }
@Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
@@ -290,13 +283,6 @@ public final class ParallelAtomicReader
AtomicReader reader = fieldToReader.get(field);
return reader == null ? null : reader.getSortedDocValues(field);
}
-
- @Override
- public DocValues normValues(String field) throws IOException {
- ensureOpen();
- AtomicReader reader = fieldToReader.get(field);
- return reader == null ? null : reader.normValues(field);
- }
@Override
public NumericDocValues simpleNormValues(String field) throws IOException {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Tue Jan 15 21:00:39 2013
@@ -27,11 +27,11 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.SegmentReader.CoreClosedListener;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
@@ -56,8 +56,6 @@ final class SegmentCoreReaders {
final FieldsProducer fields;
final SimpleDVProducer simpleDVProducer;
final SimpleDVProducer simpleNormsProducer;
- final PerDocProducer perDocProducer;
- final PerDocProducer norms;
final int termsIndexDivisor;
@@ -133,8 +131,6 @@ final class SegmentCoreReaders {
// ask codec for its Norms:
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
- norms = codec.normsFormat().docsProducer(segmentReadState);
- perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
// nocommit shouldn't need null check:
if (codec.simpleDocValuesFormat() != null) {
if (fieldInfos.hasDocValues()) {
@@ -192,7 +188,7 @@ final class SegmentCoreReaders {
// Field was not indexed with doc values
return null;
}
- if (!DocValues.isNumber(fi.getDocValuesType()) && !DocValues.isFloat(fi.getDocValuesType())) {
+ if (fi.getDocValuesType() != DocValuesType.NUMERIC) {
// DocValues were not numeric
return null;
}
@@ -223,7 +219,7 @@ final class SegmentCoreReaders {
// Field was not indexed with doc values
return null;
}
- if (!DocValues.isBytes(fi.getDocValuesType())) {
+ if (fi.getDocValuesType() != DocValuesType.BINARY) {
// DocValues were not binary
return null;
}
@@ -254,7 +250,7 @@ final class SegmentCoreReaders {
// Field was not indexed with doc values
return null;
}
- if (!DocValues.isSortedBytes(fi.getDocValuesType())) {
+ if (fi.getDocValuesType() != DocValuesType.SORTED) {
// DocValues were not sorted
return null;
}
@@ -303,8 +299,7 @@ final class SegmentCoreReaders {
void decRef() throws IOException {
if (ref.decrementAndGet() == 0) {
IOUtils.close(termVectorsLocal, fieldsReaderLocal, simpleDocValuesLocal, simpleNormsLocal, fields, simpleDVProducer,
- perDocProducer, termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms,
- simpleNormsProducer);
+ termVectorsReaderOrig, fieldsReaderOrig, cfsReader, simpleNormsProducer);
notifyCoreClosedListeners();
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Tue Jan 15 21:00:39 2013
@@ -26,10 +26,10 @@ import java.util.Map;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
@@ -97,7 +97,7 @@ final class SegmentMerger {
// threads.
mergeState.segmentInfo.setDocCount(setDocMaps());
- mergeDocValuesAndNormsFieldInfos();
+ mergeFieldInfos();
setMatchingSegmentReaders();
long t0 = 0;
if (mergeState.infoStream.isEnabled("SM")) {
@@ -124,7 +124,9 @@ final class SegmentMerger {
if (mergeState.infoStream.isEnabled("SM")) {
t0 = System.nanoTime();
}
- mergePerDoc(segmentWriteState);
+ if (mergeState.fieldInfos.hasDocValues()) {
+ mergeSimpleDocValues(segmentWriteState);
+ }
if (mergeState.infoStream.isEnabled("SM")) {
long t1 = System.nanoTime();
mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge doc values [" + numMerged + " docs]");
@@ -134,7 +136,6 @@ final class SegmentMerger {
if (mergeState.infoStream.isEnabled("SM")) {
t0 = System.nanoTime();
}
- mergeNorms(segmentWriteState);
mergeSimpleNorms(segmentWriteState);
if (mergeState.infoStream.isEnabled("SM")) {
long t1 = System.nanoTime();
@@ -142,12 +143,6 @@ final class SegmentMerger {
}
}
- // Merge simple doc values:
- if (mergeState.fieldInfos.hasDocValues()) {
- // nocommit shouldn't need null check:
- mergeSimpleDocValues(segmentWriteState);
- }
-
if (mergeState.fieldInfos.hasVectors()) {
if (mergeState.infoStream.isEnabled("SM")) {
t0 = System.nanoTime();
@@ -174,9 +169,9 @@ final class SegmentMerger {
boolean success = false;
try {
for (FieldInfo field : mergeState.fieldInfos) {
- DocValues.Type type = field.getDocValuesType();
+ DocValuesType type = field.getDocValuesType();
if (type != null) {
- if (DocValues.isNumber(type) || DocValues.isFloat(type)) {
+ if (type == DocValuesType.NUMERIC) {
List<NumericDocValues> toMerge = new ArrayList<NumericDocValues>();
for (AtomicReader reader : mergeState.readers) {
NumericDocValues values = reader.getNumericDocValues(field.name);
@@ -186,7 +181,7 @@ final class SegmentMerger {
toMerge.add(values);
}
consumer.mergeNumericField(field, mergeState, toMerge);
- } else if (DocValues.isBytes(type)) {
+ } else if (type == DocValuesType.BINARY) {
List<BinaryDocValues> toMerge = new ArrayList<BinaryDocValues>();
for (AtomicReader reader : mergeState.readers) {
BinaryDocValues values = reader.getBinaryDocValues(field.name);
@@ -196,7 +191,7 @@ final class SegmentMerger {
toMerge.add(values);
}
consumer.mergeBinaryField(field, mergeState, toMerge);
- } else if (DocValues.isSortedBytes(type)) {
+ } else if (type == DocValuesType.SORTED) {
List<SortedDocValues> toMerge = new ArrayList<SortedDocValues>();
for (AtomicReader reader : mergeState.readers) {
SortedDocValues values = reader.getSortedDocValues(field.name);
@@ -295,72 +290,15 @@ final class SegmentMerger {
}
}
- // returns an updated typepromoter (tracking type and size) given a previous one,
- // and a newly encountered docvalues
- private TypePromoter mergeDocValuesType(TypePromoter previous, DocValues docValues) {
- TypePromoter incoming = TypePromoter.create(docValues.getType(), docValues.getValueSize());
- if (previous == null) {
- previous = TypePromoter.getIdentityPromoter();
- }
- return previous.promote(incoming);
- }
-
- // NOTE: this is actually merging all the fieldinfos
- public void mergeDocValuesAndNormsFieldInfos() throws IOException {
- // mapping from all docvalues fields found to their promoted types
- // this is because FieldInfos does not store the
- // valueSize
- Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
- Map<FieldInfo,TypePromoter> normValuesTypes = new HashMap<FieldInfo,TypePromoter>();
-
+ public void mergeFieldInfos() throws IOException {
for (AtomicReader reader : mergeState.readers) {
FieldInfos readerFieldInfos = reader.getFieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- FieldInfo merged = fieldInfosBuilder.add(fi);
- // update the type promotion mapping for this reader
- if (fi.hasDocValues()) {
- TypePromoter previous = docValuesTypes.get(merged);
- docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
- }
- if (fi.hasNorms()) {
- TypePromoter previous = normValuesTypes.get(merged);
- normValuesTypes.put(merged, mergeDocValuesType(previous, reader.normValues(fi.name)));
- }
+ fieldInfosBuilder.add(fi);
}
}
- updatePromoted(normValuesTypes, true);
- updatePromoted(docValuesTypes, false);
mergeState.fieldInfos = fieldInfosBuilder.finish();
}
-
- protected void updatePromoted(Map<FieldInfo,TypePromoter> infoAndPromoter, boolean norms) {
- // update any promoted doc values types:
- for (Map.Entry<FieldInfo,TypePromoter> e : infoAndPromoter.entrySet()) {
- FieldInfo fi = e.getKey();
- TypePromoter promoter = e.getValue();
- if (promoter == null) {
- if (norms) {
- fi.setNormValueType(null);
- } else {
- fi.setDocValuesType(null);
- }
- } else {
- assert promoter != TypePromoter.getIdentityPromoter();
- if (norms) {
- if (fi.getNormType() != promoter.type() && !fi.omitsNorms()) {
- // reset the type if we got promoted
- fi.setNormValueType(promoter.type());
- }
- } else {
- if (fi.getDocValuesType() != promoter.type()) {
- // reset the type if we got promoted
- fi.setDocValuesType(promoter.type());
- }
- }
- }
- }
- }
-
/**
*
@@ -451,38 +389,4 @@ final class SegmentMerger {
}
}
}
-
- private void mergePerDoc(SegmentWriteState segmentWriteState) throws IOException {
- final PerDocConsumer docsConsumer = codec.docValuesFormat()
- .docsConsumer(new PerDocWriteState(segmentWriteState));
- assert docsConsumer != null;
- boolean success = false;
- try {
- docsConsumer.merge(mergeState);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(docsConsumer);
- } else {
- IOUtils.closeWhileHandlingException(docsConsumer);
- }
- }
- }
-
- private void mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
- final PerDocConsumer docsConsumer = codec.normsFormat()
- .docsConsumer(new PerDocWriteState(segmentWriteState));
- assert docsConsumer != null;
- boolean success = false;
- try {
- docsConsumer.merge(mergeState);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(docsConsumer);
- } else {
- IOUtils.closeWhileHandlingException(docsConsumer);
- }
- }
- }
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Tue Jan 15 21:00:39 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.search.FieldCache; // javadocs
@@ -249,31 +248,10 @@ public final class SegmentReader extends
}
@Override
- public DocValues docValues(String field) throws IOException {
- ensureOpen();
- final PerDocProducer perDoc = core.perDocProducer;
- if (perDoc == null) {
- return null;
- }
- return perDoc.docValues(field);
- }
-
- @Override
public NumericDocValues simpleNormValues(String field) throws IOException {
ensureOpen();
return core.getSimpleNormValues(field);
}
-
- @Override
- public DocValues normValues(String field) throws IOException {
- ensureOpen();
- final PerDocProducer perDoc = core.norms;
- if (perDoc == null) {
- return null;
- }
- return perDoc.docValues(field);
- }
-
/**
* Called when the shared core for this SegmentReader
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Tue Jan 15 21:00:39 2013
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.lucene.util.Bits;
@@ -31,7 +29,7 @@ import org.apache.lucene.index.MultiRead
* MultiReader} or {@link DirectoryReader}) to emulate an
* atomic reader. This requires implementing the postings
* APIs on-the-fly, using the static methods in {@link
- * MultiFields}, {@link MultiDocValues}, by stepping through
+ * MultiFields}, {@link MultiSimpleDocValues}, by stepping through
* the sub-readers to merge fields/terms, appending docs, etc.
*
* <p><b>NOTE</b>: this class almost always results in a
@@ -44,7 +42,6 @@ import org.apache.lucene.index.MultiRead
public final class SlowCompositeReaderWrapper extends AtomicReader {
private final CompositeReader in;
- private final Map<String, DocValues> normsCache = new HashMap<String, DocValues>();
private final Fields fields;
private final Bits liveDocs;
@@ -83,12 +80,6 @@ public final class SlowCompositeReaderWr
}
@Override
- public DocValues docValues(String field) throws IOException {
- ensureOpen();
- return MultiDocValues.getDocValues(in, field);
- }
-
- @Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
ensureOpen();
return MultiSimpleDocValues.simpleNumericValues(in, field);
@@ -105,17 +96,6 @@ public final class SlowCompositeReaderWr
ensureOpen();
return MultiSimpleDocValues.simpleSortedValues(in, field);
}
-
- @Override
- public synchronized DocValues normValues(String field) throws IOException {
- ensureOpen();
- DocValues values = normsCache.get(field);
- if (values == null) {
- values = MultiDocValues.getNormDocValues(in, field);
- normsCache.put(field, values);
- }
- return values;
- }
@Override
public NumericDocValues simpleNormValues(String field) throws IOException {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StorableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StorableFieldType.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StorableFieldType.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StorableFieldType.java Tue Jan 15 21:00:39 2013
@@ -1,5 +1,7 @@
package org.apache.lucene.index;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -27,5 +29,5 @@ public interface StorableFieldType {
/** DocValues type; if non-null then the field's value
* will be indexed into docValues */
- public DocValues.Type docValueType();
+ public DocValuesType docValueType();
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Tue Jan 15 21:00:39 2013
@@ -20,9 +20,8 @@ package org.apache.lucene.search.similar
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -136,9 +135,9 @@ public class BM25Similarity extends Simi
@Override
- public final void computeNorm(FieldInvertState state, Norm norm) {
+ public final long computeNorm(FieldInvertState state) {
final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength();
- norm.setByte(encodeNormValue(state.getBoost(), numTerms));
+ return encodeNormValue(state.getBoost(), numTerms);
}
/**
@@ -215,7 +214,7 @@ public class BM25Similarity extends Simi
@Override
public final ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
BM25Stats bm25stats = (BM25Stats) stats;
- final DocValues norms = context.reader().normValues(bm25stats.field);
+ final NumericDocValues norms = context.reader().simpleNormValues(bm25stats.field);
return norms == null
? new ExactBM25DocScorerNoNorms(bm25stats)
: new ExactBM25DocScorer(bm25stats, norms);
@@ -224,26 +223,26 @@ public class BM25Similarity extends Simi
@Override
public final SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
BM25Stats bm25stats = (BM25Stats) stats;
- return new SloppyBM25DocScorer(bm25stats, context.reader().normValues(bm25stats.field));
+ return new SloppyBM25DocScorer(bm25stats, context.reader().simpleNormValues(bm25stats.field));
}
private class ExactBM25DocScorer extends ExactSimScorer {
private final BM25Stats stats;
private final float weightValue;
- private final byte[] norms;
+ private final NumericDocValues norms;
private final float[] cache;
- ExactBM25DocScorer(BM25Stats stats, DocValues norms) throws IOException {
+ ExactBM25DocScorer(BM25Stats stats, NumericDocValues norms) throws IOException {
assert norms != null;
this.stats = stats;
this.weightValue = stats.weight * (k1 + 1); // boost * idf * (k1 + 1)
this.cache = stats.cache;
- this.norms = (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, int freq) {
- return weightValue * freq / (freq + cache[norms[doc] & 0xFF]);
+ return weightValue * freq / (freq + cache[(byte)norms.get(doc) & 0xFF]);
}
@Override
@@ -283,20 +282,20 @@ public class BM25Similarity extends Simi
private class SloppyBM25DocScorer extends SloppySimScorer {
private final BM25Stats stats;
private final float weightValue; // boost * idf * (k1 + 1)
- private final byte[] norms;
+ private final NumericDocValues norms;
private final float[] cache;
- SloppyBM25DocScorer(BM25Stats stats, DocValues norms) throws IOException {
+ SloppyBM25DocScorer(BM25Stats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.weight * (k1 + 1);
this.cache = stats.cache;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, float freq) {
// if there are no norms, we act as if b=0
- float norm = norms == null ? k1 : cache[norms[doc] & 0xFF];
+ float norm = norms == null ? k1 : cache[(byte)norms.get(doc) & 0xFF];
return weightValue * freq / (freq + norm);
}
@@ -356,7 +355,7 @@ public class BM25Similarity extends Simi
}
}
- private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, byte[] norms) {
+ private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) {
Explanation result = new Explanation();
result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
@@ -374,7 +373,7 @@ public class BM25Similarity extends Simi
tfNormExpl.addDetail(new Explanation(0, "parameter b (norms omitted for field)"));
tfNormExpl.setValue((freq.getValue() * (k1 + 1)) / (freq.getValue() + k1));
} else {
- float doclen = decodeNormValue(norms[doc]);
+ float doclen = decodeNormValue((byte)norms.get(doc));
tfNormExpl.addDetail(new Explanation(b, "parameter b"));
tfNormExpl.addDetail(new Explanation(stats.avgdl, "avgFieldLength"));
tfNormExpl.addDetail(new Explanation(doclen, "fieldLength"));
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Tue Jan 15 21:00:39 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.similar
*/
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.util.BytesRef;
/** Expert: Default scoring implementation. */
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java Tue Jan 15 21:00:39 2013
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -44,8 +43,8 @@ public class MultiSimilarity extends Sim
}
@Override
- public void computeNorm(FieldInvertState state, Norm norm) {
- sims[0].computeNorm(state, norm);
+ public long computeNorm(FieldInvertState state) {
+ return sims[0].computeNorm(state);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java Tue Jan 15 21:00:39 2013
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
@@ -42,8 +41,8 @@ public abstract class PerFieldSimilarity
public PerFieldSimilarityWrapper() {}
@Override
- public final void computeNorm(FieldInvertState state, Norm norm) {
- get(state.getName()).computeNorm(state, norm);
+ public final long computeNorm(FieldInvertState state) {
+ return get(state.getName()).computeNorm(state);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java Tue Jan 15 21:00:39 2013
@@ -23,9 +23,7 @@ import org.apache.lucene.document.ByteDo
import org.apache.lucene.document.FloatDocValuesField; // javadoc
import org.apache.lucene.index.AtomicReader; // javadoc
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
@@ -53,9 +51,9 @@ import org.apache.lucene.util.SmallFloat
* <a href="#querytime">query-time</a>.
* <p>
* <a name="indextime"/>
- * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState, Norm)}, allowing
+ * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing
* the Similarity implementation to set a per-document value for the field that will
- * be later accessible via {@link AtomicReader#normValues(String)}. Lucene makes no assumption
+ * be later accessible via {@link AtomicReader#simpleNormValues(String)}. Lucene makes no assumption
* about what is in this norm, but it is most useful for encoding length normalization
* information.
* <p>
@@ -72,7 +70,7 @@ import org.apache.lucene.util.SmallFloat
* Additional scoring factors can be stored in named
* <code>*DocValuesField</code>s (such as {@link
* ByteDocValuesField} or {@link FloatDocValuesField}), and accessed
- * at query-time with {@link AtomicReader#docValues(String)}.
+ * at query-time with {@link AtomicReader#getNumericDocValues(String)}.
* <p>
* Finally, using index-time boosts (either via folding into the normalization byte or
* via DocValues), is an inefficient way to boost the scores of different fields if the
@@ -150,9 +148,6 @@ public abstract class Similarity {
/**
* Computes the normalization value for a field, given the accumulated
* state of term processing for this field (see {@link FieldInvertState}).
- *
- * <p>Implementations should calculate a norm value based on the field
- * state and set that value to the given {@link Norm}.
*
* <p>Matches in longer fields are less precise, so implementations of this
* method usually set smaller values when <code>state.getLength()</code> is large,
@@ -161,21 +156,9 @@ public abstract class Similarity {
* @lucene.experimental
*
* @param state current processing state for this field
- * @param norm holds the computed norm value when this method returns
+ * @return computed norm value
*/
- public abstract void computeNorm(FieldInvertState state, Norm norm);
-
- // nocommit replace computeNorm w/ this, make it abstract:
- public long computeSimpleNorm(FieldInvertState state) {
- Norm normScratch = new Norm();
- computeNorm(state, normScratch);
- if (normScratch.type() != null && DocValues.isNumber(normScratch.type())) {
- return normScratch.field().numericValue().longValue();
- } else {
- // nocommit hmm how to return "not set"...?
- return -1;
- }
- }
+ public abstract long computeNorm(FieldInvertState state);
/**
* Compute any collection-level weight (e.g. IDF, average document length, etc) needed for scoring a query.
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Tue Jan 15 21:00:39 2013
@@ -20,9 +20,8 @@ package org.apache.lucene.search.similar
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -199,12 +198,12 @@ public abstract class SimilarityBase ext
ExactSimScorer subScorers[] = new ExactSimScorer[subStats.length];
for (int i = 0; i < subScorers.length; i++) {
BasicStats basicstats = (BasicStats) subStats[i];
- subScorers[i] = new BasicExactDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ subScorers[i] = new BasicExactDocScorer(basicstats, context.reader().simpleNormValues(basicstats.field));
}
return new MultiSimilarity.MultiExactDocScorer(subScorers);
} else {
BasicStats basicstats = (BasicStats) stats;
- return new BasicExactDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ return new BasicExactDocScorer(basicstats, context.reader().simpleNormValues(basicstats.field));
}
}
@@ -217,12 +216,12 @@ public abstract class SimilarityBase ext
SloppySimScorer subScorers[] = new SloppySimScorer[subStats.length];
for (int i = 0; i < subScorers.length; i++) {
BasicStats basicstats = (BasicStats) subStats[i];
- subScorers[i] = new BasicSloppyDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ subScorers[i] = new BasicSloppyDocScorer(basicstats, context.reader().simpleNormValues(basicstats.field));
}
return new MultiSimilarity.MultiSloppyDocScorer(subScorers);
} else {
BasicStats basicstats = (BasicStats) stats;
- return new BasicSloppyDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ return new BasicSloppyDocScorer(basicstats, context.reader().simpleNormValues(basicstats.field));
}
}
@@ -247,13 +246,13 @@ public abstract class SimilarityBase ext
/** Encodes the document length in the same way as {@link TFIDFSimilarity}. */
@Override
- public void computeNorm(FieldInvertState state, Norm norm) {
+ public long computeNorm(FieldInvertState state) {
final float numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength() / state.getBoost();
- norm.setByte(encodeNormValue(state.getBoost(), numTerms));
+ return encodeNormValue(state.getBoost(), numTerms);
}
/** Decodes a normalization factor (document length) stored in an index.
@@ -286,24 +285,24 @@ public abstract class SimilarityBase ext
*/
private class BasicExactDocScorer extends ExactSimScorer {
private final BasicStats stats;
- private final byte[] norms;
+ private final NumericDocValues norms;
- BasicExactDocScorer(BasicStats stats, DocValues norms) throws IOException {
+ BasicExactDocScorer(BasicStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, int freq) {
// We have to supply something in case norms are omitted
return SimilarityBase.this.score(stats, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
@Override
public Explanation explain(int doc, Explanation freq) {
return SimilarityBase.this.explain(stats, doc, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
}
@@ -315,23 +314,23 @@ public abstract class SimilarityBase ext
*/
private class BasicSloppyDocScorer extends SloppySimScorer {
private final BasicStats stats;
- private final byte[] norms;
+ private final NumericDocValues norms;
- BasicSloppyDocScorer(BasicStats stats, DocValues norms) throws IOException {
+ BasicSloppyDocScorer(BasicStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, float freq) {
// We have to supply something in case norms are omitted
return SimilarityBase.this.score(stats, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
@Override
public Explanation explain(int doc, Explanation freq) {
return SimilarityBase.this.explain(stats, doc, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Tue Jan 15 21:00:39 2013
@@ -17,13 +17,10 @@ package org.apache.lucene.search.similar
* limitations under the License.
*/
-
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
@@ -683,9 +680,9 @@ public abstract class TFIDFSimilarity ex
public abstract float lengthNorm(FieldInvertState state);
@Override
- public final void computeNorm(FieldInvertState state, Norm norm) {
+ public final long computeNorm(FieldInvertState state) {
float normValue = lengthNorm(state);
- norm.setByte(encodeNormValue(normValue));
+ return encodeNormValue(normValue);
}
/** Cache of decoded bytes. */
@@ -760,65 +757,28 @@ public abstract class TFIDFSimilarity ex
@Override
public final ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
IDFStats idfstats = (IDFStats) stats;
- NumericDocValues normValues = context.reader().simpleNormValues(idfstats.field);
- if (normValues != null) {
- return new SimpleExactTFIDFDocScorer(idfstats, normValues);
- } else {
- return new ExactTFIDFDocScorer(idfstats, context.reader().normValues(idfstats.field));
- }
+ return new ExactTFIDFDocScorer(idfstats, context.reader().simpleNormValues(idfstats.field));
}
@Override
public final SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
IDFStats idfstats = (IDFStats) stats;
- return new SloppyTFIDFDocScorer(idfstats, context.reader().normValues(idfstats.field));
+ return new SloppyTFIDFDocScorer(idfstats, context.reader().simpleNormValues(idfstats.field));
}
// TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot.
- private final class SimpleExactTFIDFDocScorer extends ExactSimScorer {
- private final IDFStats stats;
- private final float weightValue;
- private final NumericDocValues norms;
- private static final int SCORE_CACHE_SIZE = 32;
- private float[] scoreCache = new float[SCORE_CACHE_SIZE];
-
- SimpleExactTFIDFDocScorer(IDFStats stats, NumericDocValues norms) throws IOException {
- this.stats = stats;
- this.weightValue = stats.value;
- this.norms = norms;
- for (int i = 0; i < SCORE_CACHE_SIZE; i++) {
- scoreCache[i] = tf(i) * weightValue;
- }
- }
-
- @Override
- public float score(int doc, int freq) {
- final float raw = // compute tf(f)*weight
- freq < SCORE_CACHE_SIZE // check cache
- ? scoreCache[freq] // cache hit
- : tf(freq)*weightValue; // cache miss
-
- return norms == null ? raw : raw * decodeNormValue((byte) norms.get(doc)); // normalize for field
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) {
- return explainScore(doc, freq, stats, norms);
- }
- }
-
private final class ExactTFIDFDocScorer extends ExactSimScorer {
private final IDFStats stats;
private final float weightValue;
- private final byte[] norms;
+ private final NumericDocValues norms;
private static final int SCORE_CACHE_SIZE = 32;
private float[] scoreCache = new float[SCORE_CACHE_SIZE];
- ExactTFIDFDocScorer(IDFStats stats, DocValues norms) throws IOException {
+ ExactTFIDFDocScorer(IDFStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.value;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
for (int i = 0; i < SCORE_CACHE_SIZE; i++)
scoreCache[i] = tf(i) * weightValue;
}
@@ -830,7 +790,7 @@ public abstract class TFIDFSimilarity ex
? scoreCache[freq] // cache hit
: tf(freq)*weightValue; // cache miss
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
+ return norms == null ? raw : raw * decodeNormValue((byte)norms.get(doc)); // normalize for field
}
@Override
@@ -842,19 +802,19 @@ public abstract class TFIDFSimilarity ex
private final class SloppyTFIDFDocScorer extends SloppySimScorer {
private final IDFStats stats;
private final float weightValue;
- private final byte[] norms;
+ private final NumericDocValues norms;
- SloppyTFIDFDocScorer(IDFStats stats, DocValues norms) throws IOException {
+ SloppyTFIDFDocScorer(IDFStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.value;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, float freq) {
final float raw = tf(freq) * weightValue; // compute tf(f)*weight
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
+ return norms == null ? raw : raw * decodeNormValue((byte)norms.get(doc)); // normalize for field
}
@Override
@@ -904,63 +864,7 @@ public abstract class TFIDFSimilarity ex
queryWeight *= this.queryNorm; // normalize query weight
value = queryWeight * idf.getValue(); // idf for document
}
- }
-
- private Explanation explainScore(int doc, Explanation freq, IDFStats stats, byte[] norms) {
- Explanation result = new Explanation();
- result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight, product of:");
-
- Explanation boostExpl = new Explanation(stats.queryBoost, "boost");
- if (stats.queryBoost != 1.0f)
- queryExpl.addDetail(boostExpl);
- queryExpl.addDetail(stats.idf);
-
- Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- stats.idf.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- Explanation fieldExpl = new Explanation();
- fieldExpl.setDescription("fieldWeight in "+doc+
- ", product of:");
-
- Explanation tfExplanation = new Explanation();
- tfExplanation.setValue(tf(freq.getValue()));
- tfExplanation.setDescription("tf(freq="+freq.getValue()+"), with freq of:");
- tfExplanation.addDetail(freq);
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(stats.idf);
-
- Explanation fieldNormExpl = new Explanation();
- float fieldNorm =
- norms!=null ? decodeNormValue(norms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setValue(tfExplanation.getValue() *
- stats.idf.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
-
- if (queryExpl.getValue() == 1.0f)
- return fieldExpl;
-
- return result;
- }
+ }
private Explanation explainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) {
Explanation result = new Explanation();
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java Tue Jan 15 21:00:39 2013
@@ -17,25 +17,17 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.io.IOException;
-import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DocValues.Source;
-import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.ExactSimScorer;
-import org.apache.lucene.search.similarities.Similarity.SimWeight;
-import org.apache.lucene.search.similarities.Similarity.SloppySimScorer;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -72,106 +64,18 @@ public class TestCustomNorms extends Luc
writer.commit();
writer.close();
AtomicReader open = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
- DocValues normValues = open.normValues(floatTestField);
- assertNotNull(normValues);
- Source source = normValues.getSource();
- assertTrue(source.hasArray());
- assertEquals(Type.FLOAT_32, normValues.getType());
- float[] norms = (float[]) source.getArray();
+ NumericDocValues norms = open.simpleNormValues(floatTestField);
+ assertNotNull(norms);
for (int i = 0; i < open.maxDoc(); i++) {
StoredDocument document = open.document(i);
float expected = Float.parseFloat(document.get(floatTestField));
- assertEquals(expected, norms[i], 0.0f);
+ assertEquals(expected, Float.intBitsToFloat((int)norms.get(i)), 0.0f);
}
open.close();
dir.close();
docs.close();
}
- public void testExceptionOnRandomType() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- Similarity provider = new MySimProvider();
- config.setSimilarity(provider);
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
- final LineFileDocs docs = new LineFileDocs(random());
- int num = atLeast(100);
- try {
- for (int i = 0; i < num; i++) {
- Document doc = docs.nextDoc();
- float nextFloat = random().nextFloat();
- Field f = new TextField(exceptionTestField, "" + nextFloat, Field.Store.YES);
- f.setBoost(nextFloat);
-
- doc.add(f);
- writer.addDocument(doc);
- doc.removeField(exceptionTestField);
- if (rarely()) {
- writer.commit();
- }
- }
- fail("expected exception - incompatible types");
- } catch (IllegalArgumentException e) {
- // expected
- }
- writer.commit();
- writer.close();
- dir.close();
- docs.close();
-
- }
-
- public void testIllegalCustomEncoder() throws Exception {
- Directory dir = newDirectory();
- IllegalCustomEncodingSimilarity similarity = new IllegalCustomEncodingSimilarity();
- IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- config.setSimilarity(similarity);
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
- Document doc = new Document();
- Field foo = newTextField("foo", "", Field.Store.NO);
- Field bar = newTextField("bar", "", Field.Store.NO);
- doc.add(foo);
- doc.add(bar);
-
- int numAdded = 0;
- for (int i = 0; i < 100; i++) {
- try {
- bar.setStringValue("singleton");
- similarity.useByte = random().nextBoolean();
- writer.addDocument(doc);
- numAdded++;
- } catch (IllegalArgumentException e) {}
- }
-
-
- IndexReader reader = writer.getReader();
- writer.close();
- assertEquals(numAdded, reader.numDocs());
- IndexReaderContext topReaderContext = reader.getContext();
- for (final AtomicReaderContext ctx : topReaderContext.leaves()) {
- AtomicReader atomicReader = ctx.reader();
- Source source = random().nextBoolean() ? atomicReader.normValues("foo").getSource() : atomicReader.normValues("foo").getDirectSource();
- Bits liveDocs = atomicReader.getLiveDocs();
- Type t = source.getType();
- for (int i = 0; i < atomicReader.maxDoc(); i++) {
- assertEquals(0, source.getFloat(i), 0.000f);
- }
-
-
- source = random().nextBoolean() ? atomicReader.normValues("bar").getSource() : atomicReader.normValues("bar").getDirectSource();
- for (int i = 0; i < atomicReader.maxDoc(); i++) {
- if (liveDocs == null || liveDocs.get(i)) {
- assertEquals("type: " + t, 1, source.getFloat(i), 0.000f);
- } else {
- assertEquals("type: " + t, 0, source.getFloat(i), 0.000f);
- }
- }
- }
- reader.close();
- dir.close();
- }
-
public class MySimProvider extends PerFieldSimilarityWrapper {
Similarity delegate = new DefaultSimilarity();
@@ -184,8 +88,6 @@ public class TestCustomNorms extends Luc
public Similarity get(String field) {
if (floatTestField.equals(field)) {
return new FloatEncodingBoostSimilarity();
- } else if (exceptionTestField.equals(field)) {
- return new RandomTypeSimilarity(random());
} else {
return delegate;
}
@@ -200,9 +102,8 @@ public class TestCustomNorms extends Luc
public static class FloatEncodingBoostSimilarity extends Similarity {
@Override
- public void computeNorm(FieldInvertState state, Norm norm) {
- float boost = state.getBoost();
- norm.setFloat(boost);
+ public long computeNorm(FieldInvertState state) {
+ return Float.floatToIntBits(state.getBoost());
}
@Override
@@ -220,87 +121,4 @@ public class TestCustomNorms extends Luc
throw new UnsupportedOperationException();
}
}
-
- public static class RandomTypeSimilarity extends Similarity {
-
- private final Random random;
-
- public RandomTypeSimilarity(Random random) {
- this.random = random;
- }
-
- @Override
- public void computeNorm(FieldInvertState state, Norm norm) {
- float boost = state.getBoost();
- int nextInt = random.nextInt(10);
- switch (nextInt) {
- case 0:
- norm.setDouble((double) boost);
- break;
- case 1:
- norm.setFloat(boost);
- break;
- case 2:
- norm.setLong((long) boost);
- break;
- case 3:
- norm.setBytes(new BytesRef(new byte[6]));
- break;
- case 4:
- norm.setInt((int) boost);
- break;
- case 5:
- norm.setShort((short) boost);
- break;
- default:
- norm.setByte((byte) boost);
- }
-
- }
-
- @Override
- public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
- throw new UnsupportedOperationException();
- }
- }
-
- class IllegalCustomEncodingSimilarity extends Similarity {
-
- public boolean useByte = false;
-
- @Override
- public void computeNorm(FieldInvertState state, Norm norm) {
- if (useByte) {
- norm.setByte((byte)state.getLength());
- } else {
- norm.setFloat((float)state.getLength());
- }
- }
-
- @Override
- public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
- throw new UnsupportedOperationException();
- }
- }
-
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Tue Jan 15 21:00:39 2013
@@ -546,7 +546,6 @@ public void testFilesOpenClose() throws
// TODO: maybe this can reuse the logic of test dueling codecs?
public static void assertIndexEquals(DirectoryReader index1, DirectoryReader index2) throws IOException {
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs());
assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc());
assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions());
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java Tue Jan 15 21:00:39 2013
@@ -39,12 +39,10 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.junit.Assume;
public class TestDirectoryReaderReopen extends LuceneTestCase {
public void testReopen() throws Exception {
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
final Directory dir1 = newDirectory();
createIndex(random(), dir1, false);
@@ -196,7 +194,6 @@ public class TestDirectoryReaderReopen e
}
public void testThreadSafety() throws Exception {
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
final Directory dir = newDirectory();
// NOTE: this also controls the number of threads!
final int n = _TestUtil.nextInt(random(), 20, 40);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java Tue Jan 15 21:00:39 2013
@@ -97,7 +97,7 @@ public class TestDocumentWriter extends
// omitNorms is true
for (FieldInfo fi : reader.getFieldInfos()) {
if (fi.isIndexed()) {
- assertTrue(fi.omitsNorms() == (reader.normValues(fi.name) == null));
+ assertTrue(fi.omitsNorms() == (reader.simpleNormValues(fi.name) == null));
}
}
reader.close();
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Tue Jan 15 21:00:39 2013
@@ -38,11 +38,9 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
-import org.junit.Assume;
/**
* Compares one codec against another
@@ -68,7 +66,6 @@ public class TestDuelingCodecs extends L
leftCodec = Codec.forName("SimpleText");
rightCodec = new RandomCodec(random());
- Assume.assumeTrue(rightCodec.simpleNormsFormat() != null);
leftDir = newDirectory();
rightDir = newDirectory();
@@ -519,7 +516,6 @@ public class TestDuelingCodecs extends L
* checks that norms are the same across all fields
*/
public void assertNorms(IndexReader leftReader, IndexReader rightReader) throws Exception {
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
Fields leftFields = MultiFields.getFields(leftReader);
Fields rightFields = MultiFields.getFields(rightReader);
// Fields could be null if there are no postings,
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Tue Jan 15 21:00:39 2013
@@ -25,7 +25,7 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocIdSetIterator;
@@ -89,7 +89,7 @@ public class TestIndexableField extends
}
@Override
- public Type docValueType() {
+ public DocValuesType docValueType() {
return null;
}
};
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Tue Jan 15 21:00:39 2013
@@ -30,7 +30,6 @@ import org.apache.lucene.search.similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.junit.Assume;
/**
* Tests the maxTermFrequency statistic in FieldInvertState
@@ -68,8 +67,6 @@ public class TestMaxTermFrequency extend
}
public void test() throws Exception {
- // nocommit remove
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
NumericDocValues fooNorms = MultiSimpleDocValues.simpleNormValues(reader, "foo");
for (int i = 0; i < reader.maxDoc(); i++) {
assertEquals(expected.get(i).intValue(), fooNorms.get(i) & 0xff);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestNorms.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestNorms.java Tue Jan 15 21:00:39 2013
@@ -24,8 +24,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DocValues.Source;
-import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.DefaultSimilarity;
@@ -37,7 +35,6 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.junit.Assume;
/**
* Test that norms info is preserved during index life - including
@@ -68,8 +65,6 @@ public class TestNorms extends LuceneTes
// LUCENE-1260
public void testCustomEncoder() throws Exception {
- // nocommit remove:
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
Directory dir = newDirectory();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
config.setSimilarity(new CustomNormEncodingSimilarity());
@@ -104,99 +99,33 @@ public class TestNorms extends LuceneTes
public void testMaxByteNorms() throws IOException {
Directory dir = newFSDirectory(_TestUtil.getTempDir("TestNorms.testMaxByteNorms"));
- buildIndex(dir, true);
+ buildIndex(dir);
AtomicReader open = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
- DocValues normValues = open.normValues(byteTestField);
+ NumericDocValues normValues = open.simpleNormValues(byteTestField);
assertNotNull(normValues);
- Source source = normValues.getSource();
- assertTrue(source.hasArray());
- assertEquals(Type.FIXED_INTS_8, normValues.getType());
- byte[] norms = (byte[]) source.getArray();
for (int i = 0; i < open.maxDoc(); i++) {
StoredDocument document = open.document(i);
int expected = Integer.parseInt(document.get(byteTestField));
- assertEquals((byte)expected, norms[i]);
+ assertEquals((byte)expected, normValues.get(i));
}
open.close();
dir.close();
}
- /**
- * this test randomly creates segments with or without norms but not omitting
- * norms. The similarity used doesn't write a norm value if writeNorms = false is
- * passed. This differs from omitNorm since norms are simply not written for this segment
- * while merging fills in default values based on the Norm {@link Type}
- */
- public void testNormsNotPresent() throws IOException {
- Directory dir = newFSDirectory(_TestUtil.getTempDir("TestNorms.testNormsNotPresent.1"));
- boolean firstWriteNorm = random().nextBoolean();
- buildIndex(dir, firstWriteNorm);
-
- Directory otherDir = newFSDirectory(_TestUtil.getTempDir("TestNorms.testNormsNotPresent.2"));
- boolean secondWriteNorm = random().nextBoolean();
- buildIndex(otherDir, secondWriteNorm);
-
- AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(otherDir));
- FieldInfos fieldInfos = reader.getFieldInfos();
- FieldInfo fieldInfo = fieldInfos.fieldInfo(byteTestField);
- assertFalse(fieldInfo.omitsNorms());
- assertTrue(fieldInfo.isIndexed());
- if (secondWriteNorm) {
- assertTrue(fieldInfo.hasNorms());
- } else {
- assertFalse(fieldInfo.hasNorms());
- }
-
- IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
- writer.addIndexes(reader);
- AtomicReader mergedReader = SlowCompositeReaderWrapper.wrap(writer.getReader());
- if (!firstWriteNorm && !secondWriteNorm) {
- DocValues normValues = mergedReader.normValues(byteTestField);
- assertNull(normValues);
- FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
- assertFalse(fi.omitsNorms());
- assertTrue(fi.isIndexed());
- assertFalse(fi.hasNorms());
- } else {
- FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
- assertFalse(fi.omitsNorms());
- assertTrue(fi.isIndexed());
- assertTrue(fi.hasNorms());
-
- DocValues normValues = mergedReader.normValues(byteTestField);
- assertNotNull(normValues);
- Source source = normValues.getSource();
- assertTrue(source.hasArray());
- assertEquals(Type.FIXED_INTS_8, normValues.getType());
- byte[] norms = (byte[]) source.getArray();
- for (int i = 0; i < mergedReader.maxDoc(); i++) {
- StoredDocument document = mergedReader.document(i);
- int expected = Integer.parseInt(document.get(byteTestField));
- assertEquals((byte) expected, norms[i]);
- }
- }
- mergedReader.close();
- reader.close();
-
- writer.close();
- dir.close();
- otherDir.close();
- }
+ // TODO: create a testNormsNotPresent ourselves by adding/deleting/merging docs
- public void buildIndex(Directory dir, boolean writeNorms) throws IOException {
+ public void buildIndex(Directory dir) throws IOException {
Random random = random();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random()));
- Similarity provider = new MySimProvider(writeNorms);
+ Similarity provider = new MySimProvider();
config.setSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
final LineFileDocs docs = new LineFileDocs(random, true);
int num = atLeast(100);
for (int i = 0; i < num; i++) {
Document doc = docs.nextDoc();
- int boost = writeNorms ? 1 + random().nextInt(255) : 0;
+ int boost = random().nextInt(255);
Field f = new TextField(byteTestField, "" + boost, Field.Store.YES);
f.setBoost(boost);
doc.add(f);
@@ -214,10 +143,7 @@ public class TestNorms extends LuceneTes
public class MySimProvider extends PerFieldSimilarityWrapper {
Similarity delegate = new DefaultSimilarity();
- private boolean writeNorms;
- public MySimProvider(boolean writeNorms) {
- this.writeNorms = writeNorms;
- }
+
@Override
public float queryNorm(float sumOfSquaredWeights) {
@@ -227,7 +153,7 @@ public class TestNorms extends LuceneTes
@Override
public Similarity get(String field) {
if (byteTestField.equals(field)) {
- return new ByteEncodingBoostSimilarity(writeNorms);
+ return new ByteEncodingBoostSimilarity();
} else {
return delegate;
}
@@ -242,18 +168,10 @@ public class TestNorms extends LuceneTes
public static class ByteEncodingBoostSimilarity extends Similarity {
- private boolean writeNorms;
-
- public ByteEncodingBoostSimilarity(boolean writeNorms) {
- this.writeNorms = writeNorms;
- }
-
@Override
- public void computeNorm(FieldInvertState state, Norm norm) {
- if (writeNorms) {
- int boost = (int) state.getBoost();
- norm.setByte((byte) (0xFF & boost));
- }
+ public long computeNorm(FieldInvertState state) {
+ int boost = (int) state.getBoost();
+ return (0xFF & boost);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java Tue Jan 15 21:00:39 2013
@@ -28,7 +28,6 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.junit.Assume;
public class TestOmitNorms extends LuceneTestCase {
// Tests whether the DocumentWriter correctly enable the
@@ -267,8 +266,6 @@ public class TestOmitNorms extends Lucen
* returns the norms for "field".
*/
NumericDocValues getNorms(String field, Field f1, Field f2) throws IOException {
- // nocommit remove
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Jan 15 21:00:39 2013
@@ -30,8 +30,6 @@ import org.apache.lucene.util.InfoStream
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts;
-import org.junit.Assume;
-
public class TestSegmentMerger extends LuceneTestCase {
//The variables for the new merged segment
@@ -79,7 +77,6 @@ public class TestSegmentMerger extends L
}
public void testMerge() throws IOException {
- Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
final Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null, null);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java Tue Jan 15 21:00:39 2013
@@ -179,9 +179,9 @@ public class TestSegmentReader extends L
for (int i=0; i<DocHelper.fields.length; i++) {
IndexableField f = DocHelper.fields[i];
if (f.fieldType().indexed()) {
- assertEquals(reader.normValues(f.name()) != null, !f.fieldType().omitNorms());
- assertEquals(reader.normValues(f.name()) != null, !DocHelper.noNorms.containsKey(f.name()));
- if (reader.normValues(f.name()) == null) {
+ assertEquals(reader.simpleNormValues(f.name()) != null, !f.fieldType().omitNorms());
+ assertEquals(reader.simpleNormValues(f.name()) != null, !DocHelper.noNorms.containsKey(f.name()));
+ if (reader.simpleNormValues(f.name()) == null) {
// test for norms of null
NumericDocValues norms = MultiSimpleDocValues.simpleNormValues(reader, f.name());
assertNull(norms);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSimpleDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSimpleDocValuesIndexing.java?rev=1433646&r1=1433645&r2=1433646&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSimpleDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestSimpleDocValuesIndexing.java Tue Jan 15 21:00:39 2013
@@ -80,7 +80,6 @@ public class TestSimpleDocValuesIndexing
* Simple test case to show how to use the API
*/
public void testDocValuesSimple() throws IOException {
- Assume.assumeTrue(_TestUtil.canUseSimpleDV());
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, writerConfig(false));
for (int i = 0; i < 5; i++) {