You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/01/07 20:33:25 UTC
svn commit: r1429978 - in /lucene/dev/branches/lucene4547/lucene:
codecs/src/java/org/apache/lucene/codecs/memory/
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/codecs/lucen...
Author: mikemccand
Date: Mon Jan 7 19:33:25 2013
New Revision: 1429978
URL: http://svn.apache.org/viewvc?rev=1429978&view=rev
Log:
cutover to Iterable<Number> instead of NumericDVConsumer so e.g. codec can iterate once to gather stats and again to do the writing
Removed:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/NumericDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java
Modified:
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java Mon Jan 7 19:33:25 2013
@@ -59,8 +59,13 @@ public class MemoryDocValuesFormat exten
NumericDocValues valuesIn = producer.getNumeric(field);
final int maxDoc = valuesIn.size();
- final long minValue = valuesIn.minValue();
- final long maxValue = valuesIn.maxValue();
+ long minValue = Long.MAX_VALUE;
+ long maxValue = Long.MIN_VALUE;
+ for(int docID=0;docID<maxDoc;docID++) {
+ long v = valuesIn.get(docID);
+ minValue = Math.min(minValue, v);
+ maxValue = Math.max(maxValue, v);
+ }
final long delta = maxValue - minValue;
final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
@@ -69,27 +74,19 @@ public class MemoryDocValuesFormat exten
values.set(docID, valuesIn.get(docID) - minValue);
}
+ final long finalMinValue = minValue;
+
return new NumericDocValues() {
@Override
public long get(int docID) {
- return minValue + values.get(docID);
+ return finalMinValue + values.get(docID);
}
@Override
public int size() {
return maxDoc;
}
-
- @Override
- public long minValue() {
- return minValue;
- }
-
- @Override
- public long maxValue() {
- return maxValue;
- }
};
}
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java Mon Jan 7 19:33:25 2013
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.codecs.BinaryDocValuesConsumer;
-import org.apache.lucene.codecs.NumericDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
@@ -62,7 +61,6 @@ public class SimpleTextSimpleDocValuesFo
final static BytesRef FIELD = new BytesRef("field ");
// used for numerics
final static BytesRef MINVALUE = new BytesRef(" minvalue ");
- final static BytesRef MAXVALUE = new BytesRef(" maxvalue ");
final static BytesRef PATTERN = new BytesRef(" pattern ");
// used for bytes
final static BytesRef FIXEDLENGTH = new BytesRef(" fixedlength ");
@@ -91,7 +89,6 @@ public class SimpleTextSimpleDocValuesFo
* <pre>
* field myField
* minvalue 0
- * maxvalue 234
* pattern 000
* 005
* 234
@@ -165,23 +162,26 @@ public class SimpleTextSimpleDocValuesFo
}
@Override
- public NumericDocValuesConsumer addNumericField(FieldInfo field, final long minValue, long maxValue) throws IOException {
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
assert fieldSeen(field.name);
assert (field.getDocValuesType() != null && (DocValues.isNumber(field.getDocValuesType()) || DocValues.isFloat(field.getDocValuesType()))) ||
(field.getNormType() != null && (DocValues.isNumber(field.getNormType()) || DocValues.isFloat(field.getNormType()))): "field=" + field.name;
writeFieldEntry(field);
+
+ // first pass to find min/max
+ long minValue = Long.MAX_VALUE;
+ long maxValue = Long.MIN_VALUE;
+ for(Number n : values) {
+ long v = n.longValue();
+ minValue = Math.min(minValue, v);
+ maxValue = Math.max(maxValue, v);
+ }
// write our minimum value to the .dat, all entries are deltas from that
SimpleTextUtil.write(data, MINVALUE);
SimpleTextUtil.write(data, Long.toString(minValue), scratch);
SimpleTextUtil.writeNewline(data);
- SimpleTextUtil.write(data, MAXVALUE);
- SimpleTextUtil.write(data, Long.toString(maxValue), scratch);
- SimpleTextUtil.writeNewline(data);
-
- assert maxValue >= minValue;
-
// build up our fixed-width "simple text packed ints"
// format
BigInteger maxBig = BigInteger.valueOf(maxValue);
@@ -201,26 +201,23 @@ public class SimpleTextSimpleDocValuesFo
final String patternString = sb.toString();
final DecimalFormat encoder = new DecimalFormat(patternString, new DecimalFormatSymbols(Locale.ROOT));
- return new NumericDocValuesConsumer() {
- int numDocsWritten = 0;
+
+ int numDocsWritten = 0;
- @Override
- public void add(long value) throws IOException {
- assert value >= minValue;
- Number delta = BigInteger.valueOf(value).subtract(BigInteger.valueOf(minValue));
- String s = encoder.format(delta);
- assert s.length() == patternString.length();
- SimpleTextUtil.write(data, s, scratch);
- SimpleTextUtil.writeNewline(data);
- numDocsWritten++;
- assert numDocsWritten <= numDocs;
- }
+ // second pass to write the values
+ for(Number n : values) {
+ long value = n.longValue();
+ assert value >= minValue;
+ Number delta = BigInteger.valueOf(value).subtract(BigInteger.valueOf(minValue));
+ String s = encoder.format(delta);
+ assert s.length() == patternString.length();
+ SimpleTextUtil.write(data, s, scratch);
+ SimpleTextUtil.writeNewline(data);
+ numDocsWritten++;
+ assert numDocsWritten <= numDocs;
+ }
- @Override
- public void finish() throws IOException {
- assert numDocs == numDocsWritten: "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten;
- }
- };
+ assert numDocs == numDocsWritten: "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten;
}
@Override
@@ -405,7 +402,6 @@ public class SimpleTextSimpleDocValuesFo
int maxLength;
boolean fixedLength;
long minValue;
- long maxValue;
int numValues;
};
@@ -444,9 +440,6 @@ public class SimpleTextSimpleDocValuesFo
assert startsWith(MINVALUE): "got " + scratch.utf8ToString() + " field=" + fieldName + " ext=" + ext;
field.minValue = Long.parseLong(stripPrefix(MINVALUE));
readLine();
- assert startsWith(MAXVALUE);
- field.maxValue = Long.parseLong(stripPrefix(MAXVALUE));
- readLine();
assert startsWith(PATTERN);
field.pattern = stripPrefix(PATTERN);
field.dataStartFilePointer = data.getFilePointer();
@@ -532,16 +525,6 @@ public class SimpleTextSimpleDocValuesFo
}
@Override
- public long minValue() {
- return field.minValue;
- }
-
- @Override
- public long maxValue() {
- return field.maxValue;
- }
-
- @Override
public int size() {
return maxDoc;
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java Mon Jan 7 19:33:25 2013
@@ -19,11 +19,12 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.NumericDocValues;
@@ -37,34 +38,79 @@ public abstract class SimpleDVConsumer i
// we want codec to get necessary stuff from IW, but trading off against merge complexity.
// nocommit should we pass SegmentWriteState...?
- public abstract NumericDocValuesConsumer addNumericField(FieldInfo field, long minValue, long maxValue) throws IOException;
+ public abstract void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException;
+
public abstract BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, int maxLength) throws IOException;
// nocommit: figure out whats fair here.
public abstract SortedDocValuesConsumer addSortedField(FieldInfo field, int valueCount, boolean fixedLength, int maxLength) throws IOException;
// dead simple impl: codec can optimize
- public void mergeNumericField(FieldInfo fieldInfo, MergeState mergeState, List<NumericDocValues> toMerge) throws IOException {
- // first compute min and max value of live ones to be merged.
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
- for (int readerIDX=0;readerIDX<toMerge.size();readerIDX++) {
- AtomicReader reader = mergeState.readers.get(readerIDX);
- int maxDoc = reader.maxDoc();
- Bits liveDocs = reader.getLiveDocs();
- NumericDocValues values = toMerge.get(readerIDX);
- for (int i = 0; i < maxDoc; i++) {
- if (liveDocs == null || liveDocs.get(i)) {
- long val = values.get(i);
- minValue = Math.min(val, minValue);
- maxValue = Math.max(val, maxValue);
- }
- mergeState.checkAbort.work(300);
- }
- }
+ public void mergeNumericField(FieldInfo fieldInfo, final MergeState mergeState, final List<NumericDocValues> toMerge) throws IOException {
- // now we can merge
- NumericDocValuesConsumer field = addNumericField(fieldInfo, minValue, maxValue);
- field.merge(mergeState, toMerge);
+ addNumericField(fieldInfo,
+ new Iterable<Number>() {
+ @Override
+ public Iterator<Number> iterator() {
+ return new Iterator<Number>() {
+ int readerUpto = -1;
+ int docIDUpto;
+ long nextValue;
+ AtomicReader currentReader;
+ NumericDocValues currentValues;
+ Bits currentLiveDocs;
+ boolean nextIsSet;
+
+ @Override
+ public boolean hasNext() {
+ return nextIsSet || setNext();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Number next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ assert nextIsSet;
+ nextIsSet = false;
+ // nocommit make a mutable number
+ return nextValue;
+ }
+
+ private boolean setNext() {
+ while (true) {
+ if (readerUpto == toMerge.size()) {
+ return false;
+ }
+
+ if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ readerUpto++;
+ if (readerUpto < toMerge.size()) {
+ currentReader = mergeState.readers.get(readerUpto);
+ currentValues = toMerge.get(readerUpto);
+ currentLiveDocs = currentReader.getLiveDocs();
+ }
+ docIDUpto = 0;
+ continue;
+ }
+
+ if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
+ nextIsSet = true;
+ nextValue = currentValues.get(docIDUpto);
+ docIDUpto++;
+ return true;
+ }
+
+ docIDUpto++;
+ }
+ }
+ };
+ }
+ });
}
// dead simple impl: codec can optimize
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java Mon Jan 7 19:33:25 2013
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.NumericDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
@@ -73,10 +72,16 @@ public class Lucene41SimpleDocValuesForm
}
@Override
- public NumericDocValuesConsumer addNumericField(FieldInfo field, final long minValue, long maxValue) throws IOException {
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
meta.writeVInt(field.number);
+ long minValue = Long.MAX_VALUE;
+ long maxValue = Long.MIN_VALUE;
+ for(Number nv : values) {
+ long v = nv.longValue();
+ minValue = Math.min(minValue, v);
+ maxValue = Math.max(maxValue, v);
+ }
meta.writeLong(minValue);
- meta.writeLong(maxValue);
long delta = maxValue - minValue;
final int bitsPerValue;
if (delta < 0) {
@@ -96,17 +101,10 @@ public class Lucene41SimpleDocValuesForm
meta.writeLong(data.getFilePointer());
final PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, formatAndBits.format, maxDoc, formatAndBits.bitsPerValue, 0);
- return new NumericDocValuesConsumer() {
- @Override
- public void add(long value) throws IOException {
- writer.add(value - minValue);
- }
-
- @Override
- public void finish() throws IOException {
- writer.finish();
- }
- };
+ for(Number nv : values) {
+ writer.add(nv.longValue() - minValue);
+ }
+ writer.finish();
}
@Override
@@ -148,7 +146,6 @@ public class Lucene41SimpleDocValuesForm
long offset;
long minValue;
- long maxValue;
PackedInts.Header header;
}
@@ -218,7 +215,6 @@ public class Lucene41SimpleDocValuesForm
static NumericEntry readNumericField(IndexInput meta) throws IOException {
NumericEntry entry = new NumericEntry();
entry.minValue = meta.readLong();
- entry.maxValue = meta.readLong();
entry.header = PackedInts.readHeader(meta);
entry.offset = meta.readLong();
return entry;
@@ -243,16 +239,6 @@ public class Lucene41SimpleDocValuesForm
}
@Override
- public long minValue() {
- return entry.minValue;
- }
-
- @Override
- public long maxValue() {
- return entry.maxValue;
- }
-
- @Override
public int size() {
return maxDoc;
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java Mon Jan 7 19:33:25 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.BinaryDocValuesConsumer;
-import org.apache.lucene.codecs.NumericDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SortedDocValuesConsumer;
import org.apache.lucene.index.FieldInfo;
@@ -82,22 +81,8 @@ public class Lucene41DocValuesConsumer e
}
@Override
- public NumericDocValuesConsumer addNumericField(FieldInfo field,
- long minValue, long maxValue) throws IOException {
- String name = getDocValuesFileName(info, field, DATA_EXTENSION);
- IndexOutput dataOut = null;
- boolean success = false;
- try {
- dataOut = getDirectory().createOutput(name, context);
- Lucene41NumericDocValuesConsumer consumer = new Lucene41NumericDocValuesConsumer(
- dataOut, minValue, maxValue, info.getDocCount());
- success = true;
- return consumer;
- } finally {
- if (!success) {
- IOUtils.close(dataOut);
- }
- }
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ // ncommit
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java Mon Jan 7 19:33:25 2013
@@ -66,11 +66,8 @@ public class Lucene41DocValuesProducer e
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
- if (DocValues.isNumber(field.getDocValuesType()) || DocValues.isFloat(field.getDocValuesType())) {
- return new Lucene41NumericDocValues.Factory(this.cfs, this.info, field, context).getDirect();
- } else {
- return null;
- }
+ // nocommit
+ return null;
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Mon Jan 7 19:33:25 2013
@@ -26,7 +26,6 @@ import java.util.ServiceLoader; // javad
import java.util.TreeMap;
import org.apache.lucene.codecs.BinaryDocValuesConsumer;
-import org.apache.lucene.codecs.NumericDocValuesConsumer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
@@ -102,8 +101,8 @@ public abstract class PerFieldDocValuesF
}
@Override
- public NumericDocValuesConsumer addNumericField(FieldInfo field, long minValue, long maxValue) throws IOException {
- return getInstance(field).addNumericField(field, minValue, maxValue);
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ getInstance(field).addNumericField(field, values);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Mon Jan 7 19:33:25 2013
@@ -31,7 +31,6 @@ import java.util.Map;
import org.apache.lucene.codecs.BlockTreeTermsReader;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat; // javadocs
-import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType; // for javadocs
import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.DocValues.Source;
@@ -1442,20 +1441,9 @@ public class CheckIndex {
if (ndv == null) {
return;
}
- final long minValue = ndv.minValue();
- final long maxValue = ndv.maxValue();
- long minValue2 = Long.MAX_VALUE;
- long maxValue2 = Long.MIN_VALUE;
+ // nocommit what else to check!
for (int i = 0; i < reader.maxDoc(); i++) {
- long value = ndv.get(i);
- minValue2 = Math.min(minValue2, value);
- maxValue2 = Math.max(maxValue2, value);
- }
- if (minValue != minValue2) {
- throw new RuntimeException("dv for field: " + fieldName + " reports minValue=" + minValue + " but this is not the case: " + minValue2);
- }
- if (maxValue != maxValue2) {
- throw new RuntimeException("dv for field: " + fieldName + " reports maxValue=" + maxValue + " but this is not the case: " + maxValue2);
+ ndv.get(i);
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java Mon Jan 7 19:33:25 2013
@@ -18,12 +18,11 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Iterator;
-import org.apache.lucene.codecs.NumericDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.packed.AppendingLongBuffer;
-import org.apache.lucene.util.packed.PackedInts;
// nocommit pick numeric or number ... then fix all places ...
@@ -40,10 +39,6 @@ class NumberDVWriter extends DocValuesWr
private long bytesUsed;
private final FieldInfo fieldInfo;
- long minValue;
- long maxValue;
- private boolean anyValues;
-
public NumberDVWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
pending = new AppendingLongBuffer();
bytesUsed = pending.ramBytesUsed();
@@ -55,12 +50,10 @@ class NumberDVWriter extends DocValuesWr
if (docID < pending.size()) {
throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)");
}
- mergeValue(value);
// Fill in any holes:
for (int i = pending.size(); i < docID; ++i) {
pending.add(MISSING);
- mergeValue(0);
}
pending.add(value);
@@ -74,42 +67,52 @@ class NumberDVWriter extends DocValuesWr
bytesUsed = newBytesUsed;
}
- private void mergeValue(long value) {
- if (!anyValues) {
- anyValues = true;
- minValue = maxValue = value;
- } else {
- maxValue = Math.max(value, maxValue);
- minValue = Math.min(value, minValue);
- }
- }
-
@Override
public void finish(int maxDoc) {
- if (pending.size() < maxDoc) {
- mergeValue(0);
- }
}
@Override
public void flush(SegmentWriteState state, SimpleDVConsumer dvConsumer) throws IOException {
- NumericDocValuesConsumer consumer = dvConsumer.addNumericField(fieldInfo, minValue, maxValue);
- final int bufferedDocCount = pending.size();
- AppendingLongBuffer.Iterator it = pending.iterator();
- for(int docID=0;docID<bufferedDocCount;docID++) {
- assert it.hasNext();
- long v = it.next();
- consumer.add(v);
- }
- assert !it.hasNext();
final int maxDoc = state.segmentInfo.getDocCount();
- for(int docID=bufferedDocCount;docID<maxDoc;docID++) {
- consumer.add(0);
- }
- consumer.finish();
+
+ dvConsumer.addNumericField(fieldInfo,
+ new Iterable<Number>() {
+
+ @Override
+ public Iterator<Number> iterator() {
+ return new Iterator<Number>() {
+ int upto;
+ AppendingLongBuffer.Iterator iter = pending.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return upto < maxDoc;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Number next() {
+ // nocommit make
+ // mutable Number:
+ long value;
+ if (upto < pending.size()) {
+ value = iter.next();
+ } else {
+ value = 0;
+ }
+ upto++;
+ return value;
+ }
+ };
+ }
+ });
+
reset();
- //System.out.println("FLUSH");
}
public void abort() {
@@ -121,8 +124,5 @@ class NumberDVWriter extends DocValuesWr
void reset() {
pending = new AppendingLongBuffer();
updateBytesUsed();
- anyValues = false;
- minValue = maxValue = 0;
}
-
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java Mon Jan 7 19:33:25 2013
@@ -20,8 +20,6 @@ package org.apache.lucene.index;
public abstract class NumericDocValues {
public abstract long get(int docID);
- public abstract long minValue();
- public abstract long maxValue();
public abstract int size();
public static final class EMPTY extends NumericDocValues {
@@ -37,16 +35,6 @@ public abstract class NumericDocValues {
}
@Override
- public long minValue() {
- return 0;
- }
-
- @Override
- public long maxValue() {
- return 0;
- }
-
- @Override
public int size() {
return size;
}
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java?rev=1429978&r1=1429977&r2=1429978&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java Mon Jan 7 19:33:25 2013
@@ -27,8 +27,6 @@ public class MultiSimpleDocValues {
public static NumericDocValues simpleNormValues(final IndexReader r, final String field) throws IOException {
final List<AtomicReaderContext> leaves = r.leaves();
boolean anyReal = false;
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
for(AtomicReaderContext ctx : leaves) {
NumericDocValues norms = ctx.reader().simpleNormValues(field);
@@ -37,16 +35,11 @@ public class MultiSimpleDocValues {
} else {
anyReal = true;
}
-
- maxValue = Math.max(norms.maxValue(), maxValue);
- minValue = Math.min(norms.minValue(), minValue);
}
if (!anyReal) {
return null;
} else {
- final long finalMaxValue = maxValue;
- final long finalMinValue = minValue;
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -68,16 +61,6 @@ public class MultiSimpleDocValues {
public int size() {
return r.maxDoc();
}
-
- @Override
- public long minValue() {
- return finalMinValue;
- }
-
- @Override
- public long maxValue() {
- return finalMaxValue;
- }
};
}
}
@@ -85,8 +68,6 @@ public class MultiSimpleDocValues {
public static NumericDocValues simpleNumericValues(final IndexReader r, final String field) throws IOException {
final List<AtomicReaderContext> leaves = r.leaves();
boolean anyReal = false;
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
for(AtomicReaderContext ctx : leaves) {
NumericDocValues values = ctx.reader().getNumericDocValues(field);
@@ -95,16 +76,11 @@ public class MultiSimpleDocValues {
} else {
anyReal = true;
}
-
- maxValue = Math.max(values.maxValue(), maxValue);
- minValue = Math.min(values.minValue(), minValue);
}
if (!anyReal) {
return null;
} else {
- final long finalMaxValue = maxValue;
- final long finalMinValue = minValue;
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -126,16 +102,6 @@ public class MultiSimpleDocValues {
public int size() {
return r.maxDoc();
}
-
- @Override
- public long minValue() {
- return finalMinValue;
- }
-
- @Override
- public long maxValue() {
- return finalMaxValue;
- }
};
}
}