You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/06/12 22:42:27 UTC
svn commit: r1602286 [2/3] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/memory/
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/ lucene/core/
lucene/core/src/java/org/apache/l...
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java Thu Jun 12 20:42:26 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
@@ -44,7 +43,6 @@ public class Lucene49DocValuesConsumer e
static final int BLOCK_SIZE = 16384;
static final int ADDRESS_INTERVAL = 16;
- static final Number MISSING_ORD = Long.valueOf(-1);
/** Compressed using packed blocks of ints. */
public static final int DELTA_COMPRESSED = 0;
@@ -64,10 +62,10 @@ public class Lucene49DocValuesConsumer e
/** Standard storage for sorted set values with 1 level of indirection:
* docId -> address -> ord. */
- public static final int SORTED_SET_WITH_ADDRESSES = 0;
+ public static final int SORTED_WITH_ADDRESSES = 0;
/** Single-valued sorted set values, encoded as sorted values, so no level
* of indirection: docId -> ord. */
- public static final int SORTED_SET_SINGLE_VALUED_SORTED = 1;
+ public static final int SORTED_SINGLE_VALUED = 1;
IndexOutput data, meta;
final int maxDoc;
@@ -366,13 +364,21 @@ public class Lucene49DocValuesConsumer e
addNumericField(field, docToOrd, false);
}
- private static boolean isSingleValued(Iterable<Number> docToOrdCount) {
- for (Number ordCount : docToOrdCount) {
- if (ordCount.longValue() > 1) {
- return false;
- }
+ @Override
+ public void addSortedNumericField(FieldInfo field, final Iterable<Number> docToValueCount, final Iterable<Number> values) throws IOException {
+ meta.writeVInt(field.number);
+ meta.writeByte(Lucene49DocValuesFormat.SORTED_NUMERIC);
+ if (isSingleValued(docToValueCount)) {
+ meta.writeVInt(SORTED_SINGLE_VALUED);
+ // The field is single-valued, we can encode it as NUMERIC
+ addNumericField(field, singletonView(docToValueCount, values, null));
+ } else {
+ meta.writeVInt(SORTED_WITH_ADDRESSES);
+ // write the stream of values as a numeric field
+ addNumericField(field, values, true);
+ // write the doc -> ord count as a absolute index to the stream
+ addAddresses(field, docToValueCount);
}
- return true;
}
@Override
@@ -381,55 +387,26 @@ public class Lucene49DocValuesConsumer e
meta.writeByte(Lucene49DocValuesFormat.SORTED_SET);
if (isSingleValued(docToOrdCount)) {
- meta.writeVInt(SORTED_SET_SINGLE_VALUED_SORTED);
+ meta.writeVInt(SORTED_SINGLE_VALUED);
// The field is single-valued, we can encode it as SORTED
- addSortedField(field, values, new Iterable<Number>() {
-
- @Override
- public Iterator<Number> iterator() {
- final Iterator<Number> docToOrdCountIt = docToOrdCount.iterator();
- final Iterator<Number> ordsIt = ords.iterator();
- return new Iterator<Number>() {
-
- @Override
- public boolean hasNext() {
- assert ordsIt.hasNext() ? docToOrdCountIt.hasNext() : true;
- return docToOrdCountIt.hasNext();
- }
-
- @Override
- public Number next() {
- final Number ordCount = docToOrdCountIt.next();
- if (ordCount.longValue() == 0) {
- return MISSING_ORD;
- } else {
- assert ordCount.longValue() == 1;
- return ordsIt.next();
- }
- }
+ addSortedField(field, values, singletonView(docToOrdCount, ords, -1L));
+ } else {
+ meta.writeVInt(SORTED_WITH_ADDRESSES);
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
+ // write the ord -> byte[] as a binary field
+ addTermsDict(field, values);
- };
- }
+ // write the stream of ords as a numeric field
+ // NOTE: we could return an iterator that delta-encodes these within a doc
+ addNumericField(field, ords, false);
- });
- return;
+ // write the doc -> ord count as a absolute index to the stream
+ addAddresses(field, docToOrdCount);
}
-
- meta.writeVInt(SORTED_SET_WITH_ADDRESSES);
-
- // write the ord -> byte[] as a binary field
- addTermsDict(field, values);
-
- // write the stream of ords as a numeric field
- // NOTE: we could return an iterator that delta-encodes these within a doc
- addNumericField(field, ords, false);
-
- // write the doc -> ord count as a absolute index to the stream
+ }
+
+ // writes addressing information as MONOTONIC_COMPRESSED integer
+ private void addAddresses(FieldInfo field, Iterable<Number> values) throws IOException {
meta.writeVInt(field.number);
meta.writeByte(Lucene49DocValuesFormat.NUMERIC);
meta.writeVInt(MONOTONIC_COMPRESSED);
@@ -442,7 +419,7 @@ public class Lucene49DocValuesConsumer e
final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
long addr = 0;
writer.add(addr);
- for (Number v : docToOrdCount) {
+ for (Number v : values) {
addr += v.longValue();
writer.add(addr);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java Thu Jun 12 20:42:26 2014
@@ -35,7 +35,7 @@ import org.apache.lucene.util.packed.Mon
/**
* Lucene 4.9 DocValues format.
* <p>
- * Encodes the four per-document value types (Numeric,Binary,Sorted,SortedSet) with these strategies:
+ * Encodes the five per-document value types (Numeric,Binary,Sorted,SortedSet,SortedNumeric) with these strategies:
* <p>
* {@link DocValuesType#NUMERIC NUMERIC}:
* <ul>
@@ -75,6 +75,12 @@ import org.apache.lucene.util.packed.Mon
* above.
* </ul>
* <p>
+ * {@link DocValuesType#SORTED_NUMERIC SORTED_NUMERIC}:
+ * <ul>
+ * <li>SortedNumeric: a value list and per-document index into this list are written using the numeric
+ * strategies above.
+ * </ul>
+ * <p>
* Files:
* <ol>
* <li><tt>.dvd</tt>: DocValues data</li>
@@ -87,7 +93,7 @@ import org.apache.lucene.util.packed.Mon
* DocValues data (.dvd)</p>
* <p>DocValues metadata (.dvm) --> Header,<Entry><sup>NumFields</sup>,Footer</p>
* <ul>
- * <li>Entry --> NumericEntry | BinaryEntry | SortedEntry | SortedSetEntry</li>
+ * <li>Entry --> NumericEntry | BinaryEntry | SortedEntry | SortedSetEntry | SortedNumericEntry</li>
* <li>NumericEntry --> GCDNumericEntry | TableNumericEntry | DeltaNumericEntry</li>
* <li>GCDNumericEntry --> NumericHeader,MinValue,GCD,BitsPerValue</li>
* <li>TableNumericEntry --> NumericHeader,TableSize,{@link DataOutput#writeLong Int64}<sup>TableSize</sup>,BitsPerValue</li>
@@ -101,6 +107,7 @@ import org.apache.lucene.util.packed.Mon
* <li>BinaryHeader --> FieldNumber,EntryType,BinaryType,MissingOffset,MinLength,MaxLength,DataOffset</li>
* <li>SortedEntry --> FieldNumber,EntryType,BinaryEntry,NumericEntry</li>
* <li>SortedSetEntry --> EntryType,BinaryEntry,NumericEntry,NumericEntry</li>
+ * <li>SortedNumericEntry --> EntryType,NumericEntry,NumericEntry</li>
* <li>FieldNumber,PackedVersion,MinLength,MaxLength,BlockSize,ValueCount --> {@link DataOutput#writeVInt VInt}</li>
* <li>EntryType,CompressionType --> {@link DataOutput#writeByte Byte}</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
@@ -112,6 +119,8 @@ import org.apache.lucene.util.packed.Mon
* and an ordinary NumericEntry for the document-to-ord metadata.</p>
* <p>SortedSet fields have three entries: a BinaryEntry with the value metadata,
* and two NumericEntries for the document-to-ord-index and ordinal list metadata.</p>
+ * <p>SortedNumeric fields have two entries: A NumericEntry with the value metadata,
+ * and a numeric entry with the document-to-value index.</p>
* <p>FieldNumber of -1 indicates the end of metadata.</p>
* <p>EntryType is a 0 (NumericEntry) or 1 (BinaryEntry)</p>
* <p>DataOffset is the pointer to the start of the data in the DocValues data (.dvd)</p>
@@ -182,4 +191,5 @@ public final class Lucene49DocValuesForm
static final byte BINARY = 1;
static final byte SORTED = 2;
static final byte SORTED_SET = 3;
+ static final byte SORTED_NUMERIC = 4;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java Thu Jun 12 20:42:26 2014
@@ -23,8 +23,8 @@ import static org.apache.lucene.codecs.l
import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.DELTA_COMPRESSED;
import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.GCD_COMPRESSED;
import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.MONOTONIC_COMPRESSED;
-import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.SORTED_SET_SINGLE_VALUED_SORTED;
-import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.SORTED_SET_WITH_ADDRESSES;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.SORTED_SINGLE_VALUED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.SORTED_WITH_ADDRESSES;
import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.TABLE_COMPRESSED;
import java.io.Closeable; // javadocs
@@ -48,6 +48,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
@@ -67,6 +68,7 @@ public class Lucene49DocValuesProducer e
private final Map<Integer,NumericEntry> numerics;
private final Map<Integer,BinaryEntry> binaries;
private final Map<Integer,SortedSetEntry> sortedSets;
+ private final Map<Integer,SortedSetEntry> sortedNumerics;
private final Map<Integer,NumericEntry> ords;
private final Map<Integer,NumericEntry> ordIndexes;
private final AtomicLong ramBytesUsed;
@@ -94,6 +96,7 @@ public class Lucene49DocValuesProducer e
ordIndexes = new HashMap<>();
binaries = new HashMap<>();
sortedSets = new HashMap<>();
+ sortedNumerics = new HashMap<>();
readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
@@ -197,9 +200,9 @@ public class Lucene49DocValuesProducer e
} else if (type == Lucene49DocValuesFormat.SORTED_SET) {
SortedSetEntry ss = readSortedSetEntry(meta);
sortedSets.put(fieldNumber, ss);
- if (ss.format == SORTED_SET_WITH_ADDRESSES) {
+ if (ss.format == SORTED_WITH_ADDRESSES) {
readSortedSetFieldWithAddresses(fieldNumber, meta, infos);
- } else if (ss.format == SORTED_SET_SINGLE_VALUED_SORTED) {
+ } else if (ss.format == SORTED_SINGLE_VALUED) {
if (meta.readVInt() != fieldNumber) {
throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
@@ -210,6 +213,28 @@ public class Lucene49DocValuesProducer e
} else {
throw new AssertionError();
}
+ } else if (type == Lucene49DocValuesFormat.SORTED_NUMERIC) {
+ SortedSetEntry ss = readSortedSetEntry(meta);
+ sortedNumerics.put(fieldNumber, ss);
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ numerics.put(fieldNumber, readNumericEntry(meta));
+ if (ss.format == SORTED_WITH_ADDRESSES) {
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry ordIndex = readNumericEntry(meta);
+ ordIndexes.put(fieldNumber, ordIndex);
+ } else if (ss.format != SORTED_SINGLE_VALUED) {
+ throw new AssertionError();
+ }
} else {
throw new CorruptIndexException("invalid type: " + type + ", resource=" + meta);
}
@@ -286,7 +311,7 @@ public class Lucene49DocValuesProducer e
SortedSetEntry readSortedSetEntry(IndexInput meta) throws IOException {
SortedSetEntry entry = new SortedSetEntry();
entry.format = meta.readVInt();
- if (entry.format != SORTED_SET_SINGLE_VALUED_SORTED && entry.format != SORTED_SET_WITH_ADDRESSES) {
+ if (entry.format != SORTED_SINGLE_VALUED && entry.format != SORTED_WITH_ADDRESSES) {
throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
}
return entry;
@@ -310,7 +335,6 @@ public class Lucene49DocValuesProducer e
LongValues getNumeric(NumericEntry entry) throws IOException {
RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset);
-
switch (entry.format) {
case DELTA_COMPRESSED:
final long delta = entry.minValue;
@@ -520,14 +544,51 @@ public class Lucene49DocValuesProducer e
}
return ordIndex;
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ SortedSetEntry ss = sortedNumerics.get(field.number);
+ NumericEntry numericEntry = numerics.get(field.number);
+ final LongValues values = getNumeric(numericEntry);
+ if (ss.format == SORTED_SINGLE_VALUED) {
+ final Bits docsWithField = getMissingBits(numericEntry.missingOffset);
+ return DocValues.singleton(values, docsWithField);
+ } else if (ss.format == SORTED_WITH_ADDRESSES) {
+ final IndexInput data = this.data.clone();
+ final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number));
+
+ return new SortedNumericDocValues() {
+ long startOffset;
+ long endOffset;
+
+ @Override
+ public void setDocument(int doc) {
+ startOffset = ordIndex.get(doc);
+ endOffset = ordIndex.get(doc+1L);
+ }
+
+ @Override
+ public long valueAt(int index) {
+ return values.get(startOffset + index);
+ }
+
+ @Override
+ public int count() {
+ return (int) (endOffset - startOffset);
+ }
+ };
+ } else {
+ throw new AssertionError();
+ }
+ }
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetEntry ss = sortedSets.get(field.number);
- if (ss.format == SORTED_SET_SINGLE_VALUED_SORTED) {
+ if (ss.format == SORTED_SINGLE_VALUED) {
final SortedDocValues values = getSorted(field);
return DocValues.singleton(values);
- } else if (ss.format != SORTED_SET_WITH_ADDRESSES) {
+ } else if (ss.format != SORTED_WITH_ADDRESSES) {
throw new AssertionError();
}
@@ -630,6 +691,8 @@ public class Lucene49DocValuesProducer e
switch(field.getDocValuesType()) {
case SORTED_SET:
return DocValues.docsWithValue(getSortedSet(field), maxDoc);
+ case SORTED_NUMERIC:
+ return DocValues.docsWithValue(getSortedNumeric(field), maxDoc);
case SORTED:
return DocValues.docsWithValue(getSorted(field), maxDoc);
case BINARY:
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsConsumer.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsConsumer.java Thu Jun 12 20:42:26 2014
@@ -205,4 +205,9 @@ class Lucene49NormsConsumer extends DocV
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new UnsupportedOperationException();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java Thu Jun 12 20:42:26 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.index.IndexFile
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
@@ -217,6 +218,11 @@ class Lucene49NormsProducer extends DocV
}
@Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ throw new IllegalStateException();
+ }
+
+ @Override
public Bits getDocsWithField(FieldInfo field) throws IOException {
throw new IllegalStateException();
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html Thu Jun 12 20:42:26 2014
@@ -386,6 +386,9 @@ allow updating NumericDocValues fields.<
<li>In version 4.8, checksum footers were added to the end of each index file
for improved data integrity. Specifically, the last 8 bytes of every index file
contain the zlib-crc32 checksum of the file.</li>
+<li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric)
+that is suitable for faceting/sorting/analytics.
+</li>
</ul>
<a name="Limitations" id="Limitations"></a>
<h2>Limitations</h2>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Thu Jun 12 20:42:26 2014
@@ -35,6 +35,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -117,6 +118,11 @@ public abstract class PerFieldDocValuesF
}
@Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ getInstance(field).addSortedNumericField(field, docToValueCount, values);
+ }
+
+ @Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
getInstance(field).addSortedSetField(field, values, docToOrdCount, ords);
}
@@ -280,6 +286,12 @@ public abstract class PerFieldDocValuesF
}
@Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ DocValuesProducer producer = fields.get(field.name);
+ return producer == null ? null : producer.getSortedNumeric(field);
+ }
+
+ @Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getSortedSet(field);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Thu Jun 12 20:42:26 2014
@@ -288,6 +288,12 @@ public abstract class AtomicReader exten
* this field. The returned instance should only be
* used by a single thread. */
public abstract SortedDocValues getSortedDocValues(String field) throws IOException;
+
+ /** Returns {@link SortedNumericDocValues} for this field, or
+ * null if no {@link SortedNumericDocValues} were indexed for
+ * this field. The returned instance should only be
+ * used by a single thread. */
+ public abstract SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException;
/** Returns {@link SortedSetDocValues} for this field, or
* null if no {@link SortedSetDocValues} were indexed for
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Thu Jun 12 20:42:26 2014
@@ -313,6 +313,9 @@ public class CheckIndex {
/** Total number of sorted fields */
public long totalSortedFields;
+ /** Total number of sortednumeric fields */
+ public long totalSortedNumericFields;
+
/** Total number of sortedset fields */
public long totalSortedSetFields;
@@ -1410,6 +1413,7 @@ public class CheckIndex {
+ status.totalBinaryFields + " BINARY; "
+ status.totalNumericFields + " NUMERIC; "
+ status.totalSortedFields + " SORTED; "
+ + status.totalSortedNumericFields + " SORTED_NUMERIC; "
+ status.totalSortedSetFields + " SORTED_SET]");
} catch (Throwable e) {
msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
@@ -1540,6 +1544,30 @@ public class CheckIndex {
lastValue = BytesRef.deepCopyOf(term);
}
}
+
+ private static void checkSortedNumericDocValues(String fieldName, AtomicReader reader, SortedNumericDocValues ndv, Bits docsWithField) {
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ ndv.setDocument(i);
+ int count = ndv.count();
+ if (docsWithField.get(i)) {
+ if (count == 0) {
+ throw new RuntimeException("dv for field: " + fieldName + " is not marked missing but has zero count for doc: " + i);
+ }
+ long previous = Long.MIN_VALUE;
+ for (int j = 0; j < count; j++) {
+ long value = ndv.valueAt(j);
+ if (value < previous) {
+ throw new RuntimeException("values out of order: " + value + " < " + previous + " for doc: " + i);
+ }
+ previous = value;
+ }
+ } else {
+ if (count != 0) {
+ throw new RuntimeException("dv for field: " + fieldName + " is marked missing but has count=" + count + " for doc: " + i);
+ }
+ }
+ }
+ }
private static void checkNumericDocValues(String fieldName, AtomicReader reader, NumericDocValues ndv, Bits docsWithField) {
for (int i = 0; i < reader.maxDoc(); i++) {
@@ -1563,15 +1591,27 @@ public class CheckIndex {
checkSortedDocValues(fi.name, reader, reader.getSortedDocValues(fi.name), docsWithField);
if (reader.getBinaryDocValues(fi.name) != null ||
reader.getNumericDocValues(fi.name) != null ||
+ reader.getSortedNumericDocValues(fi.name) != null ||
reader.getSortedSetDocValues(fi.name) != null) {
throw new RuntimeException(fi.name + " returns multiple docvalues types!");
}
break;
+ case SORTED_NUMERIC:
+ status.totalSortedNumericFields++;
+ checkSortedNumericDocValues(fi.name, reader, reader.getSortedNumericDocValues(fi.name), docsWithField);
+ if (reader.getBinaryDocValues(fi.name) != null ||
+ reader.getNumericDocValues(fi.name) != null ||
+ reader.getSortedSetDocValues(fi.name) != null ||
+ reader.getSortedDocValues(fi.name) != null) {
+ throw new RuntimeException(fi.name + " returns multiple docvalues types!");
+ }
+ break;
case SORTED_SET:
status.totalSortedSetFields++;
checkSortedSetDocValues(fi.name, reader, reader.getSortedSetDocValues(fi.name), docsWithField);
if (reader.getBinaryDocValues(fi.name) != null ||
reader.getNumericDocValues(fi.name) != null ||
+ reader.getSortedNumericDocValues(fi.name) != null ||
reader.getSortedDocValues(fi.name) != null) {
throw new RuntimeException(fi.name + " returns multiple docvalues types!");
}
@@ -1581,6 +1621,7 @@ public class CheckIndex {
checkBinaryDocValues(fi.name, reader, reader.getBinaryDocValues(fi.name), docsWithField);
if (reader.getNumericDocValues(fi.name) != null ||
reader.getSortedDocValues(fi.name) != null ||
+ reader.getSortedNumericDocValues(fi.name) != null ||
reader.getSortedSetDocValues(fi.name) != null) {
throw new RuntimeException(fi.name + " returns multiple docvalues types!");
}
@@ -1590,6 +1631,7 @@ public class CheckIndex {
checkNumericDocValues(fi.name, reader, reader.getNumericDocValues(fi.name), docsWithField);
if (reader.getBinaryDocValues(fi.name) != null ||
reader.getSortedDocValues(fi.name) != null ||
+ reader.getSortedNumericDocValues(fi.name) != null ||
reader.getSortedSetDocValues(fi.name) != null) {
throw new RuntimeException(fi.name + " returns multiple docvalues types!");
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Thu Jun 12 20:42:26 2014
@@ -64,9 +64,6 @@ final class DefaultIndexingChain extends
private int totalFieldCount;
private long nextFieldGen;
- // Lazy init:
- private NumericDocValuesWriter norms;
-
// Holds fields seen in each document
private PerField[] fields = new PerField[1];
@@ -437,6 +434,13 @@ final class DefaultIndexingChain extends
}
((SortedDocValuesWriter) fp.docValuesWriter).addValue(docID, field.binaryValue());
break;
+
+ case SORTED_NUMERIC:
+ if (fp.docValuesWriter == null) {
+ fp.docValuesWriter = new SortedNumericDocValuesWriter(fp.fieldInfo, bytesUsed);
+ }
+ ((SortedNumericDocValuesWriter) fp.docValuesWriter).addValue(docID, field.numericValue().longValue());
+ break;
case SORTED_SET:
if (fp.docValuesWriter == null) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValues.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValues.java Thu Jun 12 20:42:26 2014
@@ -77,6 +77,26 @@ public final class DocValues {
}
};
}
+
+ /**
+ * An empty SortedNumericDocValues which returns zero values for every document
+ */
+ public static final SortedNumericDocValues emptySortedNumeric() {
+ return new SortedNumericDocValues() {
+ @Override
+ public void setDocument(int doc) {}
+
+ @Override
+ public long valueAt(int index) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ @Override
+ public int count() {
+ return 0;
+ }
+ };
+ }
/**
* An empty SortedDocValues which returns {@link SortedSetDocValues#NO_MORE_ORDS} for every document
@@ -122,7 +142,7 @@ public final class DocValues {
/**
* Returns a single-valued view of the SortedSetDocValues, if it was previously
- * wrapped with {@link #singleton}, or null.
+ * wrapped with {@link #singleton(SortedDocValues)}, or null.
*/
public static SortedDocValues unwrapSingleton(SortedSetDocValues dv) {
if (dv instanceof SingletonSortedSetDocValues) {
@@ -132,6 +152,38 @@ public final class DocValues {
}
}
+ /**
+ * Returns a single-valued view of the SortedNumericDocValues, if it was previously
+ * wrapped with {@link #singleton(NumericDocValues, Bits)}, or null.
+ * @see #unwrapSingletonBits(SortedNumericDocValues)
+ */
+ public static NumericDocValues unwrapSingleton(SortedNumericDocValues dv) {
+ if (dv instanceof SingletonSortedNumericDocValues) {
+ return ((SingletonSortedNumericDocValues)dv).getNumericDocValues();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the documents with a value for the SortedNumericDocValues, if it was previously
+ * wrapped with {@link #singleton(NumericDocValues, Bits)}, or null.
+ */
+ public static Bits unwrapSingletonBits(SortedNumericDocValues dv) {
+ if (dv instanceof SingletonSortedNumericDocValues) {
+ return ((SingletonSortedNumericDocValues)dv).getDocsWithField();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns a multi-valued view over the provided NumericDocValues
+ */
+ public static SortedNumericDocValues singleton(NumericDocValues dv, Bits docsWithField) {
+ return new SingletonSortedNumericDocValues(dv, docsWithField);
+ }
+
/**
* Returns a Bits representing all documents from <code>dv</code> that have a value.
*/
@@ -167,6 +219,24 @@ public final class DocValues {
};
}
+ /**
+ * Returns a Bits representing all documents from <code>dv</code> that have a value.
+ */
+ public static Bits docsWithValue(final SortedNumericDocValues dv, final int maxDoc) {
+ return new Bits() {
+ @Override
+ public boolean get(int index) {
+ dv.setDocument(index);
+ return dv.count() != 0;
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ };
+ }
+
// some helpers, for transition from fieldcache apis.
// as opposed to the AtomicReader apis (which must be strict for consistency), these are lenient
@@ -209,6 +279,22 @@ public final class DocValues {
}
/**
+ * Returns SortedNumericDocValues for the reader, or {@link #emptySortedNumeric} if it has none.
+ */
+ public static SortedNumericDocValues getSortedNumeric(AtomicReader in, String field) throws IOException {
+ SortedNumericDocValues dv = in.getSortedNumericDocValues(field);
+ if (dv == null) {
+ NumericDocValues single = in.getNumericDocValues(field);
+ if (single == null) {
+ return emptySortedNumeric();
+ }
+ Bits bits = in.getDocsWithField(field);
+ return singleton(single, bits);
+ }
+ return dv;
+ }
+
+ /**
* Returns SortedSetDocValues for the reader, or {@link #emptySortedSet} if it has none.
*/
public static SortedSetDocValues getSortedSet(AtomicReader in, String field) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Thu Jun 12 20:42:26 2014
@@ -106,6 +106,11 @@ public final class FieldInfo {
*/
SORTED,
/**
+ * A pre-sorted Number[]. Fields with this type store numeric values in sorted
+ * order according to {@link Long#compare(long, long)}.
+ */
+ SORTED_NUMERIC,
+ /**
* A pre-sorted Set<byte[]>. Fields with this type only store distinct byte values
* and store additional offset pointers per document to dereference the shared
* byte[]s. The stored byte[] is presorted and allows access via document id,
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Thu Jun 12 20:42:26 2014
@@ -425,6 +425,12 @@ public class FilterAtomicReader extends
ensureOpen();
return in.getSortedDocValues(field);
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ ensureOpen();
+ return in.getSortedNumericDocValues(field);
+ }
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java Thu Jun 12 20:42:26 2014
@@ -228,6 +228,63 @@ public class MultiDocValues {
}
}
+ /** Returns a SortedNumericDocValues for a reader's docvalues (potentially merging on-the-fly)
+ * <p>
+ * This is a slow way to access sorted numeric values. Instead, access them per-segment
+ * with {@link AtomicReader#getSortedNumericDocValues(String)}
+ * </p>
+ * */
+ public static SortedNumericDocValues getSortedNumericValues(final IndexReader r, final String field) throws IOException {
+ final List<AtomicReaderContext> leaves = r.leaves();
+ final int size = leaves.size();
+ if (size == 0) {
+ return null;
+ } else if (size == 1) {
+ return leaves.get(0).reader().getSortedNumericDocValues(field);
+ }
+
+ boolean anyReal = false;
+ final SortedNumericDocValues[] values = new SortedNumericDocValues[size];
+ final int[] starts = new int[size+1];
+ for (int i = 0; i < size; i++) {
+ AtomicReaderContext context = leaves.get(i);
+ SortedNumericDocValues v = context.reader().getSortedNumericDocValues(field);
+ if (v == null) {
+ v = DocValues.emptySortedNumeric();
+ } else {
+ anyReal = true;
+ }
+ values[i] = v;
+ starts[i] = context.docBase;
+ }
+ starts[size] = r.maxDoc();
+
+ if (!anyReal) {
+ return null;
+ } else {
+ return new SortedNumericDocValues() {
+ SortedNumericDocValues current;
+
+ @Override
+ public void setDocument(int doc) {
+ int subIndex = ReaderUtil.subIndex(doc, starts);
+ current = values[subIndex];
+ current.setDocument(doc - starts[subIndex]);
+ }
+
+ @Override
+ public long valueAt(int index) {
+ return current.valueAt(index);
+ }
+
+ @Override
+ public int count() {
+ return current.count();
+ }
+ };
+ }
+ }
+
/** Returns a SortedDocValues for a reader's docvalues (potentially doing extremely slow things).
* <p>
* This is an extremely slow way to access sorted values. Instead, access them per-segment
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java Thu Jun 12 20:42:26 2014
@@ -287,6 +287,13 @@ public class ParallelAtomicReader extend
AtomicReader reader = fieldToReader.get(field);
return reader == null ? null : reader.getSortedDocValues(field);
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ ensureOpen();
+ AtomicReader reader = fieldToReader.get(field);
+ return reader == null ? null : reader.getSortedNumericDocValues(field);
+ }
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Thu Jun 12 20:42:26 2014
@@ -211,6 +211,16 @@ final class SegmentMerger {
toMerge.add(values);
}
consumer.mergeSortedSetField(field, mergeState, toMerge);
+ } else if (type == DocValuesType.SORTED_NUMERIC) {
+ List<SortedNumericDocValues> toMerge = new ArrayList<>();
+ for (AtomicReader reader : mergeState.readers) {
+ SortedNumericDocValues values = reader.getSortedNumericDocValues(field.name);
+ if (values == null) {
+ values = DocValues.emptySortedNumeric();
+ }
+ toMerge.add(values);
+ }
+ consumer.mergeSortedNumericField(field, mergeState, toMerge);
} else {
throw new AssertionError("type=" + type);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Thu Jun 12 20:42:26 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -538,6 +537,27 @@ public final class SegmentReader extends
return dv;
}
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ ensureOpen();
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ Object previous = dvFields.get(field);
+ if (previous != null && previous instanceof SortedNumericDocValues) {
+ return (SortedNumericDocValues) previous;
+ } else {
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED_NUMERIC);
+ if (fi == null) {
+ return null;
+ }
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
+ SortedNumericDocValues dv = dvProducer.getSortedNumeric(fi);
+ dvFields.put(field, dv);
+ return dv;
+ }
+ }
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java Thu Jun 12 20:42:26 2014
@@ -65,6 +65,9 @@ public class SimpleMergedSegmentWarmer e
case SORTED:
reader.getSortedDocValues(info.name);
break;
+ case SORTED_NUMERIC:
+ reader.getSortedNumericDocValues(info.name);
+ break;
case SORTED_SET:
reader.getSortedSetDocValues(info.name);
break;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Thu Jun 12 20:42:26 2014
@@ -110,6 +110,12 @@ public final class SlowCompositeReaderWr
ensureOpen();
return MultiDocValues.getBinaryValues(in, field);
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ ensureOpen();
+ return MultiDocValues.getSortedNumericValues(in, field);
+ }
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Thu Jun 12 20:42:26 2014
@@ -331,14 +331,14 @@ public abstract class FieldComparator<T>
/** Parses field's values as double (using {@link
* FieldCache#getDoubles} and sorts by ascending value */
- public static final class DoubleComparator extends NumericComparator<Double> {
+ public static class DoubleComparator extends NumericComparator<Double> {
private final double[] values;
private final DoubleParser parser;
private FieldCache.Doubles currentReaderValues;
private double bottom;
private double topValue;
- DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) {
+ public DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) {
super(field, missingValue);
values = new double[numHits];
this.parser = (DoubleParser) parser;
@@ -377,10 +377,15 @@ public abstract class FieldComparator<T>
public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
// NOTE: must do this before calling super otherwise
// we compute the docsWithField Bits twice!
- currentReaderValues = FieldCache.DEFAULT.getDoubles(context.reader(), field, parser, missingValue != null);
+ currentReaderValues = getDoubleValues(context, field);
return super.setNextReader(context);
}
+ /** Expert: provides source of doubles for sorting */
+ protected FieldCache.Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException {
+ return FieldCache.DEFAULT.getDoubles(context.reader(), field, parser, missingValue != null);
+ }
+
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
@@ -410,14 +415,14 @@ public abstract class FieldComparator<T>
/** Parses field's values as float (using {@link
* FieldCache#getFloats} and sorts by ascending value */
- public static final class FloatComparator extends NumericComparator<Float> {
+ public static class FloatComparator extends NumericComparator<Float> {
private final float[] values;
private final FloatParser parser;
private FieldCache.Floats currentReaderValues;
private float bottom;
private float topValue;
- FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) {
+ public FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) {
super(field, missingValue);
values = new float[numHits];
this.parser = (FloatParser) parser;
@@ -457,10 +462,15 @@ public abstract class FieldComparator<T>
public FieldComparator<Float> setNextReader(AtomicReaderContext context) throws IOException {
// NOTE: must do this before calling super otherwise
// we compute the docsWithField Bits twice!
- currentReaderValues = FieldCache.DEFAULT.getFloats(context.reader(), field, parser, missingValue != null);
+ currentReaderValues = getFloatValues(context, field);
return super.setNextReader(context);
}
+ /** Expert: provides source of floats for sorting */
+ protected FieldCache.Floats getFloatValues(AtomicReaderContext context, String field) throws IOException {
+ return FieldCache.DEFAULT.getFloats(context.reader(), field, parser, missingValue != null);
+ }
+
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
@@ -570,14 +580,14 @@ public abstract class FieldComparator<T>
/** Parses field's values as int (using {@link
* FieldCache#getInts} and sorts by ascending value */
- public static final class IntComparator extends NumericComparator<Integer> {
+ public static class IntComparator extends NumericComparator<Integer> {
private final int[] values;
private final IntParser parser;
private FieldCache.Ints currentReaderValues;
private int bottom; // Value of bottom of queue
private int topValue;
- IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) {
+ public IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) {
super(field, missingValue);
values = new int[numHits];
this.parser = (IntParser) parser;
@@ -616,10 +626,15 @@ public abstract class FieldComparator<T>
public FieldComparator<Integer> setNextReader(AtomicReaderContext context) throws IOException {
// NOTE: must do this before calling super otherwise
// we compute the docsWithField Bits twice!
- currentReaderValues = FieldCache.DEFAULT.getInts(context.reader(), field, parser, missingValue != null);
+ currentReaderValues = getIntValues(context, field);
return super.setNextReader(context);
}
+ /** Expert: provides source of ints for sorting */
+ protected FieldCache.Ints getIntValues(AtomicReaderContext context, String field) throws IOException {
+ return FieldCache.DEFAULT.getInts(context.reader(), field, parser, missingValue != null);
+ }
+
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
@@ -649,14 +664,14 @@ public abstract class FieldComparator<T>
/** Parses field's values as long (using {@link
* FieldCache#getLongs} and sorts by ascending value */
- public static final class LongComparator extends NumericComparator<Long> {
+ public static class LongComparator extends NumericComparator<Long> {
private final long[] values;
private final LongParser parser;
private FieldCache.Longs currentReaderValues;
private long bottom;
private long topValue;
- LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) {
+ public LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) {
super(field, missingValue);
values = new long[numHits];
this.parser = (LongParser) parser;
@@ -697,10 +712,15 @@ public abstract class FieldComparator<T>
public FieldComparator<Long> setNextReader(AtomicReaderContext context) throws IOException {
// NOTE: must do this before calling super otherwise
// we compute the docsWithField Bits twice!
- currentReaderValues = FieldCache.DEFAULT.getLongs(context.reader(), field, parser, missingValue != null);
+ currentReaderValues = getLongValues(context, field);
return super.setNextReader(context);
}
+ /** Expert: provides source of longs for sorting */
+ protected FieldCache.Longs getLongValues(AtomicReaderContext context, String field) throws IOException {
+ return FieldCache.DEFAULT.getLongs(context.reader(), field, parser, missingValue != null);
+ }
+
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
Copied: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java (from r1602277, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java?p2=lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java&r1=1602277&r2=1602286&rev=1602286&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java Thu Jun 12 20:42:26 2014
@@ -137,31 +137,55 @@ public class SortedNumericSortField exte
public FieldComparator<?> getComparator(int numHits, int sortPos) throws IOException {
switch(type) {
case INT:
- return new FieldComparator.IntComparator(numHits, getField(), (Integer) missingValue) {
+ return new FieldComparator.IntComparator(numHits, getField(), null, (Integer) missingValue) {
@Override
- protected NumericDocValues getNumericDocValues(AtomicReaderContext context, String field) throws IOException {
- return SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ protected FieldCache.Ints getIntValues(AtomicReaderContext context, String field) throws IOException {
+ final NumericDocValues dv = SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ return new FieldCache.Ints() {
+ @Override
+ public int get(int docID) {
+ return (int) dv.get(docID);
+ }
+ };
}
};
case FLOAT:
- return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
+ return new FieldComparator.FloatComparator(numHits, getField(), null, (Float) missingValue) {
@Override
- protected NumericDocValues getNumericDocValues(AtomicReaderContext context, String field) throws IOException {
- return SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ protected FieldCache.Floats getFloatValues(AtomicReaderContext context, String field) throws IOException {
+ final NumericDocValues dv = SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ return new FieldCache.Floats() {
+ @Override
+ public float get(int docID) {
+ return Float.intBitsToFloat((int)dv.get(docID));
+ }
+ };
}
};
case LONG:
- return new FieldComparator.LongComparator(numHits, getField(), (Long) missingValue) {
+ return new FieldComparator.LongComparator(numHits, getField(), null, (Long) missingValue) {
@Override
- protected NumericDocValues getNumericDocValues(AtomicReaderContext context, String field) throws IOException {
- return SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ protected FieldCache.Longs getLongValues(AtomicReaderContext context, String field) throws IOException {
+ final NumericDocValues dv = SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ return new FieldCache.Longs() {
+ @Override
+ public long get(int docID) {
+ return dv.get(docID);
+ }
+ };
}
};
case DOUBLE:
- return new FieldComparator.DoubleComparator(numHits, getField(), (Double) missingValue) {
+ return new FieldComparator.DoubleComparator(numHits, getField(), null, (Double) missingValue) {
@Override
- protected NumericDocValues getNumericDocValues(AtomicReaderContext context, String field) throws IOException {
- return SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ protected FieldCache.Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException {
+ final NumericDocValues dv = SortedNumericSelector.wrap(DocValues.getSortedNumeric(context.reader(), field), selector, type);
+ return new FieldCache.Doubles() {
+ @Override
+ public double get(int docID) {
+ return Double.longBitsToDouble(dv.get(docID));
+ }
+ };
}
};
default:
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java Thu Jun 12 20:42:26 2014
@@ -274,9 +274,7 @@ public final class NumericUtils {
* @see #sortableLongToDouble
*/
public static long doubleToSortableLong(double val) {
- long f = Double.doubleToLongBits(val);
- if (f<0) f ^= 0x7fffffffffffffffL;
- return f;
+ return sortableDoubleBits(Double.doubleToLongBits(val));
}
/**
@@ -284,8 +282,7 @@ public final class NumericUtils {
* @see #doubleToSortableLong
*/
public static double sortableLongToDouble(long val) {
- if (val<0) val ^= 0x7fffffffffffffffL;
- return Double.longBitsToDouble(val);
+ return Double.longBitsToDouble(sortableDoubleBits(val));
}
/**
@@ -298,9 +295,7 @@ public final class NumericUtils {
* @see #sortableIntToFloat
*/
public static int floatToSortableInt(float val) {
- int f = Float.floatToIntBits(val);
- if (f<0) f ^= 0x7fffffff;
- return f;
+ return sortableFloatBits(Float.floatToIntBits(val));
}
/**
@@ -308,8 +303,17 @@ public final class NumericUtils {
* @see #floatToSortableInt
*/
public static float sortableIntToFloat(int val) {
- if (val<0) val ^= 0x7fffffff;
- return Float.intBitsToFloat(val);
+ return Float.intBitsToFloat(sortableFloatBits(val));
+ }
+
+ /** Converts IEEE 754 representation of a double to sortable order (or back to the original) */
+ public static long sortableDoubleBits(long bits) {
+ return bits ^ (bits >> 63) & 0x7fffffffffffffffL;
+ }
+
+ /** Converts IEEE 754 representation of a float to sortable order (or back to the original) */
+ public static int sortableFloatBits(int bits) {
+ return bits ^ (bits >> 31) & 0x7fffffff;
}
/**
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java Thu Jun 12 20:42:26 2014
@@ -19,14 +19,19 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.junit.BeforeClass;
/**
* Tests Lucene45DocValuesFormat
*/
public class TestLucene45DocValuesFormat extends BaseCompressingDocValuesFormatTestCase {
- private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene45DocValuesFormat());
-
+ private final Codec codec = new Lucene45RWCodec();
+
+ @BeforeClass
+ public static void beforeClass() {
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true; // explicitly instantiates ancient codec
+ }
+
@Override
protected Codec getCodec() {
return codec;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Thu Jun 12 20:42:26 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -142,6 +143,14 @@ public class TestDuelingCodecs extends L
if (random.nextInt(4) == 2) {
document.add(new NumericDocValuesField("sparsenumeric", random.nextInt()));
}
+ // add sortednumeric sometimes
+ document.removeFields("sparsesortednum");
+ if (random.nextInt(5) == 1) {
+ document.add(new SortedNumericDocValuesField("sparsesortednum", random.nextLong()));
+ if (random.nextBoolean()) {
+ document.add(new SortedNumericDocValuesField("sparsesortednum", random.nextLong()));
+ }
+ }
writer.addDocument(document);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Jun 12 20:42:26 2014
@@ -49,6 +49,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@@ -1020,6 +1021,10 @@ public class TestIndexWriter extends Luc
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("one")));
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("two")));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 4));
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 3));
+ }
w.addDocument(doc);
doc = new Document();
doc.add(newStringField(random, "id", "501", Field.Store.NO));
@@ -1033,6 +1038,10 @@ public class TestIndexWriter extends Luc
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("two")));
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("three")));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 6));
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 1));
+ }
w.addDocument(doc);
w.deleteDocuments(new Term("id", "500"));
w.close();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu Jun 12 20:42:26 2014
@@ -42,6 +42,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@@ -161,6 +162,10 @@ public class TestIndexWriterExceptions e
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("hellllo")));
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("again")));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 10));
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 5));
+ }
doc.add(newField(r, "content7", "aaa bbb ccc ddd", DocCopyIterator.custom4));
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java Thu Jun 12 20:42:26 2014
@@ -37,6 +37,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
@@ -112,6 +113,10 @@ public class TestIndexWriterExceptions2
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i))));
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i-1))));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("dv5", i));
+ doc.add(new SortedNumericDocValuesField("dv5", i-1));
+ }
doc.add(newTextField("text1", TestUtil.randomAnalysisString(random(), 20, true), Field.Store.NO));
// ensure we store something
doc.add(new StoredField("stored1", "foo"));
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java Thu Jun 12 20:42:26 2014
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
@@ -132,6 +133,10 @@ public class TestIndexWriterOutOfMemory
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i))));
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i-1))));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("dv5", i));
+ doc.add(new SortedNumericDocValuesField("dv5", i-1));
+ }
doc.add(newTextField("text1", TestUtil.randomAnalysisString(random(), 20, true), Field.Store.NO));
// ensure we store something
doc.add(new StoredField("stored1", "foo"));
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java Thu Jun 12 20:42:26 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
@@ -319,6 +320,58 @@ public class TestMultiDocValues extends
dir.close();
}
+ public void testSortedNumeric() throws Exception {
+ assumeTrue("codec does not support SORTED_NUMERIC", defaultCodecSupportsSortedNumeric());
+ Directory dir = newDirectory();
+
+ IndexWriterConfig iwc = newIndexWriterConfig(random(), TEST_VERSION_CURRENT, null);
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ int numDocs = atLeast(500);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ int numValues = random().nextInt(5);
+ for (int j = 0; j < numValues; j++) {
+ doc.add(new SortedNumericDocValuesField("nums", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
+ }
+ iw.addDocument(doc);
+ if (random().nextInt(17) == 0) {
+ iw.commit();
+ }
+ }
+ DirectoryReader ir = iw.getReader();
+ iw.forceMerge(1);
+ DirectoryReader ir2 = iw.getReader();
+ AtomicReader merged = getOnlySegmentReader(ir2);
+ iw.close();
+
+ SortedNumericDocValues multi = MultiDocValues.getSortedNumericValues(ir, "nums");
+ SortedNumericDocValues single = merged.getSortedNumericDocValues("nums");
+ if (multi == null) {
+ assertNull(single);
+ } else {
+ // check values
+ for (int i = 0; i < numDocs; i++) {
+ single.setDocument(i);
+ ArrayList<Long> expectedList = new ArrayList<>();
+ for (int j = 0; j < single.count(); j++) {
+ expectedList.add(single.valueAt(j));
+ }
+
+ multi.setDocument(i);
+ assertEquals(expectedList.size(), multi.count());
+ for (int j = 0; j < single.count(); j++) {
+ assertEquals(expectedList.get(j).longValue(), multi.valueAt(j));
+ }
+ }
+ }
+
+ ir.close();
+ ir2.close();
+ dir.close();
+ }
+
public void testDocsWithField() throws Exception {
assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
Copied: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java (from r1602277, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java?p2=lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java&r1=1602277&r2=1602286&rev=1602286&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java Thu Jun 12 20:42:26 2014
@@ -39,7 +39,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.NumericUtils;
/** Simple tests for SortedNumericSortField */
-@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42", "Lucene45", "Lucene46"})
+@SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45", "Lucene46"})
// avoid codecs that don't support sortednumeric
public class TestSortedNumericSortField extends LuceneTestCase {
@@ -90,7 +90,7 @@ public class TestSortedNumericSortField
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortedNumericSortField("value", SortField.Type.INT));
@@ -119,7 +119,7 @@ public class TestSortedNumericSortField
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortedNumericSortField("value", SortField.Type.INT, true));
@@ -150,7 +150,7 @@ public class TestSortedNumericSortField
doc.add(newStringField("id", "3", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sortField = new SortedNumericSortField("value", SortField.Type.INT);
@@ -185,7 +185,7 @@ public class TestSortedNumericSortField
doc.add(newStringField("id", "3", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sortField = new SortedNumericSortField("value", SortField.Type.INT);
@@ -216,7 +216,7 @@ public class TestSortedNumericSortField
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortedNumericSortField("value", SortField.Type.INT));
@@ -244,7 +244,7 @@ public class TestSortedNumericSortField
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortedNumericSortField("value", SortField.Type.FLOAT));
@@ -272,7 +272,7 @@ public class TestSortedNumericSortField
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
- writer.shutdown();
+ writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortedNumericSortField("value", SortField.Type.DOUBLE));
Modified: lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu Jun 12 20:42:26 2014
@@ -45,6 +45,7 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.TermState;
@@ -795,6 +796,11 @@ public class MemoryIndex {
}
@Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) {
+ return null;
+ }
+
+ @Override
public SortedSetDocValues getSortedSetDocValues(String field) {
return null;
}
Modified: lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Thu Jun 12 20:42:26 2014
@@ -146,7 +146,7 @@ public class TestDocValuesFieldSources e
public void test() throws IOException {
for (DocValuesType type : DocValuesType.values()) {
- if (type != DocValuesType.SORTED_SET) {
+ if (type != DocValuesType.SORTED_SET && type != DocValuesType.SORTED_NUMERIC) {
test(type);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Thu Jun 12 20:42:26 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -131,6 +132,30 @@ public class AssertingDocValuesFormat ex
}
@Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ long valueCount = 0;
+ Iterator<Number> valueIterator = values.iterator();
+ for (Number count : docToValueCount) {
+ assert count != null;
+ assert count.intValue() >= 0;
+ valueCount += count.intValue();
+ long previous = Long.MIN_VALUE;
+ for (int i = 0; i < count.intValue(); i++) {
+ assert valueIterator.hasNext();
+ Number next = valueIterator.next();
+ assert next != null;
+ long nextValue = next.longValue();
+ assert nextValue >= previous;
+ previous = nextValue;
+ }
+ }
+ assert valueIterator.hasNext() == false;
+ checkIterator(docToValueCount.iterator(), maxDoc, false);
+ checkIterator(values.iterator(), valueCount, false);
+ in.addSortedNumericField(field, docToValueCount, values);
+ }
+
+ @Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
long valueCount = 0;
BytesRef lastValue = null;
@@ -217,6 +242,11 @@ public class AssertingDocValuesFormat ex
public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
throw new IllegalStateException();
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new IllegalStateException();
+ }
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
@@ -281,6 +311,14 @@ public class AssertingDocValuesFormat ex
}
@Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_NUMERIC;
+ SortedNumericDocValues values = in.getSortedNumeric(field);
+ assert values != null;
+ return new AssertingAtomicReader.AssertingSortedNumericDocValues(values, maxDoc);
+ }
+
+ @Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_SET;
SortedSetDocValues values = in.getSortedSet(field);
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyDocValuesFormat.java Thu Jun 12 20:42:26 2014
@@ -93,6 +93,14 @@ class CrankyDocValuesFormat extends DocV
}
delegate.addSortedField(field, values, docToOrd);
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException from DocValuesConsumer.addSortedNumericField()");
+ }
+ delegate.addSortedNumericField(field, docToValueCount, values);
+ }
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java Thu Jun 12 20:42:26 2014
@@ -95,4 +95,9 @@ class PreFlexRWNormsConsumer extends Doc
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
throw new AssertionError();
}
+
+ @Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new AssertionError();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java?rev=1602286&r1=1602285&r2=1602286&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java Thu Jun 12 20:42:26 2014
@@ -531,6 +531,11 @@ class Lucene40DocValuesWriter extends Do
}
@Override
+ public void addSortedNumericField(FieldInfo field, Iterable<Number> docToValueCount, Iterable<Number> values) throws IOException {
+ throw new UnsupportedOperationException("Lucene 4.0 does not support SortedNumeric docvalues");
+ }
+
+ @Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
throw new UnsupportedOperationException("Lucene 4.0 does not support SortedSet docvalues");
}