You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/06/16 21:26:23 UTC
svn commit: r1602968 [3/5] - in /lucene/dev/branches/lucene5752: ./
dev-tools/ dev-tools/idea/.idea/ lucene/ lucene/analysis/
lucene/analysis/common/
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/
lucene/codecs/ lucene/codecs/src/...
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Mon Jun 16 19:26:19 2014
@@ -178,18 +178,20 @@ class ReadersAndUpdates {
/**
* Returns a ref to a clone. NOTE: you should decRef() the reader when you're
- * dont (ie do not call close()).
+ * done (ie do not call close()).
*/
public synchronized SegmentReader getReadOnlyClone(IOContext context) throws IOException {
if (reader == null) {
getReader(context).decRef();
assert reader != null;
}
+ // force new liveDocs in initWritableLiveDocs even if it's null
liveDocsShared = true;
if (liveDocs != null) {
return new SegmentReader(reader.getSegmentInfo(), reader, liveDocs, info.info.getDocCount() - info.getDelCount() - pendingDeleteCount);
} else {
- assert reader.getLiveDocs() == liveDocs;
+ // liveDocs == null and reader != null. That can only be if there are no deletes
+ assert reader.getLiveDocs() == null;
reader.incRef();
return reader;
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Mon Jun 16 19:26:19 2014
@@ -209,6 +209,16 @@ final class SegmentMerger {
toMerge.add(values);
}
consumer.mergeSortedSetField(field, mergeState, toMerge);
+ } else if (type == DocValuesType.SORTED_NUMERIC) {
+ List<SortedNumericDocValues> toMerge = new ArrayList<>();
+ for (AtomicReader reader : mergeState.readers) {
+ SortedNumericDocValues values = reader.getSortedNumericDocValues(field.name);
+ if (values == null) {
+ values = DocValues.emptySortedNumeric();
+ }
+ toMerge.add(values);
+ }
+ consumer.mergeSortedNumericField(field, mergeState, toMerge);
} else {
throw new AssertionError("type=" + type);
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Mon Jun 16 19:26:19 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -531,6 +530,27 @@ public final class SegmentReader extends
return dv;
}
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ ensureOpen();
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ Object previous = dvFields.get(field);
+ if (previous != null && previous instanceof SortedNumericDocValues) {
+ return (SortedNumericDocValues) previous;
+ } else {
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED_NUMERIC);
+ if (fi == null) {
+ return null;
+ }
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
+ SortedNumericDocValues dv = dvProducer.getSortedNumeric(fi);
+ dvFields.put(field, dv);
+ return dv;
+ }
+ }
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java Mon Jun 16 19:26:19 2014
@@ -65,6 +65,9 @@ public class SimpleMergedSegmentWarmer e
case SORTED:
reader.getSortedDocValues(info.name);
break;
+ case SORTED_NUMERIC:
+ reader.getSortedNumericDocValues(info.name);
+ break;
case SORTED_SET:
reader.getSortedSetDocValues(info.name);
break;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Mon Jun 16 19:26:19 2014
@@ -110,6 +110,12 @@ public final class SlowCompositeReaderWr
ensureOpen();
return MultiDocValues.getBinaryValues(in, field);
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+ ensureOpen();
+ return MultiDocValues.getSortedNumericValues(in, field);
+ }
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Mon Jun 16 19:26:19 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import static org.apache.lucene.search.DocIdSet.EMPTY;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -28,7 +30,6 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.WAH8DocIdSet;
/**
@@ -60,14 +61,14 @@ public class CachingWrapperFilter extend
* by the wrapped Filter. <p>This implementation returns the given {@link DocIdSet},
* if {@link DocIdSet#isCacheable} returns <code>true</code>, else it calls
* {@link #cacheImpl(DocIdSetIterator,AtomicReader)}
- * <p>Note: This method returns {@linkplain #EMPTY_DOCIDSET} if the given docIdSet
+ * <p>Note: This method returns {@linkplain DocIdSet#EMPTY} if the given docIdSet
* is <code>null</code> or if {@link DocIdSet#iterator()} return <code>null</code>. The empty
* instance is use as a placeholder in the cache instead of the <code>null</code> value.
*/
protected DocIdSet docIdSetToCache(DocIdSet docIdSet, AtomicReader reader) throws IOException {
if (docIdSet == null) {
// this is better than returning null, as the nonnull result can be cached
- return EMPTY_DOCIDSET;
+ return EMPTY;
} else if (docIdSet.isCacheable()) {
return docIdSet;
} else {
@@ -76,7 +77,7 @@ public class CachingWrapperFilter extend
// in this case we wrap with the sentinel set,
// which is cacheable.
if (it == null) {
- return EMPTY_DOCIDSET;
+ return EMPTY;
} else {
return cacheImpl(it, reader);
}
@@ -110,7 +111,7 @@ public class CachingWrapperFilter extend
cache.put(key, docIdSet);
}
- return docIdSet == EMPTY_DOCIDSET ? null : BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
+ return docIdSet == EMPTY ? null : BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
}
@Override
@@ -129,26 +130,6 @@ public class CachingWrapperFilter extend
public int hashCode() {
return (filter.hashCode() ^ getClass().hashCode());
}
-
- /** An empty {@code DocIdSet} instance */
- protected static final DocIdSet EMPTY_DOCIDSET = new DocIdSet() {
-
- @Override
- public DocIdSetIterator iterator() {
- return DocIdSetIterator.empty();
- }
-
- @Override
- public boolean isCacheable() {
- return true;
- }
-
- // we explicitly provide no random access, as this filter is 100% sparse and iterator exits faster
- @Override
- public Bits bits() {
- return null;
- }
- };
@Override
public long ramBytesUsed() {
@@ -161,7 +142,7 @@ public class CachingWrapperFilter extend
long total = 0;
for(DocIdSet dis : docIdSets) {
- total += RamUsageEstimator.sizeOf(dis);
+ total += dis.ramBytesUsed();
}
return total;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java Mon Jun 16 19:26:19 2014
@@ -18,13 +18,40 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
/**
* A DocIdSet contains a set of doc ids. Implementing classes must
* only implement {@link #iterator} to provide access to the set.
*/
-public abstract class DocIdSet {
+public abstract class DocIdSet implements Accountable {
+
+ /** An empty {@code DocIdSet} instance */
+ public static final DocIdSet EMPTY = new DocIdSet() {
+
+ @Override
+ public DocIdSetIterator iterator() {
+ return DocIdSetIterator.empty();
+ }
+
+ @Override
+ public boolean isCacheable() {
+ return true;
+ }
+
+ // we explicitly provide no random access, as this filter is 100% sparse and iterator exits faster
+ @Override
+ public Bits bits() {
+ return null;
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
+ };
/** Provides a {@link DocIdSetIterator} to access the set.
* This implementation can return <code>null</code> if there
@@ -65,4 +92,5 @@ public abstract class DocIdSet {
public boolean isCacheable() {
return false;
}
+
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java Mon Jun 16 19:26:19 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
@@ -55,6 +56,11 @@ public abstract class DocValuesDocIdSet
}
@Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
+
+ @Override
public final Bits bits() {
return (acceptDocs == null) ? new Bits() {
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Mon Jun 16 19:26:19 2014
@@ -226,6 +226,7 @@ public abstract class FieldComparator<T>
protected final T missingValue;
protected final String field;
protected Bits docsWithField;
+ protected NumericDocValues currentReaderValues;
public NumericComparator(String field, T missingValue) {
this.field = field;
@@ -234,6 +235,7 @@ public abstract class FieldComparator<T>
@Override
public FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
+ currentReaderValues = getNumericDocValues(context, field);
if (missingValue != null) {
docsWithField = DocValues.getDocsWithField(context.reader(), field);
// optimization to remove unneeded checks on the bit interface:
@@ -245,13 +247,17 @@ public abstract class FieldComparator<T>
}
return this;
}
+
+ /** Retrieves the NumericDocValues for the field in this segment */
+ protected NumericDocValues getNumericDocValues(AtomicReaderContext context, String field) throws IOException {
+ return DocValues.getNumeric(context.reader(), field);
+ }
}
/** Parses field's values as double (using {@link
* AtomicReader#getNumericDocValues} and sorts by ascending value */
- public static final class DoubleComparator extends NumericComparator<Double> {
+ public static class DoubleComparator extends NumericComparator<Double> {
private final double[] values;
- private NumericDocValues currentReaderValues;
private double bottom;
private double topValue;
@@ -288,12 +294,6 @@ public abstract class FieldComparator<T>
values[slot] = v2;
}
-
- @Override
- public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
- currentReaderValues = DocValues.getNumeric(context.reader(), field);
- return super.setNextReader(context);
- }
@Override
public void setBottom(final int bottom) {
@@ -324,9 +324,8 @@ public abstract class FieldComparator<T>
/** Parses field's values as float (using {@link
* AtomicReader#getNumericDocValues(String)} and sorts by ascending value */
- public static final class FloatComparator extends NumericComparator<Float> {
+ public static class FloatComparator extends NumericComparator<Float> {
private final float[] values;
- private NumericDocValues currentReaderValues;
private float bottom;
private float topValue;
@@ -364,12 +363,6 @@ public abstract class FieldComparator<T>
values[slot] = v2;
}
-
- @Override
- public FieldComparator<Float> setNextReader(AtomicReaderContext context) throws IOException {
- currentReaderValues = DocValues.getNumeric(context.reader(), field);
- return super.setNextReader(context);
- }
@Override
public void setBottom(final int bottom) {
@@ -400,9 +393,8 @@ public abstract class FieldComparator<T>
/** Parses field's values as int (using {@link
* AtomicReader#getNumericDocValues(String)} and sorts by ascending value */
- public static final class IntComparator extends NumericComparator<Integer> {
+ public static class IntComparator extends NumericComparator<Integer> {
private final int[] values;
- private NumericDocValues currentReaderValues;
private int bottom; // Value of bottom of queue
private int topValue;
@@ -439,12 +431,6 @@ public abstract class FieldComparator<T>
values[slot] = v2;
}
-
- @Override
- public FieldComparator<Integer> setNextReader(AtomicReaderContext context) throws IOException {
- currentReaderValues = DocValues.getNumeric(context.reader(), field);
- return super.setNextReader(context);
- }
@Override
public void setBottom(final int bottom) {
@@ -475,9 +461,8 @@ public abstract class FieldComparator<T>
/** Parses field's values as long (using {@link
* AtomicReader#getNumericDocValues(String)} and sorts by ascending value */
- public static final class LongComparator extends NumericComparator<Long> {
+ public static class LongComparator extends NumericComparator<Long> {
private final long[] values;
- private NumericDocValues currentReaderValues;
private long bottom;
private long topValue;
@@ -516,12 +501,6 @@ public abstract class FieldComparator<T>
values[slot] = v2;
}
-
- @Override
- public FieldComparator<Long> setNextReader(AtomicReaderContext context) throws IOException {
- currentReaderValues = DocValues.getNumeric(context.reader(), field);
- return super.setNextReader(context);
- }
@Override
public void setBottom(final int bottom) {
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java Mon Jun 16 19:26:19 2014
@@ -18,7 +18,9 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Abstract decorator class for a DocIdSet implementation
@@ -55,7 +57,12 @@ public abstract class FilteredDocIdSet e
public boolean isCacheable() {
return _innerSet.isCacheable();
}
-
+
+ @Override
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_OBJECT_REF + _innerSet.ramBytesUsed();
+ }
+
@Override
public Bits bits() throws IOException {
final Bits bits = _innerSet.bits();
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java Mon Jun 16 19:26:19 2014
@@ -58,8 +58,11 @@ public class QueryWrapperFilter extends
public DocIdSetIterator iterator() throws IOException {
return weight.scorer(privateContext, acceptDocs);
}
+
@Override
- public boolean isCacheable() { return false; }
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java Mon Jun 16 19:26:19 2014
@@ -21,11 +21,14 @@ import java.io.EOFException;
import java.io.IOException;
/** Base implementation class for buffered {@link IndexInput}. */
-public abstract class BufferedIndexInput extends IndexInput {
+public abstract class BufferedIndexInput extends IndexInput implements RandomAccessInput {
/** Default buffer size set to {@value #BUFFER_SIZE}. */
public static final int BUFFER_SIZE = 1024;
+ /** Minimum buffer size allowed */
+ public static final int MIN_BUFFER_SIZE = 8;
+
// The normal read buffer size defaults to 1024, but
// increasing this during merging seems to yield
// performance gains. However we don't want to increase
@@ -104,8 +107,8 @@ public abstract class BufferedIndexInput
}
private void checkBufferSize(int bufferSize) {
- if (bufferSize <= 0)
- throw new IllegalArgumentException("bufferSize must be greater than 0 (got " + bufferSize + ")");
+ if (bufferSize <= MIN_BUFFER_SIZE)
+ throw new IllegalArgumentException("bufferSize must be greater than MIN_BUFFER_SIZE (got " + bufferSize + ")");
}
@Override
@@ -255,6 +258,74 @@ public abstract class BufferedIndexInput
}
}
+ @Override
+ public final byte readByte(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ return buffer[(int)index];
+ }
+
+ @Override
+ public final short readShort(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength-1) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ return (short) (((buffer[(int)index] & 0xFF) << 8) |
+ (buffer[(int)index+1] & 0xFF));
+ }
+
+ @Override
+ public final int readInt(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength-3) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ return ((buffer[(int)index] & 0xFF) << 24) |
+ ((buffer[(int)index+1] & 0xFF) << 16) |
+ ((buffer[(int)index+2] & 0xFF) << 8) |
+ (buffer[(int)index+3] & 0xFF);
+ }
+
+ @Override
+ public final long readLong(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength-7) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ final int i1 = ((buffer[(int)index] & 0xFF) << 24) |
+ ((buffer[(int)index+1] & 0xFF) << 16) |
+ ((buffer[(int)index+2] & 0xFF) << 8) |
+ (buffer[(int)index+3] & 0xFF);
+ final int i2 = ((buffer[(int)index+4] & 0xFF) << 24) |
+ ((buffer[(int)index+5] & 0xFF) << 16) |
+ ((buffer[(int)index+6] & 0xFF) << 8) |
+ (buffer[(int)index+7] & 0xFF);
+ return (((long)i1) << 32) | (i2 & 0xFFFFFFFFL);
+ }
+
private void refill() throws IOException {
long start = bufferStart + bufferPosition;
long end = start + bufferSize;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java Mon Jun 16 19:26:19 2014
@@ -277,12 +277,6 @@ abstract class ByteBufferIndexInput exte
return buildSlice(sliceDescription, offset, length);
}
-
- @Override
- public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
- // note: technically we could even avoid the clone...
- return slice(null, offset, length);
- }
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). **/
protected ByteBufferIndexInput buildSlice(String sliceDescription, long offset, long length) {
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/IndexInput.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/store/IndexInput.java Mon Jun 16 19:26:19 2014
@@ -92,35 +92,41 @@ public abstract class IndexInput extends
/**
* Creates a random-access slice of this index input, with the given offset and length.
* <p>
- * The default implementation calls {@link #slice}, and implements absolute reads as
- * seek+read.
+ * The default implementation calls {@link #slice}, and it doesn't support random access,
+ * it implements absolute reads as seek+read.
*/
public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
final IndexInput slice = slice("randomaccess", offset, length);
- return new RandomAccessInput() {
- @Override
- public byte readByte(long pos) throws IOException {
- slice.seek(pos);
- return slice.readByte();
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- slice.seek(pos);
- return slice.readShort();
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- slice.seek(pos);
- return slice.readInt();
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- slice.seek(pos);
- return slice.readLong();
- }
- };
+ if (slice instanceof RandomAccessInput) {
+ // slice() already supports random access
+ return (RandomAccessInput) slice;
+ } else {
+ // return default impl
+ return new RandomAccessInput() {
+ @Override
+ public byte readByte(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readByte();
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readShort();
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readInt();
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readLong();
+ }
+ };
+ }
}
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/DocIdBitSet.java Mon Jun 16 19:26:19 2014
@@ -25,6 +25,12 @@ import org.apache.lucene.search.DocIdSet
/** Simple DocIdSet and DocIdSetIterator backed by a BitSet */
public class DocIdBitSet extends DocIdSet implements Bits {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(DocIdBitSet.class)
+ + RamUsageEstimator.shallowSizeOfInstance(BitSet.class)
+ + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; // the array that stores the bits
+
private final BitSet bitSet;
public DocIdBitSet(BitSet bitSet) {
@@ -64,7 +70,14 @@ public class DocIdBitSet extends DocIdSe
// the size may not be correct...
return bitSet.size();
}
-
+
+ @Override
+ public long ramBytesUsed() {
+ // unfortunately this is likely underestimated if the Bitset implementation
+ // over-sizes the array that stores the bits
+ return BASE_RAM_BYTES_USED + (bitSet.size() + 7) >>> 3;
+ }
+
private static class DocIdBitSetIterator extends DocIdSetIterator {
private int docId;
private BitSet bitSet;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Mon Jun 16 19:26:19 2014
@@ -33,6 +33,8 @@ import org.apache.lucene.search.DocIdSet
*/
public final class FixedBitSet extends DocIdSet implements Bits {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);
+
/**
* A {@link DocIdSetIterator} which iterates over set bits in a
* {@link FixedBitSet}.
@@ -218,6 +220,11 @@ public final class FixedBitSet extends D
return true;
}
+ @Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(bits);
+ }
+
/** Expert. */
public long[] getBits() {
return bits;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java Mon Jun 16 19:26:19 2014
@@ -274,9 +274,7 @@ public final class NumericUtils {
* @see #sortableLongToDouble
*/
public static long doubleToSortableLong(double val) {
- long f = Double.doubleToLongBits(val);
- if (f<0) f ^= 0x7fffffffffffffffL;
- return f;
+ return sortableDoubleBits(Double.doubleToLongBits(val));
}
/**
@@ -284,8 +282,7 @@ public final class NumericUtils {
* @see #doubleToSortableLong
*/
public static double sortableLongToDouble(long val) {
- if (val<0) val ^= 0x7fffffffffffffffL;
- return Double.longBitsToDouble(val);
+ return Double.longBitsToDouble(sortableDoubleBits(val));
}
/**
@@ -298,9 +295,7 @@ public final class NumericUtils {
* @see #sortableIntToFloat
*/
public static int floatToSortableInt(float val) {
- int f = Float.floatToIntBits(val);
- if (f<0) f ^= 0x7fffffff;
- return f;
+ return sortableFloatBits(Float.floatToIntBits(val));
}
/**
@@ -308,8 +303,17 @@ public final class NumericUtils {
* @see #floatToSortableInt
*/
public static float sortableIntToFloat(int val) {
- if (val<0) val ^= 0x7fffffff;
- return Float.intBitsToFloat(val);
+ return Float.intBitsToFloat(sortableFloatBits(val));
+ }
+
+ /** Converts IEEE 754 representation of a double to sortable order (or back to the original) */
+ public static long sortableDoubleBits(long bits) {
+ return bits ^ (bits >> 63) & 0x7fffffffffffffffL;
+ }
+
+ /** Converts IEEE 754 representation of a float to sortable order (or back to the original) */
+ public static int sortableFloatBits(int bits) {
+ return bits ^ (bits >> 31) & 0x7fffffff;
}
/**
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/OpenBitSet.java Mon Jun 16 19:26:19 2014
@@ -75,6 +75,9 @@ Test system: AMD Opteron, 64 bit linux,
*/
public class OpenBitSet extends DocIdSet implements Bits, Cloneable {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OpenBitSet.class);
+
protected long[] bits;
protected int wlen; // number of words (elements) used in the array
@@ -131,6 +134,11 @@ public class OpenBitSet extends DocIdSet
return true;
}
+ @Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(bits);
+ }
+
/** Returns the current capacity in bits (1 greater than the index of the last bit) */
public long capacity() { return bits.length << 6; }
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PForDeltaDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PForDeltaDocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PForDeltaDocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PForDeltaDocIdSet.java Mon Jun 16 19:26:19 2014
@@ -35,6 +35,8 @@ import org.apache.lucene.util.packed.Pac
*/
public final class PForDeltaDocIdSet extends DocIdSet implements Accountable {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(PForDeltaDocIdSet.class);
+
static final int BLOCK_SIZE = 128;
static final int MAX_EXCEPTIONS = 24; // no more than 24 exceptions per block
static final PackedInts.Decoder[] DECODERS = new PackedInts.Decoder[32];
@@ -513,7 +515,17 @@ public final class PForDeltaDocIdSet ext
@Override
public long ramBytesUsed() {
- return RamUsageEstimator.alignObjectSize(3 * RamUsageEstimator.NUM_BYTES_OBJECT_REF) + docIDs.ramBytesUsed() + offsets.ramBytesUsed();
+ if (this == EMPTY) {
+ return 0L;
+ }
+ long ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(data);
+ if (docIDs != SINGLE_ZERO_BUFFER) {
+ ramBytesUsed += docIDs.ramBytesUsed();
+ }
+ if (offsets != SINGLE_ZERO_BUFFER) {
+ ramBytesUsed += offsets.ramBytesUsed();
+ }
+ return ramBytesUsed;
}
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java Mon Jun 16 19:26:19 2014
@@ -27,7 +27,6 @@ import java.util.*;
/**
* Estimates the size (memory representation) of Java objects.
*
- * @see #sizeOf(Object)
* @see #shallowSizeOf(Object)
* @see #shallowSizeOfInstance(Class)
*
@@ -235,18 +234,6 @@ public final class RamUsageEstimator {
Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
}
- /**
- * Cached information about a given class.
- */
- private static final class ClassCache {
- public final long alignedShallowInstanceSize;
- public final Field[] referenceFields;
-
- public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) {
- this.alignedShallowInstanceSize = alignedShallowInstanceSize;
- this.referenceFields = referenceFields;
- }
- }
// Object with just one field to determine the object header size by getting the offset of the dummy field:
@SuppressWarnings("unused")
@@ -320,17 +307,10 @@ public final class RamUsageEstimator {
return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
}
- /**
- * Estimates the RAM usage by the given object. It will
- * walk the object tree and sum up all referenced objects.
- *
- * <p><b>Resource Usage:</b> This method internally uses a set of
- * every object seen during traversals so it does allocate memory
- * (it isn't side-effect free). After the method exits, this memory
- * should be GCed.</p>
- */
- public static long sizeOf(Object obj) {
- return measureObjectSize(obj);
+ /** Returns the shallow size in bytes of the Object[] object. */
+ // Use this method instead of #shallowSizeOf(Object) to avoid costly reflection
+ public static long shallowSizeOf(Object[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * arr.length);
}
/**
@@ -395,119 +375,6 @@ public final class RamUsageEstimator {
return alignObjectSize(size);
}
- /*
- * Non-recursive version of object descend. This consumes more memory than recursive in-depth
- * traversal but prevents stack overflows on long chains of objects
- * or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
- * so not too much).
- */
- private static long measureObjectSize(Object root) {
- // Objects seen so far.
- final IdentityHashSet<Object> seen = new IdentityHashSet<>();
- // Class cache with reference Field and precalculated shallow size.
- final IdentityHashMap<Class<?>, ClassCache> classCache = new IdentityHashMap<>();
- // Stack of objects pending traversal. Recursion caused stack overflows.
- final ArrayList<Object> stack = new ArrayList<>();
- stack.add(root);
-
- long totalSize = 0;
- while (!stack.isEmpty()) {
- final Object ob = stack.remove(stack.size() - 1);
-
- if (ob == null || seen.contains(ob)) {
- continue;
- }
- seen.add(ob);
-
- final Class<?> obClazz = ob.getClass();
- assert obClazz != null : "jvm bug detected (Object.getClass() == null). please report this to your vendor";
- if (obClazz.isArray()) {
- /*
- * Consider an array, possibly of primitive types. Push any of its references to
- * the processing stack and accumulate this array's shallow size.
- */
- long size = NUM_BYTES_ARRAY_HEADER;
- final int len = Array.getLength(ob);
- if (len > 0) {
- Class<?> componentClazz = obClazz.getComponentType();
- if (componentClazz.isPrimitive()) {
- size += (long) len * primitiveSizes.get(componentClazz);
- } else {
- size += (long) NUM_BYTES_OBJECT_REF * len;
-
- // Push refs for traversal later.
- for (int i = len; --i >= 0 ;) {
- final Object o = Array.get(ob, i);
- if (o != null && !seen.contains(o)) {
- stack.add(o);
- }
- }
- }
- }
- totalSize += alignObjectSize(size);
- } else {
- /*
- * Consider an object. Push any references it has to the processing stack
- * and accumulate this object's shallow size.
- */
- try {
- ClassCache cachedInfo = classCache.get(obClazz);
- if (cachedInfo == null) {
- classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
- }
-
- for (Field f : cachedInfo.referenceFields) {
- // Fast path to eliminate redundancies.
- final Object o = f.get(ob);
- if (o != null && !seen.contains(o)) {
- stack.add(o);
- }
- }
-
- totalSize += cachedInfo.alignedShallowInstanceSize;
- } catch (IllegalAccessException e) {
- // this should never happen as we enabled setAccessible().
- throw new RuntimeException("Reflective field access failed?", e);
- }
- }
- }
-
- // Help the GC (?).
- seen.clear();
- stack.clear();
- classCache.clear();
-
- return totalSize;
- }
-
- /**
- * Create a cached information about shallow size and reference fields for
- * a given class.
- */
- private static ClassCache createCacheEntry(final Class<?> clazz) {
- ClassCache cachedInfo;
- long shallowInstanceSize = NUM_BYTES_OBJECT_HEADER;
- final ArrayList<Field> referenceFields = new ArrayList<>(32);
- for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
- final Field[] fields = c.getDeclaredFields();
- for (final Field f : fields) {
- if (!Modifier.isStatic(f.getModifiers())) {
- shallowInstanceSize = adjustForField(shallowInstanceSize, f);
-
- if (!f.getType().isPrimitive()) {
- f.setAccessible(true);
- referenceFields.add(f);
- }
- }
- }
- }
-
- cachedInfo = new ClassCache(
- alignObjectSize(shallowInstanceSize),
- referenceFields.toArray(new Field[referenceFields.size()]));
- return cachedInfo;
- }
-
/**
* This method returns the maximum representation size of an object. <code>sizeSoFar</code>
* is the object's size measured so far. <code>f</code> is the field being probed.
@@ -515,7 +382,7 @@ public final class RamUsageEstimator {
* <p>The returned offset will be the maximum of whatever was measured so far and
* <code>f</code> field's offset and representation size (unaligned).
*/
- private static long adjustForField(long sizeSoFar, final Field f) {
+ static long adjustForField(long sizeSoFar, final Field f) {
final Class<?> type = f.getType();
final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF;
if (objectFieldOffsetMethod != null) {
@@ -577,252 +444,4 @@ public final class RamUsageEstimator {
return bytes + " bytes";
}
}
-
- /**
- * Return a human-readable size of a given object.
- * @see #sizeOf(Object)
- * @see #humanReadableUnits(long)
- */
- public static String humanSizeOf(Object object) {
- return humanReadableUnits(sizeOf(object));
- }
-
- /**
- * An identity hash set implemented using open addressing. No null keys are allowed.
- *
- * TODO: If this is useful outside this class, make it public - needs some work
- */
- static final class IdentityHashSet<KType> implements Iterable<KType> {
- /**
- * Default load factor.
- */
- public final static float DEFAULT_LOAD_FACTOR = 0.75f;
-
- /**
- * Minimum capacity for the set.
- */
- public final static int MIN_CAPACITY = 4;
-
- /**
- * All of set entries. Always of power of two length.
- */
- public Object[] keys;
-
- /**
- * Cached number of assigned slots.
- */
- public int assigned;
-
- /**
- * The load factor for this set (fraction of allocated or deleted slots before
- * the buffers must be rehashed or reallocated).
- */
- public final float loadFactor;
-
- /**
- * Cached capacity threshold at which we must resize the buffers.
- */
- private int resizeThreshold;
-
- /**
- * Creates a hash set with the default capacity of 16.
- * load factor of {@value #DEFAULT_LOAD_FACTOR}. `
- */
- public IdentityHashSet() {
- this(16, DEFAULT_LOAD_FACTOR);
- }
-
- /**
- * Creates a hash set with the given capacity, load factor of
- * {@value #DEFAULT_LOAD_FACTOR}.
- */
- public IdentityHashSet(int initialCapacity) {
- this(initialCapacity, DEFAULT_LOAD_FACTOR);
- }
-
- /**
- * Creates a hash set with the given capacity and load factor.
- */
- public IdentityHashSet(int initialCapacity, float loadFactor) {
- initialCapacity = Math.max(MIN_CAPACITY, initialCapacity);
-
- assert initialCapacity > 0 : "Initial capacity must be between (0, "
- + Integer.MAX_VALUE + "].";
- assert loadFactor > 0 && loadFactor < 1 : "Load factor must be between (0, 1).";
- this.loadFactor = loadFactor;
- allocateBuffers(roundCapacity(initialCapacity));
- }
-
- /**
- * Adds a reference to the set. Null keys are not allowed.
- */
- public boolean add(KType e) {
- assert e != null : "Null keys not allowed.";
-
- if (assigned >= resizeThreshold) expandAndRehash();
-
- final int mask = keys.length - 1;
- int slot = rehash(e) & mask;
- Object existing;
- while ((existing = keys[slot]) != null) {
- if (e == existing) {
- return false; // already found.
- }
- slot = (slot + 1) & mask;
- }
- assigned++;
- keys[slot] = e;
- return true;
- }
-
- /**
- * Checks if the set contains a given ref.
- */
- public boolean contains(KType e) {
- final int mask = keys.length - 1;
- int slot = rehash(e) & mask;
- Object existing;
- while ((existing = keys[slot]) != null) {
- if (e == existing) {
- return true;
- }
- slot = (slot + 1) & mask;
- }
- return false;
- }
-
- /** Rehash via MurmurHash.
- *
- * <p>The implementation is based on the
- * finalization step from Austin Appleby's
- * <code>MurmurHash3</code>.
- *
- * @see "http://sites.google.com/site/murmurhash/"
- */
- private static int rehash(Object o) {
- int k = System.identityHashCode(o);
- k ^= k >>> 16;
- k *= 0x85ebca6b;
- k ^= k >>> 13;
- k *= 0xc2b2ae35;
- k ^= k >>> 16;
- return k;
- }
-
- /**
- * Expand the internal storage buffers (capacity) or rehash current keys and
- * values if there are a lot of deleted slots.
- */
- private void expandAndRehash() {
- final Object[] oldKeys = this.keys;
-
- assert assigned >= resizeThreshold;
- allocateBuffers(nextCapacity(keys.length));
-
- /*
- * Rehash all assigned slots from the old hash table.
- */
- final int mask = keys.length - 1;
- for (int i = 0; i < oldKeys.length; i++) {
- final Object key = oldKeys[i];
- if (key != null) {
- int slot = rehash(key) & mask;
- while (keys[slot] != null) {
- slot = (slot + 1) & mask;
- }
- keys[slot] = key;
- }
- }
- Arrays.fill(oldKeys, null);
- }
-
- /**
- * Allocate internal buffers for a given capacity.
- *
- * @param capacity
- * New capacity (must be a power of two).
- */
- private void allocateBuffers(int capacity) {
- this.keys = new Object[capacity];
- this.resizeThreshold = (int) (capacity * DEFAULT_LOAD_FACTOR);
- }
-
- /**
- * Return the next possible capacity, counting from the current buffers' size.
- */
- protected int nextCapacity(int current) {
- assert current > 0 && Long.bitCount(current) == 1 : "Capacity must be a power of two.";
- assert ((current << 1) > 0) : "Maximum capacity exceeded ("
- + (0x80000000 >>> 1) + ").";
-
- if (current < MIN_CAPACITY / 2) current = MIN_CAPACITY / 2;
- return current << 1;
- }
-
- /**
- * Round the capacity to the next allowed value.
- */
- protected int roundCapacity(int requestedCapacity) {
- // Maximum positive integer that is a power of two.
- if (requestedCapacity > (0x80000000 >>> 1)) return (0x80000000 >>> 1);
-
- int capacity = MIN_CAPACITY;
- while (capacity < requestedCapacity) {
- capacity <<= 1;
- }
-
- return capacity;
- }
-
- public void clear() {
- assigned = 0;
- Arrays.fill(keys, null);
- }
-
- public int size() {
- return assigned;
- }
-
- public boolean isEmpty() {
- return size() == 0;
- }
-
- @Override
- public Iterator<KType> iterator() {
- return new Iterator<KType>() {
- int pos = -1;
- Object nextElement = fetchNext();
-
- @Override
- public boolean hasNext() {
- return nextElement != null;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public KType next() {
- Object r = this.nextElement;
- if (r == null) {
- throw new NoSuchElementException();
- }
- this.nextElement = fetchNext();
- return (KType) r;
- }
-
- private Object fetchNext() {
- pos++;
- while (pos < keys.length && keys[pos] == null) {
- pos++;
- }
-
- return (pos >= keys.length ? null : keys[pos]);
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/SentinelIntSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/SentinelIntSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/SentinelIntSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/SentinelIntSet.java Mon Jun 16 19:26:19 2014
@@ -145,4 +145,12 @@ public class SentinelIntSet {
}
rehashCount = newSize - (newSize>>2);
}
+
+ /** Return the memory footprint of this class in bytes. */
+ public long ramBytesUsed() {
+ return RamUsageEstimator.alignObjectSize(
+ RamUsageEstimator.NUM_BYTES_INT * 3
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF)
+ + RamUsageEstimator.sizeOf(keys);
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/WAH8DocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/WAH8DocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/WAH8DocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/WAH8DocIdSet.java Mon Jun 16 19:26:19 2014
@@ -76,6 +76,8 @@ import org.apache.lucene.util.packed.Pac
*/
public final class WAH8DocIdSet extends DocIdSet implements Accountable {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(WAH8DocIdSet.class);
+
// Minimum index interval, intervals below this value can't guarantee anymore
// that this set implementation won't be significantly larger than a FixedBitSet
// The reason is that a single sequence saves at least one byte and an index
@@ -738,10 +740,17 @@ public final class WAH8DocIdSet extends
@Override
public long ramBytesUsed() {
- return RamUsageEstimator.alignObjectSize(3 * RamUsageEstimator.NUM_BYTES_OBJECT_REF + 2 * RamUsageEstimator.NUM_BYTES_INT)
- + RamUsageEstimator.sizeOf(data)
- + positions.ramBytesUsed()
- + wordNums.ramBytesUsed();
+ if (this == EMPTY) {
+ return 0L;
+ }
+ long ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(data);
+ if (positions != SINGLE_ZERO_BUFFER) {
+ ramBytesUsed += positions.ramBytesUsed();
+ }
+ if (wordNums != SINGLE_ZERO_BUFFER) {
+ ramBytesUsed += wordNums.ramBytesUsed();
+ }
+ return ramBytesUsed;
}
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java Mon Jun 16 19:26:19 2014
@@ -194,7 +194,7 @@ abstract class AbstractAppendingLongBuff
// TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
+ (pending != null ? RamUsageEstimator.sizeOf(pending) : 0L)
- + RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * values.length); // values
+ + RamUsageEstimator.shallowSizeOf(values);
return bytesUsed + valuesBytes;
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java Mon Jun 16 19:26:19 2014
@@ -107,7 +107,7 @@ abstract class AbstractPagedMutable<T ex
/** Return the number of bytes used by this object. */
public long ramBytesUsed() {
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed());
- bytesUsed += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * subMutables.length);
+ bytesUsed += RamUsageEstimator.alignObjectSize(RamUsageEstimator.shallowSizeOf(subMutables));
for (PackedInts.Mutable gw : subMutables) {
bytesUsed += gw.ramBytesUsed();
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java Mon Jun 16 19:26:19 2014
@@ -104,7 +104,7 @@ public final class AppendingDeltaPackedL
values[valuesOff] = new PackedInts.NullReader(pendingOff);
} else {
// build a new packed reader
- final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
+ final int bitsRequired = PackedInts.unsignedBitsRequired(delta);
for (int i = 0; i < pendingOff; ++i) {
pending[i] -= minValue;
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java Mon Jun 16 19:26:19 2014
@@ -76,7 +76,7 @@ public final class BlockPackedWriter ext
}
final long delta = max - min;
- int bitsRequired = delta < 0 ? 64 : delta == 0L ? 0 : PackedInts.bitsRequired(delta);
+ int bitsRequired = delta == 0 ? 0 : PackedInts.unsignedBitsRequired(delta);
if (bitsRequired == 64) {
// no need to delta-encode
min = 0L;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java Mon Jun 16 19:26:19 2014
@@ -221,7 +221,7 @@ class BulkOperationPacked extends BulkOp
int bitsLeft = 8;
for (int i = 0; i < byteValueCount * iterations; ++i) {
final long v = values[valuesOffset++];
- assert bitsPerValue == 64 || PackedInts.bitsRequired(v) <= bitsPerValue;
+ assert PackedInts.unsignedBitsRequired(v) <= bitsPerValue;
if (bitsPerValue < bitsLeft) {
// just buffer
nextBlock |= v << (bitsLeft - bitsPerValue);
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java Mon Jun 16 19:26:19 2014
@@ -111,17 +111,14 @@ public final class DirectWriter {
}
/**
- * Returns how many bits are required to hold values up
- * to and including maxValue
+ * Round a number of bits per value to the next amount of bits per value that
+ * is supported by this writer.
*
- * @param maxValue the maximum value that should be representable.
- * @return the amount of bits needed to represent values from 0 to maxValue.
+ * @param bitsRequired the amount of bits required
+ * @return the next number of bits per value that is gte the provided value
+ * and supported by this writer
*/
- public static int bitsRequired(long maxValue) {
- if (maxValue < 0) {
- throw new IllegalArgumentException("maxValue must be non-negative (got: " + maxValue + ")");
- }
- int bitsRequired = Math.max(1, 64 - Long.numberOfLeadingZeros(maxValue));
+ private static int roundBits(int bitsRequired) {
int index = Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsRequired);
if (index < 0) {
return SUPPORTED_BITS_PER_VALUE[-index-1];
@@ -129,7 +126,31 @@ public final class DirectWriter {
return bitsRequired;
}
}
-
+
+ /**
+ * Returns how many bits are required to hold values up
+ * to and including maxValue
+ *
+ * @param maxValue the maximum value that should be representable.
+ * @return the amount of bits needed to represent values from 0 to maxValue.
+ * @see PackedInts#bitsRequired(long)
+ */
+ public static int bitsRequired(long maxValue) {
+ return roundBits(PackedInts.bitsRequired(maxValue));
+ }
+
+ /**
+ * Returns how many bits are required to hold values up
+ * to and including maxValue, interpreted as an unsigned value.
+ *
+ * @param maxValue the maximum value that should be representable.
+ * @return the amount of bits needed to represent values from 0 to maxValue.
+ * @see PackedInts#unsignedBitsRequired(long)
+ */
+ public static int unsignedBitsRequired(long maxValue) {
+ return roundBits(PackedInts.unsignedBitsRequired(maxValue));
+ }
+
final static int SUPPORTED_BITS_PER_VALUE[] = new int[] {
1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64
};
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoDocIdSet.java Mon Jun 16 19:26:19 2014
@@ -21,14 +21,17 @@ import java.io.IOException;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
-
import org.apache.lucene.util.FixedBitSet; // for javadocs
+import org.apache.lucene.util.RamUsageEstimator;
/** A DocIdSet in Elias-Fano encoding.
* @lucene.internal
*/
public class EliasFanoDocIdSet extends DocIdSet {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(EliasFanoDocIdSet.class);
+
final EliasFanoEncoder efEncoder;
/**
@@ -126,5 +129,10 @@ public class EliasFanoDocIdSet extends D
public int hashCode() {
return efEncoder.hashCode() ^ getClass().hashCode();
}
+
+ @Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED + efEncoder.ramBytesUsed();
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java Mon Jun 16 19:26:19 2014
@@ -19,8 +19,10 @@ package org.apache.lucene.util.packed;
import java.util.Arrays;
-import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.FixedBitSet; // for javadocs
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.ToStringUtils;
/** Encode a non decreasing sequence of non negative whole numbers in the Elias-Fano encoding
@@ -82,7 +84,10 @@ import org.apache.lucene.util.FixedBitSe
* @lucene.internal
*/
-public class EliasFanoEncoder {
+public class EliasFanoEncoder implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(EliasFanoEncoder.class);
+
final long numValues;
private final long upperBound;
final int numLowBits;
@@ -350,5 +355,12 @@ public class EliasFanoEncoder {
return h;
}
+ @Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED
+ + RamUsageEstimator.sizeOf(lowerLongs)
+ + RamUsageEstimator.sizeOf(upperLongs)
+ + RamUsageEstimator.sizeOf(upperZeroBitPositionIndex);
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java Mon Jun 16 19:26:19 2014
@@ -74,7 +74,7 @@ public class GrowableWriter extends Pack
if ((value & currentMask) == value) {
return;
}
- final int bitsRequired = value < 0 ? 64 : PackedInts.bitsRequired(value);
+ final int bitsRequired = PackedInts.unsignedBitsRequired(value);
assert bitsRequired > current.getBitsPerValue();
final int valueCount = size();
PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio);
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java Mon Jun 16 19:26:19 2014
@@ -125,7 +125,7 @@ public final class MonotonicAppendingLon
if (maxDelta == 0) {
values[valuesOff] = new PackedInts.NullReader(pendingOff);
} else {
- final int bitsRequired = maxDelta < 0 ? 64 : PackedInts.bitsRequired(maxDelta);
+ final int bitsRequired = PackedInts.unsignedBitsRequired(maxDelta);
final PackedInts.Mutable mutable = PackedInts.getMutable(pendingOff, bitsRequired, acceptableOverheadRatio);
for (int i = 0; i < pendingOff; ) {
i += mutable.set(i, pending, i, pendingOff - i);
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java Mon Jun 16 19:26:19 2014
@@ -254,7 +254,7 @@ class Packed64 extends PackedInts.Mutabl
@Override
public void fill(int fromIndex, int toIndex, long val) {
- assert PackedInts.bitsRequired(val) <= getBitsPerValue();
+ assert PackedInts.unsignedBitsRequired(val) <= getBitsPerValue();
assert fromIndex <= toIndex;
// minimum number of values that use an exact number of full blocks
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java Mon Jun 16 19:26:19 2014
@@ -159,7 +159,7 @@ abstract class Packed64SingleBlock exten
public void fill(int fromIndex, int toIndex, long val) {
assert fromIndex >= 0;
assert fromIndex <= toIndex;
- assert PackedInts.bitsRequired(val) <= bitsPerValue;
+ assert PackedInts.unsignedBitsRequired(val) <= bitsPerValue;
final int valuesPerBlock = 64 / bitsPerValue;
if (toIndex - fromIndex <= valuesPerBlock << 1) {
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Mon Jun 16 19:26:19 2014
@@ -1131,6 +1131,7 @@ public class PackedInts {
/** Returns how many bits are required to hold values up
* to and including maxValue
+ * NOTE: This method returns at least 1.
* @param maxValue the maximum value that should be representable.
* @return the amount of bits needed to represent values from 0 to maxValue.
* @lucene.internal
@@ -1139,7 +1140,16 @@ public class PackedInts {
if (maxValue < 0) {
throw new IllegalArgumentException("maxValue must be non-negative (got: " + maxValue + ")");
}
- return Math.max(1, 64 - Long.numberOfLeadingZeros(maxValue));
+ return unsignedBitsRequired(maxValue);
+ }
+
+ /** Returns how many bits are required to store <code>bits</code>,
+ * interpreted as an unsigned value.
+ * NOTE: This method returns at least 1.
+ * @lucene.internal
+ */
+ public static int unsignedBitsRequired(long bits) {
+ return Math.max(1, 64 - Long.numberOfLeadingZeros(bits));
}
/**
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java Mon Jun 16 19:26:19 2014
@@ -56,7 +56,7 @@ final class PackedWriter extends PackedI
@Override
public void add(long v) throws IOException {
- assert bitsPerValue == 64 || (v >= 0 && v <= PackedInts.maxValue(bitsPerValue)) : bitsPerValue;
+ assert PackedInts.unsignedBitsRequired(v) <= bitsPerValue;
assert !finished;
if (valueCount != -1 && written >= valueCount) {
throw new EOFException("Writing past end of stream");
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java Mon Jun 16 19:26:19 2014
@@ -19,14 +19,19 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.junit.BeforeClass;
/**
* Tests Lucene45DocValuesFormat
*/
public class TestLucene45DocValuesFormat extends BaseCompressingDocValuesFormatTestCase {
- private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene45DocValuesFormat());
-
+ private final Codec codec = new Lucene45RWCodec();
+
+ @BeforeClass
+ public static void beforeClass() {
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true; // explicitly instantiates ancient codec
+ }
+
@Override
protected Codec getCodec() {
return codec;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Mon Jun 16 19:26:19 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -142,6 +143,14 @@ public class TestDuelingCodecs extends L
if (random.nextInt(4) == 2) {
document.add(new NumericDocValuesField("sparsenumeric", random.nextInt()));
}
+ // add sortednumeric sometimes
+ document.removeFields("sparsesortednum");
+ if (random.nextInt(5) == 1) {
+ document.add(new SortedNumericDocValuesField("sparsesortednum", random.nextLong()));
+ if (random.nextBoolean()) {
+ document.add(new SortedNumericDocValuesField("sparsesortednum", random.nextLong()));
+ }
+ }
writer.addDocument(document);
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java Mon Jun 16 19:26:19 2014
@@ -316,7 +316,7 @@ public class TestFlushByRamOrCountsPolic
while (pendingDocs.decrementAndGet() > -1) {
Document doc = docs.nextDoc();
writer.addDocument(doc);
- long newRamSize = writer.ramSizeInBytes();
+ long newRamSize = writer.ramBytesUsed();
if (newRamSize != ramSize) {
ramSize = newRamSize;
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Mon Jun 16 19:26:19 2014
@@ -48,6 +48,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@@ -1018,6 +1019,10 @@ public class TestIndexWriter extends Luc
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("one")));
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("two")));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 4));
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 3));
+ }
w.addDocument(doc);
doc = new Document();
doc.add(newStringField(random, "id", "501", Field.Store.NO));
@@ -1029,6 +1034,10 @@ public class TestIndexWriter extends Luc
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("two")));
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("three")));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 6));
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 1));
+ }
w.addDocument(doc);
w.deleteDocuments(new Term("id", "500"));
w.shutdown();
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon Jun 16 19:26:19 2014
@@ -41,6 +41,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@@ -159,6 +160,10 @@ public class TestIndexWriterExceptions e
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("hellllo")));
doc.add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("again")));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 10));
+ doc.add(new SortedNumericDocValuesField("sortednumericdv", 5));
+ }
doc.add(newField(r, "content7", "aaa bbb ccc ddd", DocCopyIterator.custom4));
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java Mon Jun 16 19:26:19 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
@@ -109,6 +110,10 @@ public class TestIndexWriterExceptions2
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i))));
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i-1))));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("dv5", i));
+ doc.add(new SortedNumericDocValuesField("dv5", i-1));
+ }
doc.add(newTextField("text1", TestUtil.randomAnalysisString(random(), 20, true), Field.Store.NO));
// ensure we store something
doc.add(new StoredField("stored1", "foo"));
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java Mon Jun 16 19:26:19 2014
@@ -33,6 +33,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
@@ -129,6 +130,10 @@ public class TestIndexWriterOutOfMemory
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i))));
doc.add(new SortedSetDocValuesField("dv4", new BytesRef(Integer.toString(i-1))));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField("dv5", i));
+ doc.add(new SortedNumericDocValuesField("dv5", i-1));
+ }
doc.add(newTextField("text1", TestUtil.randomAnalysisString(random(), 20, true), Field.Store.NO));
// ensure we store something
doc.add(new StoredField("stored1", "foo"));
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java Mon Jun 16 19:26:19 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
@@ -317,6 +318,58 @@ public class TestMultiDocValues extends
dir.close();
}
+ public void testSortedNumeric() throws Exception {
+ assumeTrue("codec does not support SORTED_NUMERIC", defaultCodecSupportsSortedNumeric());
+ Directory dir = newDirectory();
+
+ IndexWriterConfig iwc = newIndexWriterConfig(random(), TEST_VERSION_CURRENT, null);
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ int numDocs = atLeast(500);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ int numValues = random().nextInt(5);
+ for (int j = 0; j < numValues; j++) {
+ doc.add(new SortedNumericDocValuesField("nums", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
+ }
+ iw.addDocument(doc);
+ if (random().nextInt(17) == 0) {
+ iw.commit();
+ }
+ }
+ DirectoryReader ir = iw.getReader();
+ iw.forceMerge(1);
+ DirectoryReader ir2 = iw.getReader();
+ AtomicReader merged = getOnlySegmentReader(ir2);
+ iw.shutdown();
+
+ SortedNumericDocValues multi = MultiDocValues.getSortedNumericValues(ir, "nums");
+ SortedNumericDocValues single = merged.getSortedNumericDocValues("nums");
+ if (multi == null) {
+ assertNull(single);
+ } else {
+ // check values
+ for (int i = 0; i < numDocs; i++) {
+ single.setDocument(i);
+ ArrayList<Long> expectedList = new ArrayList<>();
+ for (int j = 0; j < single.count(); j++) {
+ expectedList.add(single.valueAt(j));
+ }
+
+ multi.setDocument(i);
+ assertEquals(expectedList.size(), multi.count());
+ for (int j = 0; j < single.count(); j++) {
+ assertEquals(expectedList.get(j).longValue(), multi.valueAt(j));
+ }
+ }
+ }
+
+ ir.close();
+ ir2.close();
+ dir.close();
+ }
+
public void testDocsWithField() throws Exception {
assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java Mon Jun 16 19:26:19 2014
@@ -69,7 +69,11 @@ final class JustCompileSearch {
public DocIdSetIterator iterator() {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
-
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
}
static final class JustCompileDocIdSetIterator extends DocIdSetIterator {
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1602968&r1=1602967&r2=1602968&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Mon Jun 16 19:26:19 2014
@@ -206,6 +206,11 @@ public class TestCachingWrapperFilter ex
public DocIdSetIterator iterator() {
return null;
}
+
+ @Override
+ public long ramBytesUsed() {
+ return 0L;
+ }
};
}
};