You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2015/04/09 19:15:00 UTC
svn commit: r1672424 - in /lucene/dev/branches/branch_5x/solr: ./ core/
core/src/java/org/apache/solr/search/
Author: yonik
Date: Thu Apr 9 17:15:00 2015
New Revision: 1672424
URL: http://svn.apache.org/r1672424
Log:
SOLR-7371: don't cache size for now and tolerate small alignment errors
Modified:
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/DocSlice.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/HashDocSet.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/BitDocSet.java?rev=1672424&r1=1672423&r2=1672424&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/BitDocSet.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/BitDocSet.java Thu Apr 9 17:15:00 2015
@@ -40,7 +40,9 @@ import org.apache.lucene.util.RamUsageEs
* @since solr 0.9
*/
public class BitDocSet extends DocSetBase {
- private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitDocSet.class);
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitDocSet.class)
+ + RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class)
+ + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; // for the array object inside the FixedBitSet. long[] array won't change alignment, so no need to calculate it.
final FixedBitSet bits;
int size; // number of docs in the set (cached for perf)
@@ -198,7 +200,7 @@ public class BitDocSet extends DocSetBas
if (other instanceof BitDocSet) {
// if we don't know our current size, this is faster than
// size + other.size - intersection_size
- return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
+ return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet) other).bits);
} else {
// they had better not call us back!
return other.unionSize(this);
@@ -369,7 +371,7 @@ public class BitDocSet extends DocSetBas
@Override
public long ramBytesUsed() {
- return BASE_RAM_BYTES_USED + bits.ramBytesUsed();
+ return BASE_RAM_BYTES_USED + ((long)bits.getBits().length << 3);
}
@Override
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/DocSlice.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/DocSlice.java?rev=1672424&r1=1672423&r2=1672424&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/DocSlice.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/DocSlice.java Thu Apr 9 17:15:00 2015
@@ -31,7 +31,7 @@ import org.apache.lucene.util.RamUsageEs
* @since solr 0.9
*/
public class DocSlice extends DocSetBase implements DocList {
- private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocSlice.class);
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocSlice.class) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
final int offset; // starting position of the docs (zero based)
final int len; // number of positions used in arrays
@@ -41,8 +41,6 @@ public class DocSlice extends DocSetBase
final int matches;
final float maxScore;
- final long ramBytesUsed;
-
/**
* Primary constructor for a DocSlice instance.
*
@@ -59,7 +57,6 @@ public class DocSlice extends DocSetBase
this.scores=scores;
this.matches=matches;
this.maxScore=maxScore;
- this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(docs) + (scores != null ? RamUsageEstimator.sizeOf(scores) : 0);
}
@Override
@@ -178,9 +175,10 @@ public class DocSlice extends DocSetBase
return null;
}
+ /** WARNING: this can over-estimate real memory use since backing arrays are shared with other DocSlice instances */
@Override
public long ramBytesUsed() {
- return ramBytesUsed;
+ return BASE_RAM_BYTES_USED + ((long)docs.length << 2) + (scores == null ? 0 : ((long)scores.length<<2)+RamUsageEstimator.NUM_BYTES_ARRAY_HEADER);
}
@Override
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/HashDocSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/HashDocSet.java?rev=1672424&r1=1672423&r2=1672424&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/HashDocSet.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/HashDocSet.java Thu Apr 9 17:15:00 2015
@@ -35,7 +35,7 @@ import org.apache.lucene.util.RamUsageEs
* @since solr 0.9
*/
public final class HashDocSet extends DocSetBase {
- private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HashDocSet.class);
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HashDocSet.class) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
/** Default load factor to use for HashDocSets. We keep track of the inverse
* since multiplication is so much faster than division. The default
@@ -52,16 +52,12 @@ public final class HashDocSet extends Do
private final static int EMPTY=-1;
private final int[] table;
private final int size;
-
private final int mask;
- private final long ramBytesUsed;
-
public HashDocSet(HashDocSet set) {
this.table = set.table.clone();
this.size = set.size;
this.mask = set.mask;
- this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(table);
}
/** Create a HashDocSet from a list of *unique* ids */
@@ -89,8 +85,6 @@ public final class HashDocSet extends Do
}
size = len;
-
- ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(table);
}
void put(int doc) {
@@ -307,7 +301,7 @@ public final class HashDocSet extends Do
@Override
public long ramBytesUsed() {
- return ramBytesUsed;
+ return BASE_RAM_BYTES_USED + (table.length<<2);
}
@Override
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java?rev=1672424&r1=1672423&r2=1672424&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java Thu Apr 9 17:15:00 2015
@@ -35,18 +35,15 @@ import org.apache.lucene.util.RamUsageEs
* <code>SortedIntDocSet</code> represents a sorted set of Lucene Document Ids.
*/
public class SortedIntDocSet extends DocSetBase {
- private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SortedIntDocSet.class);
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SortedIntDocSet.class) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
protected final int[] docs;
- protected final long ramBytesUsed;
/**
* @param docs Sorted list of ids
*/
public SortedIntDocSet(int[] docs) {
this.docs = docs;
- this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(docs);
- // if (firstNonSorted(docs,0,docs.length)>=0) throw new RuntimeException("NON SORTED DOCS!!!");
}
/**
@@ -246,7 +243,7 @@ public class SortedIntDocSet extends Doc
// if b is 8 times bigger than a, use the modified binary search.
if ((b.length>>3) >= a.length) {
- return intersectionSize(a,b);
+ return intersectionSize(a, b);
}
// if they are close in size, just do a linear walk of both.
@@ -796,7 +793,7 @@ public class SortedIntDocSet extends Doc
@Override
public long ramBytesUsed() {
- return ramBytesUsed;
+ return BASE_RAM_BYTES_USED + (docs.length << 2);
}
@Override