You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Mark Miller <ma...@gmail.com> on 2014/02/16 20:59:14 UTC
Re: svn commit: r1568738 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/util/ lucene/grouping/ lucene/grouping/src/java/org/apache/lucene/search/grouping/ solr/ solr/core/ solr/core/src/java/org/ap...
FYI, somehow this issue broke Solr tests I think:
[junit4] 2> Caused by: java.lang.AssertionError: index=36, numBits=36
[junit4] 2> at org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
[junit4] 2> at org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
[junit4] 2> at org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
[junit4] 2> at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
[junit4] 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
[junit4] 2> ... 26 more
[junit4] 2>
[junit4] 2> 2818355 T8214 oejs.ServletHandler.doHandle WARN Error for /select java.lang.AssertionError: index=36, numBits=36
[junit4] 2> at org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
[junit4] 2> at org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
[junit4] 2> at org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
[junit4] 2> at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
[junit4] 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:134)
- Mark
http://about.me/markrmiller
On Feb 16, 2014, at 2:50 AM, shaie@apache.org wrote:
> Author: shaie
> Date: Sun Feb 16 07:50:08 2014
> New Revision: 1568738
>
> URL: http://svn.apache.org/r1568738
> Log:
> LUCENE-5440: decouple OpenBitSet from DocSet and move to use FixedBitSet
>
> Modified:
> lucene/dev/branches/branch_4x/ (props changed)
> lucene/dev/branches/branch_4x/lucene/ (props changed)
> lucene/dev/branches/branch_4x/lucene/core/ (props changed)
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
> lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> lucene/dev/branches/branch_4x/solr/ (props changed)
> lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
> lucene/dev/branches/branch_4x/solr/core/ (props changed)
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestDocSet.java
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestFiltering.java
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestSort.java
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/util/BitSetPerf.java
>
> Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
> +++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Sun Feb 16 07:50:08 2014
> @@ -28,6 +28,8 @@ import org.apache.lucene.search.DocIdSet
> * long[], accessed with an int index, implementing {@link Bits} and
> * {@link DocIdSet}. If you need to manage more than 2.1B bits, use
> * {@link LongBitSet}.
> + *
> + * @lucene.internal
> */
> public final class FixedBitSet extends DocIdSet implements Bits {
>
> @@ -41,6 +43,12 @@ public final class FixedBitSet extends D
> final long[] bits;
> int doc = -1;
>
> + /** Creates an iterator over the given {@link FixedBitSet}. */
> + public FixedBitSetIterator(FixedBitSet bits) {
> + this(bits.bits, bits.numBits, bits.numWords);
> + }
> +
> + /** Creates an iterator over the given array of bits. */
> public FixedBitSetIterator(long[] bits, int numBits, int wordLength) {
> this.bits = bits;
> this.numBits = numBits;
> @@ -48,7 +56,7 @@ public final class FixedBitSet extends D
> }
>
> @Override
> - public int nextDoc() throws IOException {
> + public int nextDoc() {
> if (doc == NO_MORE_DOCS || ++doc >= numBits) {
> return doc = NO_MORE_DOCS;
> }
> @@ -81,7 +89,7 @@ public final class FixedBitSet extends D
> }
>
> @Override
> - public int advance(int target) throws IOException {
> + public int advance(int target) {
> if (doc == NO_MORE_DOCS || target >= numBits) {
> return doc = NO_MORE_DOCS;
> }
> @@ -103,10 +111,6 @@ public final class FixedBitSet extends D
> return doc = NO_MORE_DOCS;
> }
> }
> -
> - private final long[] bits;
> - private final int numBits;
> - private final int wordLength;
>
> /**
> * If the given {@link FixedBitSet} is large enough to hold {@code numBits},
> @@ -140,15 +144,53 @@ public final class FixedBitSet extends D
> return numLong;
> }
>
> + /**
> + * Returns the popcount or cardinality of the intersection of the two sets.
> + * Neither set is modified.
> + */
> + public static long intersectionCount(FixedBitSet a, FixedBitSet b) {
> + return BitUtil.pop_intersect(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
> + }
> +
> + /**
> + * Returns the popcount or cardinality of the union of the two sets. Neither
> + * set is modified.
> + */
> + public static long unionCount(FixedBitSet a, FixedBitSet b) {
> + long tot = BitUtil.pop_union(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
> + if (a.numWords < b.numWords) {
> + tot += BitUtil.pop_array(b.bits, a.numWords, b.numWords - a.numWords);
> + } else if (a.numWords > b.numWords) {
> + tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
> + }
> + return tot;
> + }
> +
> + /**
> + * Returns the popcount or cardinality of "a and not b" or
> + * "intersection(a, not(b))". Neither set is modified.
> + */
> + public static long andNotCount(FixedBitSet a, FixedBitSet b) {
> + long tot = BitUtil.pop_andnot(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
> + if (a.numWords > b.numWords) {
> + tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
> + }
> + return tot;
> + }
> +
> + final long[] bits;
> + final int numBits;
> + final int numWords;
> +
> public FixedBitSet(int numBits) {
> this.numBits = numBits;
> bits = new long[bits2words(numBits)];
> - wordLength = bits.length;
> + numWords = bits.length;
> }
>
> public FixedBitSet(long[] storedBits, int numBits) {
> - this.wordLength = bits2words(numBits);
> - if (wordLength > storedBits.length) {
> + this.numWords = bits2words(numBits);
> + if (numWords > storedBits.length) {
> throw new IllegalArgumentException("The given long array is too small to hold " + numBits + " bits");
> }
> this.numBits = numBits;
> @@ -157,7 +199,7 @@ public final class FixedBitSet extends D
>
> @Override
> public DocIdSetIterator iterator() {
> - return new FixedBitSetIterator(bits, numBits, wordLength);
> + return new FixedBitSetIterator(bits, numBits, numWords);
> }
>
> @Override
> @@ -190,7 +232,7 @@ public final class FixedBitSet extends D
>
> @Override
> public boolean get(int index) {
> - assert index >= 0 && index < numBits: "index=" + index;
> + assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
> int i = index >> 6; // div 64
> // signed shift will keep a negative index and force an
> // array-index-out-of-bounds-exception, removing the need for an explicit check.
> @@ -200,7 +242,7 @@ public final class FixedBitSet extends D
> }
>
> public void set(int index) {
> - assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
> + assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
> int wordNum = index >> 6; // div 64
> int bit = index & 0x3f; // mod 64
> long bitmask = 1L << bit;
> @@ -239,7 +281,7 @@ public final class FixedBitSet extends D
> * -1 is returned if there are no more set bits.
> */
> public int nextSetBit(int index) {
> - assert index >= 0 && index < numBits;
> + assert index >= 0 && index < numBits : "index=" + index + ", numBits=" + numBits;
> int i = index >> 6;
> final int subIndex = index & 0x3f; // index within the word
> long word = bits[i] >> subIndex; // skip all the bits to the right of index
> @@ -248,7 +290,7 @@ public final class FixedBitSet extends D
> return index + Long.numberOfTrailingZeros(word);
> }
>
> - while(++i < wordLength) {
> + while(++i < numWords) {
> word = bits[i];
> if (word != 0) {
> return (i<<6) + Long.numberOfTrailingZeros(word);
> @@ -306,12 +348,13 @@ public final class FixedBitSet extends D
>
> /** this = this OR other */
> public void or(FixedBitSet other) {
> - or(other.bits, other.wordLength);
> + or(other.bits, other.numWords);
> }
>
> - private void or(final long[] otherArr, final int otherLen) {
> + private void or(final long[] otherArr, final int otherNumWords) {
> + assert otherNumWords <= numWords : "numWords=" + numWords + ", otherNumWords=" + otherNumWords;
> final long[] thisArr = this.bits;
> - int pos = Math.min(wordLength, otherLen);
> + int pos = Math.min(numWords, otherNumWords);
> while (--pos >= 0) {
> thisArr[pos] |= otherArr[pos];
> }
> @@ -319,9 +362,10 @@ public final class FixedBitSet extends D
>
> /** this = this XOR other */
> public void xor(FixedBitSet other) {
> + assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
> final long[] thisBits = this.bits;
> final long[] otherBits = other.bits;
> - int pos = Math.min(wordLength, other.wordLength);
> + int pos = Math.min(numWords, other.numWords);
> while (--pos >= 0) {
> thisBits[pos] ^= otherBits[pos];
> }
> @@ -366,7 +410,7 @@ public final class FixedBitSet extends D
>
> /** returns true if the sets have any elements in common */
> public boolean intersects(FixedBitSet other) {
> - int pos = Math.min(wordLength, other.wordLength);
> + int pos = Math.min(numWords, other.numWords);
> while (--pos>=0) {
> if ((bits[pos] & other.bits[pos]) != 0) return true;
> }
> @@ -375,17 +419,17 @@ public final class FixedBitSet extends D
>
> /** this = this AND other */
> public void and(FixedBitSet other) {
> - and(other.bits, other.wordLength);
> + and(other.bits, other.numWords);
> }
>
> - private void and(final long[] otherArr, final int otherLen) {
> + private void and(final long[] otherArr, final int otherNumWords) {
> final long[] thisArr = this.bits;
> - int pos = Math.min(this.wordLength, otherLen);
> + int pos = Math.min(this.numWords, otherNumWords);
> while(--pos >= 0) {
> thisArr[pos] &= otherArr[pos];
> }
> - if (this.wordLength > otherLen) {
> - Arrays.fill(thisArr, otherLen, this.wordLength, 0L);
> + if (this.numWords > otherNumWords) {
> + Arrays.fill(thisArr, otherNumWords, this.numWords, 0L);
> }
> }
>
> @@ -417,9 +461,9 @@ public final class FixedBitSet extends D
> andNot(other.bits, other.bits.length);
> }
>
> - private void andNot(final long[] otherArr, final int otherLen) {
> + private void andNot(final long[] otherArr, final int otherNumWords) {
> final long[] thisArr = this.bits;
> - int pos = Math.min(this.wordLength, otherLen);
> + int pos = Math.min(this.numWords, otherNumWords);
> while(--pos >= 0) {
> thisArr[pos] &= ~otherArr[pos];
> }
> @@ -502,8 +546,8 @@ public final class FixedBitSet extends D
> * @param endIndex one-past the last bit to clear
> */
> public void clear(int startIndex, int endIndex) {
> - assert startIndex >= 0 && startIndex < numBits;
> - assert endIndex >= 0 && endIndex <= numBits;
> + assert startIndex >= 0 && startIndex < numBits : "startIndex=" + startIndex + ", numBits=" + numBits;
> + assert endIndex >= 0 && endIndex <= numBits : "endIndex=" + endIndex + ", numBits=" + numBits;
> if (endIndex <= startIndex) {
> return;
> }
> @@ -554,7 +598,7 @@ public final class FixedBitSet extends D
> @Override
> public int hashCode() {
> long h = 0;
> - for (int i = wordLength; --i>=0;) {
> + for (int i = numWords; --i>=0;) {
> h ^= bits[i];
> h = (h << 1) | (h >>> 63); // rotate left
> }
>
> Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java (original)
> +++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java Sun Feb 16 07:50:08 2014
> @@ -23,6 +23,8 @@ import java.util.Arrays;
> * BitSet of fixed length (numBits), backed by accessible ({@link #getBits})
> * long[], accessed with a long index. Use it only if you intend to store more
> * than 2.1B bits, otherwise you should use {@link FixedBitSet}.
> + *
> + * @lucene.internal
> */
> public final class LongBitSet {
>
> @@ -188,6 +190,7 @@ public final class LongBitSet {
>
> /** this = this OR other */
> public void or(LongBitSet other) {
> + assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
> int pos = Math.min(numWords, other.numWords);
> while (--pos >= 0) {
> bits[pos] |= other.bits[pos];
> @@ -196,6 +199,7 @@ public final class LongBitSet {
>
> /** this = this XOR other */
> public void xor(LongBitSet other) {
> + assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
> int pos = Math.min(numWords, other.numWords);
> while (--pos >= 0) {
> bits[pos] ^= other.bits[pos];
>
> Modified: lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java (original)
> +++ lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java Sun Feb 16 07:50:08 2014
> @@ -44,7 +44,7 @@ public abstract class AbstractAllGroupHe
>
> /**
> * @param maxDoc The maxDoc of the top level {@link IndexReader}.
> - * @return an {@link FixedBitSet} containing all group heads.
> + * @return a {@link FixedBitSet} containing all group heads.
> */
> public FixedBitSet retrieveGroupHeads(int maxDoc) {
> FixedBitSet bitSet = new FixedBitSet(maxDoc);
>
> Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
> +++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sun Feb 16 07:50:08 2014
> @@ -330,6 +330,10 @@ Optimizations
> * SOLR-5624: Enable QueryResultCache for CollapsingQParserPlugin.
> (David Boychuck, Joel Bernstein)
>
> +* LUCENE-5440: DocSet decoupled from OpenBitSet. DocSetBase moved to use
> + FixedBitSet instead of OpenBitSet. As a result BitDocSet now only works
> + with FixedBitSet. (Shai Erera)
> +
> Other Changes
> ---------------------
>
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java Sun Feb 16 07:50:08 2014
> @@ -17,7 +17,17 @@
>
> package org.apache.solr.handler.component;
>
> -import org.apache.lucene.util.OpenBitSet;
> +import java.io.IOException;
> +import java.net.URL;
> +import java.util.ArrayList;
> +import java.util.Arrays;
> +import java.util.Comparator;
> +import java.util.HashMap;
> +import java.util.LinkedHashMap;
> +import java.util.List;
> +import java.util.Map;
> +
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.solr.common.SolrException;
> import org.apache.solr.common.SolrException.ErrorCode;
> import org.apache.solr.common.params.CommonParams;
> @@ -35,10 +45,6 @@ import org.apache.solr.search.SyntaxErro
> import org.slf4j.Logger;
> import org.slf4j.LoggerFactory;
>
> -import java.io.IOException;
> -import java.net.URL;
> -import java.util.*;
> -
> /**
> * TODO!
> *
> @@ -444,8 +450,8 @@ public class FacetComponent extends Sear
> // and if it is >= smallestCount, then flag for refinement
> long maxCount = sfc.count;
> for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
> - OpenBitSet obs = dff.counted[shardNum];
> - if (obs!=null && !obs.get(sfc.termNum)) { // obs can be null if a shard request failed
> + FixedBitSet fbs = dff.counted[shardNum];
> + if (fbs!=null && !fbs.get(sfc.termNum)) { // fbs can be null if a shard request failed
> // if missing from this shard, add the max it could be
> maxCount += dff.maxPossible(sfc,shardNum);
> }
> @@ -459,8 +465,8 @@ public class FacetComponent extends Sear
> if (needRefinement) {
> // add a query for each shard missing the term that needs refinement
> for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
> - OpenBitSet obs = dff.counted[shardNum];
> - if(obs!=null && !obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
> + FixedBitSet fbs = dff.counted[shardNum];
> + if(fbs!=null && !fbs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
> dff.needRefinements = true;
> List<String> lst = dff._toRefine[shardNum];
> if (lst == null) {
> @@ -759,7 +765,7 @@ public class FacetComponent extends Sear
> public long missingMaxPossible;
> // the max possible count for a missing term for each shard (indexed by shardNum)
> public long[] missingMax;
> - public OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
> + public FixedBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
> public HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
> public int termNum;
>
> @@ -772,7 +778,7 @@ public class FacetComponent extends Sear
> super(rb, facetStr);
> // sf = rb.req.getSchema().getField(field);
> missingMax = new long[rb.shards.length];
> - counted = new OpenBitSet[rb.shards.length];
> + counted = new FixedBitSet[rb.shards.length];
> }
>
> void add(int shardNum, NamedList shardCounts, int numRequested) {
> @@ -780,7 +786,7 @@ public class FacetComponent extends Sear
> int sz = shardCounts == null ? 0 : shardCounts.size();
> int numReceived = sz;
>
> - OpenBitSet terms = new OpenBitSet(termNum+sz);
> + FixedBitSet terms = new FixedBitSet(termNum+sz);
>
> long last = 0;
> for (int i=0; i<sz; i++) {
> @@ -799,7 +805,7 @@ public class FacetComponent extends Sear
> counts.put(name, sfc);
> }
> sfc.count += count;
> - terms.fastSet(sfc.termNum);
> + terms.set(sfc.termNum);
> last = count;
> }
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Sun Feb 16 07:50:08 2014
> @@ -17,6 +17,26 @@
>
> package org.apache.solr.request;
>
> +import java.io.IOException;
> +import java.util.ArrayList;
> +import java.util.Collection;
> +import java.util.Date;
> +import java.util.EnumSet;
> +import java.util.IdentityHashMap;
> +import java.util.List;
> +import java.util.Map;
> +import java.util.Set;
> +import java.util.concurrent.Callable;
> +import java.util.concurrent.ExecutionException;
> +import java.util.concurrent.Executor;
> +import java.util.concurrent.Future;
> +import java.util.concurrent.FutureTask;
> +import java.util.concurrent.RunnableFuture;
> +import java.util.concurrent.Semaphore;
> +import java.util.concurrent.SynchronousQueue;
> +import java.util.concurrent.ThreadPoolExecutor;
> +import java.util.concurrent.TimeUnit;
> +
> import org.apache.lucene.index.AtomicReader;
> import org.apache.lucene.index.DocsEnum;
> import org.apache.lucene.index.Fields;
> @@ -37,8 +57,6 @@ import org.apache.lucene.search.grouping
> import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
> import org.apache.lucene.util.BytesRef;
> import org.apache.lucene.util.CharsRef;
> -import org.apache.lucene.util.FixedBitSet;
> -import org.apache.lucene.util.OpenBitSet;
> import org.apache.lucene.util.StringHelper;
> import org.apache.lucene.util.UnicodeUtil;
> import org.apache.solr.common.SolrException;
> @@ -80,26 +98,6 @@ import org.apache.solr.util.DateMathPars
> import org.apache.solr.util.DefaultSolrThreadFactory;
> import org.apache.solr.util.LongPriorityQueue;
>
> -import java.io.IOException;
> -import java.util.ArrayList;
> -import java.util.Collection;
> -import java.util.Date;
> -import java.util.EnumSet;
> -import java.util.IdentityHashMap;
> -import java.util.List;
> -import java.util.Map;
> -import java.util.Set;
> -import java.util.concurrent.Callable;
> -import java.util.concurrent.ExecutionException;
> -import java.util.concurrent.Executor;
> -import java.util.concurrent.Future;
> -import java.util.concurrent.FutureTask;
> -import java.util.concurrent.RunnableFuture;
> -import java.util.concurrent.Semaphore;
> -import java.util.concurrent.SynchronousQueue;
> -import java.util.concurrent.ThreadPoolExecutor;
> -import java.util.concurrent.TimeUnit;
> -
> /**
> * A class that generates simple Facet information for a request.
> *
> @@ -230,10 +228,7 @@ public class SimpleFacets {
> }
> AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0).createAllGroupCollector();
> searcher.search(new MatchAllDocsQuery(), base.getTopFilter(), allGroupHeadsCollector);
> - int maxDoc = searcher.maxDoc();
> - FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
> - long[] bits = fixedBitSet.getBits();
> - this.docs = new BitDocSet(new OpenBitSet(bits, bits.length));
> + this.docs = new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
> } else {
> this.docs = base;
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Sun Feb 16 07:50:08 2014
> @@ -27,12 +27,11 @@ import org.apache.lucene.index.DocTermOr
> import org.apache.lucene.index.SortedDocValues;
> import org.apache.lucene.index.Term;
> import org.apache.lucene.index.TermsEnum;
> -import org.apache.lucene.search.FieldCache;
> import org.apache.lucene.search.TermQuery;
> import org.apache.lucene.search.TermRangeQuery;
> import org.apache.lucene.util.BytesRef;
> import org.apache.lucene.util.CharsRef;
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.lucene.util.UnicodeUtil;
> import org.apache.solr.common.SolrException;
> import org.apache.solr.common.params.FacetParams;
> @@ -44,7 +43,11 @@ import org.apache.solr.handler.component
> import org.apache.solr.schema.FieldType;
> import org.apache.solr.schema.SchemaField;
> import org.apache.solr.schema.TrieField;
> -import org.apache.solr.search.*;
> +import org.apache.solr.search.BitDocSet;
> +import org.apache.solr.search.DocIterator;
> +import org.apache.solr.search.DocSet;
> +import org.apache.solr.search.SolrCache;
> +import org.apache.solr.search.SolrIndexSearcher;
> import org.apache.solr.util.LongPriorityQueue;
> import org.apache.solr.util.PrimUtils;
>
> @@ -269,7 +272,7 @@ public class UnInvertedField extends Doc
> && docs instanceof BitDocSet;
>
> if (doNegative) {
> - OpenBitSet bs = (OpenBitSet)((BitDocSet)docs).getBits().clone();
> + FixedBitSet bs = ((BitDocSet)docs).getBits().clone();
> bs.flip(0, maxDoc);
> // TODO: when iterator across negative elements is available, use that
> // instead of creating a new bitset and inverting.
> @@ -512,7 +515,7 @@ public class UnInvertedField extends Doc
> }
>
> if (doNegative) {
> - OpenBitSet bs = (OpenBitSet) ((BitDocSet) docs).getBits().clone();
> + FixedBitSet bs = ((BitDocSet) docs).getBits().clone();
> bs.flip(0, maxDoc);
> // TODO: when iterator across negative elements is available, use that
> // instead of creating a new bitset and inverting.
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java Sun Feb 16 07:50:08 2014
> @@ -21,38 +21,40 @@ import org.apache.lucene.index.AtomicRea
> import org.apache.lucene.index.AtomicReaderContext;
> import org.apache.lucene.search.BitsFilteredDocIdSet;
> import org.apache.lucene.search.DocIdSet;
> +import org.apache.lucene.search.DocIdSetIterator;
> import org.apache.lucene.search.Filter;
> import org.apache.lucene.util.Bits;
> -import org.apache.lucene.util.OpenBitSet;
> -import org.apache.lucene.util.OpenBitSetIterator;
> -import org.apache.lucene.search.DocIdSetIterator;
> +import org.apache.lucene.util.FixedBitSet;
> +import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
>
> /**
> * <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
> * using a BitSet. A set bit represents inclusion in the set for that document.
> *
> - *
> * @since solr 0.9
> */
> public class BitDocSet extends DocSetBase {
> - final OpenBitSet bits;
> + final FixedBitSet bits;
> int size; // number of docs in the set (cached for perf)
>
> public BitDocSet() {
> - bits = new OpenBitSet();
> + bits = new FixedBitSet(64);
> }
>
> - /** Construct a BitDocSet.
> - * The capacity of the OpenBitSet should be at least maxDoc() */
> - public BitDocSet(OpenBitSet bits) {
> + /**
> + * Construct a BitDocSet. The capacity of the {@link FixedBitSet} should be at
> + * least maxDoc()
> + */
> + public BitDocSet(FixedBitSet bits) {
> this.bits = bits;
> size=-1;
> }
>
> - /** Construct a BitDocSet, and provides the number of set bits.
> - * The capacity of the OpenBitSet should be at least maxDoc()
> + /**
> + * Construct a BitDocSet, and provides the number of set bits. The capacity of
> + * the {@link FixedBitSet} should be at least maxDoc()
> */
> - public BitDocSet(OpenBitSet bits, int size) {
> + public BitDocSet(FixedBitSet bits, int size) {
> this.bits = bits;
> this.size = size;
> }
> @@ -89,7 +91,7 @@ public class BitDocSet extends DocSetBas
> @Override
> public DocIterator iterator() {
> return new DocIterator() {
> - private final OpenBitSetIterator iter = new OpenBitSetIterator(bits);
> + private final FixedBitSetIterator iter = new FixedBitSetIterator(bits);
> private int pos = iter.nextDoc();
> @Override
> public boolean hasNext() {
> @@ -120,13 +122,11 @@ public class BitDocSet extends DocSetBas
> };
> }
>
> -
> /**
> - *
> - * @return the <b>internal</b> OpenBitSet that should <b>not</b> be modified.
> + * @return the <b>internal</b> {@link FixedBitSet} that should <b>not</b> be modified.
> */
> @Override
> - public OpenBitSet getBits() {
> + public FixedBitSet getBits() {
> return bits;
> }
>
> @@ -145,7 +145,7 @@ public class BitDocSet extends DocSetBas
> @Override
> public int size() {
> if (size!=-1) return size;
> - return size=(int)bits.cardinality();
> + return size = bits.cardinality();
> }
>
> /**
> @@ -156,18 +156,19 @@ public class BitDocSet extends DocSetBas
> size=-1;
> }
>
> - /** Returns true of the doc exists in the set.
> - * Should only be called when doc < OpenBitSet.size()
> + /**
> + * Returns true of the doc exists in the set. Should only be called when doc <
> + * {@link FixedBitSet#length()}.
> */
> @Override
> public boolean exists(int doc) {
> - return bits.fastGet(doc);
> + return bits.get(doc);
> }
>
> @Override
> public int intersectionSize(DocSet other) {
> if (other instanceof BitDocSet) {
> - return (int)OpenBitSet.intersectionCount(this.bits, ((BitDocSet)other).bits);
> + return (int) FixedBitSet.intersectionCount(this.bits, ((BitDocSet) other).bits);
> } else {
> // they had better not call us back!
> return other.intersectionSize(this);
> @@ -189,7 +190,7 @@ public class BitDocSet extends DocSetBas
> if (other instanceof BitDocSet) {
> // if we don't know our current size, this is faster than
> // size + other.size - intersection_size
> - return (int)OpenBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
> + return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
> } else {
> // they had better not call us back!
> return other.unionSize(this);
> @@ -201,42 +202,56 @@ public class BitDocSet extends DocSetBas
> if (other instanceof BitDocSet) {
> // if we don't know our current size, this is faster than
> // size - intersection_size
> - return (int)OpenBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
> + return (int) FixedBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
> } else {
> return super.andNotSize(other);
> }
> }
>
> @Override
> - public void setBitsOn(OpenBitSet target) {
> - target.union(bits);
> + public void addAllTo(DocSet target) {
> + if (target instanceof BitDocSet) {
> + ((BitDocSet) target).bits.or(bits);
> + } else {
> + super.addAllTo(target);
> + }
> }
>
> @Override
> - public DocSet andNot(DocSet other) {
> - OpenBitSet newbits = (OpenBitSet)(bits.clone());
> - if (other instanceof BitDocSet) {
> - newbits.andNot(((BitDocSet)other).bits);
> - } else {
> - DocIterator iter = other.iterator();
> - while (iter.hasNext()) newbits.clear(iter.nextDoc());
> - }
> - return new BitDocSet(newbits);
> + public DocSet andNot(DocSet other) {
> + FixedBitSet newbits = bits.clone();
> + if (other instanceof BitDocSet) {
> + newbits.andNot(((BitDocSet) other).bits);
> + } else {
> + DocIterator iter = other.iterator();
> + while (iter.hasNext()) {
> + int doc = iter.nextDoc();
> + if (doc < newbits.length()) {
> + newbits.clear(doc);
> + }
> + }
> + }
> + return new BitDocSet(newbits);
> }
> -
> +
> @Override
> - public DocSet union(DocSet other) {
> - OpenBitSet newbits = (OpenBitSet)(bits.clone());
> - if (other instanceof BitDocSet) {
> - newbits.union(((BitDocSet)other).bits);
> - } else {
> - DocIterator iter = other.iterator();
> - while (iter.hasNext()) newbits.set(iter.nextDoc());
> - }
> - return new BitDocSet(newbits);
> + public DocSet union(DocSet other) {
> + FixedBitSet newbits = bits.clone();
> + if (other instanceof BitDocSet) {
> + BitDocSet otherDocSet = (BitDocSet) other;
> + newbits = FixedBitSet.ensureCapacity(newbits, otherDocSet.bits.length());
> + newbits.or(otherDocSet.bits);
> + } else {
> + DocIterator iter = other.iterator();
> + while (iter.hasNext()) {
> + int doc = iter.nextDoc();
> + newbits = FixedBitSet.ensureCapacity(newbits, doc);
> + newbits.set(doc);
> + }
> + }
> + return new BitDocSet(newbits);
> }
> -
> -
> +
> @Override
> public long memSize() {
> return (bits.getBits().length << 3) + 16;
> @@ -244,12 +259,12 @@ public class BitDocSet extends DocSetBas
>
> @Override
> protected BitDocSet clone() {
> - return new BitDocSet((OpenBitSet)bits.clone(), size);
> + return new BitDocSet(bits.clone(), size);
> }
>
> @Override
> public Filter getTopFilter() {
> - final OpenBitSet bs = bits;
> + final FixedBitSet bs = bits;
> // TODO: if cardinality isn't cached, do a quick measure of sparseness
> // and return null from bits() if too sparse.
>
> @@ -272,8 +287,8 @@ public class BitDocSet extends DocSetBas
> @Override
> public DocIdSetIterator iterator() {
> return new DocIdSetIterator() {
> - int pos=base-1;
> - int adjustedDoc=-1;
> + int pos = base - 1;
> + int adjustedDoc = -1;
>
> @Override
> public int docID() {
> @@ -282,15 +297,24 @@ public class BitDocSet extends DocSetBas
>
> @Override
> public int nextDoc() {
> - pos = bs.nextSetBit(pos+1);
> - return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
> + if (pos >= bs.length() - 1) {
> + return adjustedDoc = NO_MORE_DOCS;
> + } else {
> + pos = bs.nextSetBit(pos + 1);
> + return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
> + }
> }
>
> @Override
> public int advance(int target) {
> - if (target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;
> - pos = bs.nextSetBit(target+base);
> - return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
> + if (target == NO_MORE_DOCS) return adjustedDoc = NO_MORE_DOCS;
> + int adjusted = target + base;
> + if (adjusted >= bs.length()) {
> + return adjustedDoc = NO_MORE_DOCS;
> + } else {
> + pos = bs.nextSetBit(adjusted);
> + return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
> + }
> }
>
> @Override
> @@ -298,7 +322,7 @@ public class BitDocSet extends DocSetBas
> // we don't want to actually compute cardinality, but
> // if its already been computed, we use it (pro-rated for the segment)
> if (size != -1) {
> - return (long)(size * ((OpenBitSet.bits2words(maxDoc)<<6) / (float)bs.capacity()));
> + return (long)(size * ((FixedBitSet.bits2words(maxDoc)<<6) / (float)bs.length()));
> } else {
> return maxDoc;
> }
> @@ -316,7 +340,7 @@ public class BitDocSet extends DocSetBas
> return new Bits() {
> @Override
> public boolean get(int index) {
> - return bs.fastGet(index + base);
> + return bs.get(index + base);
> }
>
> @Override
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java Sun Feb 16 07:50:08 2014
> @@ -17,45 +17,49 @@
>
> package org.apache.solr.search;
>
> +import java.io.IOException;
> +import java.util.Arrays;
> +import java.util.HashSet;
> +import java.util.Iterator;
> +import java.util.List;
> +import java.util.Map;
> +import java.util.Set;
> +
> +import org.apache.lucene.index.AtomicReader;
> +import org.apache.lucene.index.AtomicReaderContext;
> +import org.apache.lucene.index.DocsEnum;
> +import org.apache.lucene.index.SortedDocValues;
> +import org.apache.lucene.index.Terms;
> +import org.apache.lucene.index.TermsEnum;
> import org.apache.lucene.queries.function.FunctionQuery;
> import org.apache.lucene.queries.function.FunctionValues;
> import org.apache.lucene.queries.function.ValueSource;
> +import org.apache.lucene.search.DocIdSetIterator;
> +import org.apache.lucene.search.FieldCache;
> +import org.apache.lucene.search.IndexSearcher;
> +import org.apache.lucene.search.Query;
> +import org.apache.lucene.search.Scorer;
> +import org.apache.lucene.util.Bits;
> import org.apache.lucene.util.BytesRef;
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.solr.common.params.ModifiableSolrParams;
> +import org.apache.solr.common.params.SolrParams;
> +import org.apache.solr.common.util.NamedList;
> +import org.apache.solr.handler.component.QueryElevationComponent;
> import org.apache.solr.request.LocalSolrQueryRequest;
> +import org.apache.solr.request.SolrQueryRequest;
> import org.apache.solr.request.SolrRequestInfo;
> -import org.apache.solr.schema.TrieFloatField;
> -import org.apache.solr.schema.TrieIntField;
> -import org.apache.solr.schema.TrieLongField;
> import org.apache.solr.schema.FieldType;
> -import org.apache.solr.handler.component.QueryElevationComponent;
> -import org.apache.lucene.index.AtomicReaderContext;
> -import org.apache.lucene.index.AtomicReader;
> -import org.apache.lucene.index.SortedDocValues;
> -import org.apache.lucene.index.Terms;
> -import org.apache.lucene.index.TermsEnum;
> -import org.apache.lucene.index.DocsEnum;
> -import org.apache.lucene.search.*;
> -import org.apache.lucene.util.OpenBitSet;
> -import org.apache.lucene.util.Bits;
> -import org.apache.solr.common.params.SolrParams;
> -import org.apache.solr.request.SolrQueryRequest;
> -import org.apache.solr.common.util.NamedList;
> import org.apache.solr.schema.IndexSchema;
> import org.apache.solr.schema.SchemaField;
> +import org.apache.solr.schema.TrieFloatField;
> +import org.apache.solr.schema.TrieIntField;
> +import org.apache.solr.schema.TrieLongField;
>
> import com.carrotsearch.hppc.FloatArrayList;
> import com.carrotsearch.hppc.IntOpenHashSet;
> import com.carrotsearch.hppc.cursors.IntCursor;
>
> -import java.io.IOException;
> -import java.util.Arrays;
> -import java.util.Map;
> -import java.util.Set;
> -import java.util.HashSet;
> -import java.util.List;
> -import java.util.Iterator;
> -
> /**
>
> The <b>CollapsingQParserPlugin</b> is a PostFilter that performs field collapsing.
> @@ -419,7 +423,7 @@ public class CollapsingQParserPlugin ext
> private class CollapsingScoreCollector extends DelegatingCollector {
>
> private AtomicReaderContext[] contexts;
> - private OpenBitSet collapsedSet;
> + private FixedBitSet collapsedSet;
> private SortedDocValues values;
> private int[] ords;
> private float[] scores;
> @@ -438,14 +442,14 @@ public class CollapsingQParserPlugin ext
> IntOpenHashSet boostDocs) {
> this.maxDoc = maxDoc;
> this.contexts = new AtomicReaderContext[segments];
> - this.collapsedSet = new OpenBitSet(maxDoc);
> + this.collapsedSet = new FixedBitSet(maxDoc);
> this.boostDocs = boostDocs;
> if(this.boostDocs != null) {
> //Set the elevated docs now.
> Iterator<IntCursor> it = this.boostDocs.iterator();
> while(it.hasNext()) {
> IntCursor cursor = it.next();
> - this.collapsedSet.fastSet(cursor.value);
> + this.collapsedSet.set(cursor.value);
> }
> }
> this.values = values;
> @@ -460,16 +464,19 @@ public class CollapsingQParserPlugin ext
> }
> }
>
> + @Override
> public boolean acceptsDocsOutOfOrder() {
> //Documents must be sent in order to this collector.
> return false;
> }
>
> + @Override
> public void setNextReader(AtomicReaderContext context) throws IOException {
> this.contexts[context.ord] = context;
> this.docBase = context.docBase;
> }
>
> + @Override
> public void collect(int docId) throws IOException {
> int globalDoc = docId+this.docBase;
> int ord = values.getOrd(globalDoc);
> @@ -479,7 +486,7 @@ public class CollapsingQParserPlugin ext
> ords[ord] = globalDoc;
> scores[ord] = score;
> }
> - } else if (this.collapsedSet.fastGet(globalDoc)) {
> + } else if (this.collapsedSet.get(globalDoc)) {
> //The doc is elevated so score does not matter
> //We just want to be sure it doesn't fall into the null policy
> } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> @@ -489,24 +496,25 @@ public class CollapsingQParserPlugin ext
> nullDoc = globalDoc;
> }
> } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> - collapsedSet.fastSet(globalDoc);
> + collapsedSet.set(globalDoc);
> nullScores.add(scorer.score());
> }
> }
>
> + @Override
> public void finish() throws IOException {
> if(contexts.length == 0) {
> return;
> }
>
> if(nullScore > 0) {
> - this.collapsedSet.fastSet(nullDoc);
> + this.collapsedSet.set(nullDoc);
> }
>
> for(int i=0; i<ords.length; i++) {
> int doc = ords[i];
> if(doc > -1) {
> - collapsedSet.fastSet(doc);
> + collapsedSet.set(doc);
> }
> }
>
> @@ -677,7 +685,7 @@ public class CollapsingQParserPlugin ext
> protected FloatArrayList nullScores;
> protected float nullScore;
> protected float[] scores;
> - protected OpenBitSet collapsedSet;
> + protected FixedBitSet collapsedSet;
> protected IntOpenHashSet boostDocs;
> protected int nullDoc = -1;
> protected boolean needsScores;
> @@ -697,26 +705,26 @@ public class CollapsingQParserPlugin ext
> this.nullPolicy = nullPolicy;
> this.max = max;
> this.needsScores = needsScores;
> - this.collapsedSet = new OpenBitSet(maxDoc);
> + this.collapsedSet = new FixedBitSet(maxDoc);
> this.boostDocs = boostDocs;
> if(this.boostDocs != null) {
> Iterator<IntCursor> it = boostDocs.iterator();
> while(it.hasNext()) {
> IntCursor cursor = it.next();
> - this.collapsedSet.fastSet(cursor.value);
> + this.collapsedSet.set(cursor.value);
> }
> }
> }
>
> - public OpenBitSet getCollapsedSet() {
> + public FixedBitSet getCollapsedSet() {
> if(nullDoc > -1) {
> - this.collapsedSet.fastSet(nullDoc);
> + this.collapsedSet.set(nullDoc);
> }
>
> for(int i=0; i<ords.length; i++) {
> int doc = ords[i];
> if(doc > -1) {
> - collapsedSet.fastSet(doc);
> + collapsedSet.set(doc);
> }
> }
>
> @@ -790,7 +798,7 @@ public class CollapsingQParserPlugin ext
> scores[ord] = scorer.score();
> }
> }
> - } else if(this.collapsedSet.fastGet(globalDoc)) {
> + } else if(this.collapsedSet.get(globalDoc)) {
> // Elevated doc so do nothing.
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> if(comp.test(val, nullVal)) {
> @@ -801,7 +809,7 @@ public class CollapsingQParserPlugin ext
> }
> }
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> - this.collapsedSet.fastSet(globalDoc);
> + this.collapsedSet.set(globalDoc);
> if(needsScores) {
> nullScores.add(scorer.score());
> }
> @@ -858,7 +866,7 @@ public class CollapsingQParserPlugin ext
> scores[ord] = scorer.score();
> }
> }
> - } else if (this.collapsedSet.fastGet(globalDoc)) {
> + } else if (this.collapsedSet.get(globalDoc)) {
> //Elevated doc so do nothing
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> if(comp.test(val, nullVal)) {
> @@ -869,7 +877,7 @@ public class CollapsingQParserPlugin ext
> }
> }
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> - this.collapsedSet.fastSet(globalDoc);
> + this.collapsedSet.set(globalDoc);
> if(needsScores) {
> nullScores.add(scorer.score());
> }
> @@ -927,7 +935,7 @@ public class CollapsingQParserPlugin ext
> scores[ord] = scorer.score();
> }
> }
> - } else if (this.collapsedSet.fastGet(globalDoc)) {
> + } else if (this.collapsedSet.get(globalDoc)) {
> //Elevated doc so do nothing
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> if(comp.test(val, nullVal)) {
> @@ -938,7 +946,7 @@ public class CollapsingQParserPlugin ext
> }
> }
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> - this.collapsedSet.fastSet(globalDoc);
> + this.collapsedSet.set(globalDoc);
> if(needsScores) {
> nullScores.add(scorer.score());
> }
> @@ -1015,7 +1023,7 @@ public class CollapsingQParserPlugin ext
> scores[ord] = score;
> }
> }
> - } else if (this.collapsedSet.fastGet(globalDoc)) {
> + } else if (this.collapsedSet.get(globalDoc)) {
> //Elevated doc so do nothing
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> if(comp.test(val, nullVal)) {
> @@ -1026,7 +1034,7 @@ public class CollapsingQParserPlugin ext
> }
> }
> } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> - this.collapsedSet.fastSet(globalDoc);
> + this.collapsedSet.set(globalDoc);
> if(needsScores) {
> nullScores.add(score);
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java Sun Feb 16 07:50:08 2014
> @@ -18,7 +18,6 @@
> package org.apache.solr.search;
>
> import org.apache.lucene.search.Filter;
> -import org.apache.lucene.util.OpenBitSet;
> import org.apache.solr.common.SolrException;
>
> /**
> @@ -29,7 +28,6 @@ import org.apache.solr.common.SolrExcept
> * a cache and could be shared.
> * </p>
> *
> - *
> * @since solr 0.9
> */
> public interface DocSet /* extends Collection<Integer> */ {
> @@ -78,16 +76,6 @@ public interface DocSet /* extends Colle
> public DocIterator iterator();
>
> /**
> - * Returns a BitSet view of the DocSet. Any changes to this BitSet <b>may</b>
> - * be reflected in the DocSet, hence if the DocSet is shared or was returned from
> - * a SolrIndexSearcher method, it's not safe to modify the BitSet.
> - *
> - * @return
> - * An OpenBitSet with the bit number of every docid set in the set.
> - */
> - public OpenBitSet getBits();
> -
> - /**
> * Returns the approximate amount of memory taken by this DocSet.
> * This is only an approximation and doesn't take into account java object overhead.
> *
> @@ -145,10 +133,11 @@ public interface DocSet /* extends Colle
> public Filter getTopFilter();
>
> /**
> - * Takes the docs from this set and sets those bits on the target OpenBitSet.
> - * The target should be sized large enough to accommodate all of the documents before calling this method.
> + * Adds all the docs from this set to the target set. The target should be
> + * sized large enough to accommodate all of the documents before calling this
> + * method.
> */
> - public void setBitsOn(OpenBitSet target);
> + public void addAllTo(DocSet target);
>
> public static DocSet EMPTY = new SortedIntDocSet(new int[0], 0);
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java Sun Feb 16 07:50:08 2014
> @@ -18,18 +18,32 @@
> package org.apache.solr.search;
>
> import org.apache.lucene.index.AtomicReader;
> -import org.apache.solr.common.SolrException;
> -import org.apache.lucene.util.Bits;
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.index.AtomicReaderContext;
> +import org.apache.lucene.search.BitsFilteredDocIdSet;
> import org.apache.lucene.search.DocIdSet;
> -import org.apache.lucene.search.Filter;
> import org.apache.lucene.search.DocIdSetIterator;
> -import org.apache.lucene.search.BitsFilteredDocIdSet;
> -import org.apache.lucene.index.AtomicReaderContext;
> +import org.apache.lucene.search.Filter;
> +import org.apache.lucene.util.Bits;
> +import org.apache.lucene.util.FixedBitSet;
> +import org.apache.solr.common.SolrException;
>
> /** A base class that may be usefull for implementing DocSets */
> abstract class DocSetBase implements DocSet {
>
> + public static FixedBitSet toBitSet(DocSet set) {
> + if (set instanceof DocSetBase) {
> + return ((DocSetBase) set).getBits();
> + } else {
> + FixedBitSet bits = new FixedBitSet(64);
> + for (DocIterator iter = set.iterator(); iter.hasNext();) {
> + int nextDoc = iter.nextDoc();
> + bits = FixedBitSet.ensureCapacity(bits, nextDoc);
> + bits.set(nextDoc);
> + }
> + return bits;
> + }
> + }
> +
> // Not implemented efficiently... for testing purposes only
> @Override
> public boolean equals(Object obj) {
> @@ -49,7 +63,7 @@ abstract class DocSetBase implements Doc
> }
>
> // if (this.size() != other.size()) return false;
> - return this.getBits().equals(other.getBits());
> + return this.getBits().equals(toBitSet(other));
> }
>
> /**
> @@ -69,18 +83,20 @@ abstract class DocSetBase implements Doc
> }
>
> /**
> - * Inefficient base implementation.
> - *
> - * @see BitDocSet#getBits
> + * Return a {@link FixedBitSet} with a bit set for every document in this
> + * {@link DocSet}. The default implementation iterates on all docs and sets
> + * the relevant bits. You should override if you can provide a more efficient
> + * implementation.
> */
> - @Override
> - public OpenBitSet getBits() {
> - OpenBitSet bits = new OpenBitSet();
> + protected FixedBitSet getBits() {
> + FixedBitSet bits = new FixedBitSet(64);
> for (DocIterator iter = iterator(); iter.hasNext();) {
> - bits.set(iter.nextDoc());
> + int nextDoc = iter.nextDoc();
> + bits = FixedBitSet.ensureCapacity(bits, nextDoc);
> + bits.set(nextDoc);
> }
> return bits;
> - };
> + }
>
> @Override
> public DocSet intersection(DocSet other) {
> @@ -91,8 +107,8 @@ abstract class DocSetBase implements Doc
> }
>
> // Default... handle with bitsets.
> - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> - newbits.and(other.getBits());
> + FixedBitSet newbits = getBits().clone();
> + newbits.and(toBitSet(other));
> return new BitDocSet(newbits);
> }
>
> @@ -107,11 +123,11 @@ abstract class DocSetBase implements Doc
> return intersectionSize(other) > 0;
> }
>
> -
> @Override
> public DocSet union(DocSet other) {
> - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> - newbits.or(other.getBits());
> + FixedBitSet otherBits = toBitSet(other);
> + FixedBitSet newbits = FixedBitSet.ensureCapacity(getBits().clone(), otherBits.length());
> + newbits.or(otherBits);
> return new BitDocSet(newbits);
> }
>
> @@ -133,8 +149,8 @@ abstract class DocSetBase implements Doc
>
> @Override
> public DocSet andNot(DocSet other) {
> - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> - newbits.andNot(other.getBits());
> + FixedBitSet newbits = getBits().clone();
> + newbits.andNot(toBitSet(other));
> return new BitDocSet(newbits);
> }
>
> @@ -145,7 +161,7 @@ abstract class DocSetBase implements Doc
>
> @Override
> public Filter getTopFilter() {
> - final OpenBitSet bs = getBits();
> + final FixedBitSet bs = getBits();
>
> return new Filter() {
> @Override
> @@ -189,7 +205,7 @@ abstract class DocSetBase implements Doc
>
> @Override
> public long cost() {
> - return bs.capacity();
> + return bs.length();
> }
> };
> }
> @@ -211,10 +227,10 @@ abstract class DocSetBase implements Doc
> }
>
> @Override
> - public void setBitsOn(OpenBitSet target) {
> + public void addAllTo(DocSet target) {
> DocIterator iter = iterator();
> while (iter.hasNext()) {
> - target.fastSet(iter.nextDoc());
> + target.add(iter.nextDoc());
> }
> }
>
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java Sun Feb 16 07:50:08 2014
> @@ -17,13 +17,12 @@ package org.apache.solr.search;
> * limitations under the License.
> */
>
> -import org.apache.lucene.index.IndexReader;
> +import java.io.IOException;
> +
> import org.apache.lucene.index.AtomicReaderContext;
> import org.apache.lucene.search.Collector;
> import org.apache.lucene.search.Scorer;
> -import org.apache.lucene.util.OpenBitSet;
> -
> -import java.io.IOException;
> +import org.apache.lucene.util.FixedBitSet;
>
> /**
> *
> @@ -31,7 +30,7 @@ import java.io.IOException;
>
> public class DocSetCollector extends Collector {
> int pos=0;
> - OpenBitSet bits;
> + FixedBitSet bits;
> final int maxDoc;
> final int smallSetSize;
> int base;
> @@ -62,8 +61,8 @@ public class DocSetCollector extends Col
> } else {
> // this conditional could be removed if BitSet was preallocated, but that
> // would take up more memory, and add more GC time...
> - if (bits==null) bits = new OpenBitSet(maxDoc);
> - bits.fastSet(doc);
> + if (bits==null) bits = new FixedBitSet(maxDoc);
> + bits.set(doc);
> }
>
> pos++;
> @@ -75,7 +74,7 @@ public class DocSetCollector extends Col
> return new SortedIntDocSet(scratch, pos);
> } else {
> // set the bits for ids that were collected in the array
> - for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
> + for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
> return new BitDocSet(bits,pos);
> }
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java Sun Feb 16 07:50:08 2014
> @@ -17,13 +17,12 @@ package org.apache.solr.search;
> * limitations under the License.
> */
>
> -import org.apache.lucene.index.IndexReader;
> +import java.io.IOException;
> +
> import org.apache.lucene.index.AtomicReaderContext;
> import org.apache.lucene.search.Collector;
> import org.apache.lucene.search.Scorer;
> -import org.apache.lucene.util.OpenBitSet;
> -
> -import java.io.IOException;
> +import org.apache.lucene.util.FixedBitSet;
>
> /**
> *
> @@ -53,8 +52,8 @@ public class DocSetDelegateCollector ext
> } else {
> // this conditional could be removed if BitSet was preallocated, but that
> // would take up more memory, and add more GC time...
> - if (bits==null) bits = new OpenBitSet(maxDoc);
> - bits.fastSet(doc);
> + if (bits==null) bits = new FixedBitSet(maxDoc);
> + bits.set(doc);
> }
>
> pos++;
> @@ -67,7 +66,7 @@ public class DocSetDelegateCollector ext
> return new SortedIntDocSet(scratch, pos);
> } else {
> // set the bits for ids that were collected in the array
> - for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
> + for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
> return new BitDocSet(bits,pos);
> }
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java Sun Feb 16 07:50:08 2014
> @@ -17,13 +17,38 @@
>
> package org.apache.solr.search;
>
> +import java.io.IOException;
> +import java.util.ArrayList;
> +import java.util.Collection;
> +import java.util.LinkedHashSet;
> +import java.util.List;
> +import java.util.Locale;
> +import java.util.Map;
> +import java.util.Set;
> +
> import org.apache.commons.lang.ArrayUtils;
> import org.apache.lucene.index.IndexableField;
> import org.apache.lucene.queries.function.FunctionQuery;
> import org.apache.lucene.queries.function.ValueSource;
> import org.apache.lucene.queries.function.valuesource.QueryValueSource;
> -import org.apache.lucene.search.*;
> -import org.apache.lucene.search.grouping.*;
> +import org.apache.lucene.search.CachingCollector;
> +import org.apache.lucene.search.Collector;
> +import org.apache.lucene.search.Filter;
> +import org.apache.lucene.search.MultiCollector;
> +import org.apache.lucene.search.Query;
> +import org.apache.lucene.search.ScoreDoc;
> +import org.apache.lucene.search.Sort;
> +import org.apache.lucene.search.SortField;
> +import org.apache.lucene.search.TimeLimitingCollector;
> +import org.apache.lucene.search.TopDocs;
> +import org.apache.lucene.search.TopDocsCollector;
> +import org.apache.lucene.search.TopFieldCollector;
> +import org.apache.lucene.search.TopScoreDocCollector;
> +import org.apache.lucene.search.TotalHitCountCollector;
> +import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
> +import org.apache.lucene.search.grouping.GroupDocs;
> +import org.apache.lucene.search.grouping.SearchGroup;
> +import org.apache.lucene.search.grouping.TopGroups;
> import org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
> import org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector;
> import org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector;
> @@ -33,8 +58,6 @@ import org.apache.lucene.search.grouping
> import org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector;
> import org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector;
> import org.apache.lucene.util.BytesRef;
> -import org.apache.lucene.util.FixedBitSet;
> -import org.apache.lucene.util.OpenBitSet;
> import org.apache.lucene.util.mutable.MutableValue;
> import org.apache.solr.common.SolrException;
> import org.apache.solr.common.util.NamedList;
> @@ -47,9 +70,6 @@ import org.apache.solr.search.grouping.c
> import org.slf4j.Logger;
> import org.slf4j.LoggerFactory;
>
> -import java.io.IOException;
> -import java.util.*;
> -
> /**
> * Basic Solr Grouping infrastructure.
> * Warning NOT thread save!
> @@ -353,10 +373,7 @@ public class Grouping {
> }
>
> if (getGroupedDocSet && allGroupHeadsCollector != null) {
> - FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
> - long[] bits = fixedBitSet.getBits();
> - OpenBitSet openBitSet = new OpenBitSet(bits, bits.length);
> - qr.setDocSet(new BitDocSet(openBitSet));
> + qr.setDocSet(new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(maxDoc)));
> } else if (getDocSet) {
> qr.setDocSet(setCollector.getDocSet());
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java Sun Feb 16 07:50:08 2014
> @@ -16,12 +16,33 @@
> */
> package org.apache.solr.search;
>
> -import org.apache.lucene.index.*;
> -import org.apache.lucene.search.*;
> +import java.io.Closeable;
> +import java.io.IOException;
> +import java.util.ArrayList;
> +import java.util.Arrays;
> +import java.util.List;
> +import java.util.Set;
> +
> +import org.apache.lucene.index.AtomicReaderContext;
> +import org.apache.lucene.index.DocsEnum;
> +import org.apache.lucene.index.Fields;
> +import org.apache.lucene.index.IndexReader;
> +import org.apache.lucene.index.MultiDocsEnum;
> +import org.apache.lucene.index.Terms;
> +import org.apache.lucene.index.TermsEnum;
> +import org.apache.lucene.search.ComplexExplanation;
> +import org.apache.lucene.search.DocIdSet;
> +import org.apache.lucene.search.DocIdSetIterator;
> +import org.apache.lucene.search.Explanation;
> +import org.apache.lucene.search.Filter;
> +import org.apache.lucene.search.IndexSearcher;
> +import org.apache.lucene.search.Query;
> +import org.apache.lucene.search.Scorer;
> +import org.apache.lucene.search.Weight;
> import org.apache.lucene.search.similarities.Similarity;
> import org.apache.lucene.util.Bits;
> import org.apache.lucene.util.BytesRef;
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.lucene.util.StringHelper;
> import org.apache.solr.common.SolrException;
> import org.apache.solr.common.params.SolrParams;
> @@ -36,13 +57,6 @@ import org.apache.solr.request.SolrReque
> import org.apache.solr.schema.TrieField;
> import org.apache.solr.util.RefCounted;
>
> -import java.io.Closeable;
> -import java.io.IOException;
> -import java.util.ArrayList;
> -import java.util.Arrays;
> -import java.util.List;
> -import java.util.Set;
> -
>
> public class JoinQParserPlugin extends QParserPlugin {
> public static final String NAME = "join";
> @@ -270,7 +284,7 @@ class JoinQuery extends Query {
>
>
> public DocSet getDocSet() throws IOException {
> - OpenBitSet resultBits = null;
> + FixedBitSet resultBits = null;
>
> // minimum docFreq to use the cache
> int minDocFreqFrom = Math.max(5, fromSearcher.maxDoc() >> 13);
> @@ -387,7 +401,7 @@ class JoinQuery extends Query {
> int df = toTermsEnum.docFreq();
> toTermHitsTotalDf += df;
> if (resultBits==null && df + resultListDocs > maxSortedIntSize && resultList.size() > 0) {
> - resultBits = new OpenBitSet(toSearcher.maxDoc());
> + resultBits = new FixedBitSet(toSearcher.maxDoc());
> }
>
> // if we don't have a bitset yet, or if the resulting set will be too large
> @@ -397,10 +411,10 @@ class JoinQuery extends Query {
> DocSet toTermSet = toSearcher.getDocSet(toDeState);
> resultListDocs += toTermSet.size();
> if (resultBits != null) {
> - toTermSet.setBitsOn(resultBits);
> + toTermSet.addAllTo(new BitDocSet(resultBits));
> } else {
> if (toTermSet instanceof BitDocSet) {
> - resultBits = (OpenBitSet)((BitDocSet)toTermSet).bits.clone();
> + resultBits = ((BitDocSet)toTermSet).bits.clone();
> } else {
> resultList.add(toTermSet);
> }
> @@ -422,14 +436,14 @@ class JoinQuery extends Query {
> int docid;
> while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> resultListDocs++;
> - resultBits.fastSet(docid + base);
> + resultBits.set(docid + base);
> }
> }
> } else {
> int docid;
> while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> resultListDocs++;
> - resultBits.fastSet(docid);
> + resultBits.set(docid);
> }
> }
> }
> @@ -443,10 +457,11 @@ class JoinQuery extends Query {
> smallSetsDeferred = resultList.size();
>
> if (resultBits != null) {
> + BitDocSet bitSet = new BitDocSet(resultBits);
> for (DocSet set : resultList) {
> - set.setBitsOn(resultBits);
> + set.addAllTo(bitSet);
> }
> - return new BitDocSet(resultBits);
> + return bitSet;
> }
>
> if (resultList.size()==0) {
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sun Feb 16 07:50:08 2014
> @@ -65,11 +65,11 @@ import org.apache.lucene.search.Constant
> import org.apache.lucene.search.DocIdSet;
> import org.apache.lucene.search.DocIdSetIterator;
> import org.apache.lucene.search.Explanation;
> +import org.apache.lucene.search.FieldDoc;
> import org.apache.lucene.search.Filter;
> import org.apache.lucene.search.IndexSearcher;
> import org.apache.lucene.search.MatchAllDocsQuery;
> import org.apache.lucene.search.Query;
> -import org.apache.lucene.search.FieldDoc;
> import org.apache.lucene.search.ScoreDoc;
> import org.apache.lucene.search.Scorer;
> import org.apache.lucene.search.Sort;
> @@ -77,16 +77,16 @@ import org.apache.lucene.search.SortFiel
> import org.apache.lucene.search.TermQuery;
> import org.apache.lucene.search.TimeLimitingCollector;
> import org.apache.lucene.search.TopDocs;
> -import org.apache.lucene.search.TopFieldDocs;
> import org.apache.lucene.search.TopDocsCollector;
> import org.apache.lucene.search.TopFieldCollector;
> +import org.apache.lucene.search.TopFieldDocs;
> import org.apache.lucene.search.TopScoreDocCollector;
> import org.apache.lucene.search.TotalHitCountCollector;
> import org.apache.lucene.search.Weight;
> import org.apache.lucene.store.Directory;
> import org.apache.lucene.util.Bits;
> import org.apache.lucene.util.BytesRef;
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.solr.common.SolrException;
> import org.apache.solr.common.SolrException.ErrorCode;
> import org.apache.solr.common.params.ModifiableSolrParams;
> @@ -104,7 +104,6 @@ import org.apache.solr.request.UnInverte
> import org.apache.solr.response.SolrQueryResponse;
> import org.apache.solr.schema.IndexSchema;
> import org.apache.solr.schema.SchemaField;
> -import org.apache.solr.spelling.QueryConverter;
> import org.apache.solr.update.SolrIndexConfig;
> import org.slf4j.Logger;
> import org.slf4j.LoggerFactory;
> @@ -1087,7 +1086,7 @@ public class SolrIndexSearcher extends I
> final int[] docs = deState.scratch;
> int upto = 0;
> int bitsSet = 0;
> - OpenBitSet obs = null;
> + FixedBitSet fbs = null;
>
> DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, DocsEnum.FLAG_NONE);
> if (deState.docsEnum == null) {
> @@ -1104,9 +1103,9 @@ public class SolrIndexSearcher extends I
> int docid;
>
> if (largestPossible > docs.length) {
> - if (obs == null) obs = new OpenBitSet(maxDoc());
> + if (fbs == null) fbs = new FixedBitSet(maxDoc());
> while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> - obs.fastSet(docid + base);
> + fbs.set(docid + base);
> bitsSet++;
> }
> } else {
> @@ -1118,9 +1117,9 @@ public class SolrIndexSearcher extends I
> } else {
> int docid;
> if (largestPossible > docs.length) {
> - if (obs == null) obs = new OpenBitSet(maxDoc());
> + if (fbs == null) fbs = new FixedBitSet(maxDoc());
> while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> - obs.fastSet(docid);
> + fbs.set(docid);
> bitsSet++;
> }
> } else {
> @@ -1131,12 +1130,12 @@ public class SolrIndexSearcher extends I
> }
>
> DocSet result;
> - if (obs != null) {
> + if (fbs != null) {
> for (int i=0; i<upto; i++) {
> - obs.fastSet(docs[i]);
> + fbs.set(docs[i]);
> }
> bitsSet += upto;
> - result = new BitDocSet(obs, bitsSet);
> + result = new BitDocSet(fbs, bitsSet);
> } else {
> result = upto==0 ? DocSet.EMPTY : new SortedIntDocSet(Arrays.copyOf(docs, upto));
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java Sun Feb 16 07:50:08 2014
> @@ -18,13 +18,13 @@
> package org.apache.solr.search;
>
> import org.apache.lucene.index.AtomicReader;
> -import org.apache.lucene.util.Bits;
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.index.AtomicReaderContext;
> import org.apache.lucene.search.BitsFilteredDocIdSet;
> import org.apache.lucene.search.DocIdSet;
> import org.apache.lucene.search.DocIdSetIterator;
> import org.apache.lucene.search.Filter;
> -import org.apache.lucene.index.AtomicReaderContext;
> +import org.apache.lucene.util.Bits;
> +import org.apache.lucene.util.FixedBitSet;
>
> /**
> * <code>SortedIntDocSet</code> represents a sorted set of Lucene Document Ids.
> @@ -563,13 +563,12 @@ public class SortedIntDocSet extends Doc
> }
>
> @Override
> - public void setBitsOn(OpenBitSet target) {
> + public void addAllTo(DocSet target) {
> for (int doc : docs) {
> - target.fastSet(doc);
> + target.add(doc);
> }
> }
>
> -
> @Override
> public boolean exists(int doc) {
> // this could be faster by estimating where in the list the doc is likely to appear,
> @@ -630,16 +629,15 @@ public class SortedIntDocSet extends Doc
> }
>
> @Override
> - public OpenBitSet getBits() {
> + public FixedBitSet getBits() {
> int maxDoc = size() > 0 ? docs[size()-1] : 0;
> - OpenBitSet bs = new OpenBitSet(maxDoc+1);
> + FixedBitSet bs = new FixedBitSet(maxDoc+1);
> for (int doc : docs) {
> - bs.fastSet(doc);
> + bs.set(doc);
> }
> return bs;
> }
>
> -
> public static int findIndex(int[] arr, int value, int low, int high) {
> // binary search
> while (low <= high) {
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java Sun Feb 16 07:50:08 2014
> @@ -17,22 +17,31 @@ package org.apache.solr.search.grouping;
> * limitations under the License.
> */
>
> -import org.apache.lucene.search.*;
> +import java.io.IOException;
> +import java.util.ArrayList;
> +import java.util.List;
> +
> +import org.apache.lucene.search.Collector;
> +import org.apache.lucene.search.Filter;
> +import org.apache.lucene.search.MultiCollector;
> +import org.apache.lucene.search.Query;
> +import org.apache.lucene.search.TimeLimitingCollector;
> +import org.apache.lucene.search.TotalHitCountCollector;
> import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
> import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.solr.common.util.NamedList;
> -import org.apache.solr.search.*;
> -import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
> +import org.apache.solr.search.BitDocSet;
> +import org.apache.solr.search.DocSet;
> +import org.apache.solr.search.DocSetCollector;
> +import org.apache.solr.search.DocSetDelegateCollector;
> import org.apache.solr.search.QueryUtils;
> +import org.apache.solr.search.SolrIndexSearcher;
> +import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
> import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
> import org.slf4j.Logger;
> import org.slf4j.LoggerFactory;
>
> -import java.io.IOException;
> -import java.util.ArrayList;
> -import java.util.List;
> -
> /**
> * Responsible for executing a search with a number of {@link Command} instances.
> * A typical search can have more then one {@link Command} instances.
> @@ -159,9 +168,7 @@ public class CommandHandler {
> searchWithTimeLimiter(query, filter, MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])));
> }
>
> - int maxDoc = searcher.maxDoc();
> - long[] bits = termAllGroupHeadsCollector.retrieveGroupHeads(maxDoc).getBits();
> - return new BitDocSet(new OpenBitSet(bits, bits.length));
> + return new BitDocSet(termAllGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
> }
>
> private DocSet computeDocSet(Query query, ProcessedFilter filter, List<Collector> collectors) throws IOException {
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java Sun Feb 16 07:50:08 2014
> @@ -17,30 +17,30 @@
>
> package org.apache.solr.search.join;
>
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.FixedBitSet;
>
> class BitSetSlice {
> - private final OpenBitSet obs;
> + private final FixedBitSet fbs;
> private final int off;
> private final int len;
>
> - BitSetSlice(OpenBitSet obs, int off, int len) {
> - this.obs = obs;
> + BitSetSlice(FixedBitSet fbs, int off, int len) {
> + this.fbs = fbs;
> this.off = off;
> this.len = len;
> }
>
> public boolean get(int pos) {
> - return obs.get(pos + off);
> + return fbs.get(pos + off);
> }
>
> public int prevSetBit(int pos) {
> - int result = obs.prevSetBit(pos + off) - off;
> + int result = fbs.prevSetBit(pos + off) - off;
> return (result < 0) ? -1 : result;
> }
>
> public int nextSetBit(int pos) {
> - int result = obs.nextSetBit(pos + off) - off;
> + int result = fbs.nextSetBit(pos + off) - off;
> return (result < 0 || result >= len) ? -1 : result;
> }
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java Sun Feb 16 07:50:08 2014
> @@ -19,7 +19,7 @@ package org.apache.solr.store.blockcache
>
> import java.util.concurrent.atomic.AtomicLongArray;
>
> -import org.apache.lucene.util.OpenBitSet;
> +import org.apache.lucene.util.LongBitSet;
>
> public class BlockLocks {
>
> @@ -27,7 +27,7 @@ public class BlockLocks {
> private int wlen;
>
> public BlockLocks(long numBits) {
> - int length = OpenBitSet.bits2words(numBits);
> + int length = LongBitSet.bits2words(numBits);
> bits = new AtomicLongArray(length);
> wlen = length;
> }
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Sun Feb 16 07:50:08 2014
> @@ -17,6 +17,10 @@
>
> package org.apache.solr.update;
>
> +import java.io.IOException;
> +import java.util.ArrayList;
> +import java.util.List;
> +
> import org.apache.lucene.index.AtomicReader;
> import org.apache.lucene.index.AtomicReaderContext;
> import org.apache.lucene.index.DocsEnum;
> @@ -26,11 +30,12 @@ import org.apache.lucene.index.IndexRead
> import org.apache.lucene.index.IndexWriter;
> import org.apache.lucene.index.Terms;
> import org.apache.lucene.index.TermsEnum;
> +import org.apache.lucene.search.DocIdSetIterator;
> import org.apache.lucene.util.Bits;
> import org.apache.lucene.util.BytesRef;
> import org.apache.lucene.util.CharsRef;
> +import org.apache.lucene.util.FixedBitSet;
> import org.apache.lucene.util.IOUtils;
> -import org.apache.lucene.util.OpenBitSet;
> import org.apache.solr.common.cloud.CompositeIdRouter;
> import org.apache.solr.common.cloud.DocRouter;
> import org.apache.solr.common.cloud.HashBasedRouter;
> @@ -41,10 +46,6 @@ import org.apache.solr.util.RefCounted;
> import org.slf4j.Logger;
> import org.slf4j.LoggerFactory;
>
> -import java.io.IOException;
> -import java.util.ArrayList;
> -import java.util.List;
> -
> public class SolrIndexSplitter {
> public static Logger log = LoggerFactory.getLogger(SolrIndexSplitter.class);
>
> @@ -89,13 +90,13 @@ public class SolrIndexSplitter {
> public void split() throws IOException {
>
> List<AtomicReaderContext> leaves = searcher.getTopReaderContext().leaves();
> - List<OpenBitSet[]> segmentDocSets = new ArrayList<OpenBitSet[]>(leaves.size());
> + List<FixedBitSet[]> segmentDocSets = new ArrayList<FixedBitSet[]>(leaves.size());
>
> log.info("SolrIndexSplitter: partitions=" + numPieces + " segments="+leaves.size());
>
> for (AtomicReaderContext readerContext : leaves) {
> assert readerContext.ordInParent == segmentDocSets.size(); // make sure we're going in order
> - OpenBitSet[] docSets = split(readerContext);
> + FixedBitSet[] docSets = split(readerContext);
> segmentDocSets.add( docSets );
> }
>
> @@ -150,11 +151,11 @@ public class SolrIndexSplitter {
>
>
>
> - OpenBitSet[] split(AtomicReaderContext readerContext) throws IOException {
> + FixedBitSet[] split(AtomicReaderContext readerContext) throws IOException {
> AtomicReader reader = readerContext.reader();
> - OpenBitSet[] docSets = new OpenBitSet[numPieces];
> + FixedBitSet[] docSets = new FixedBitSet[numPieces];
> for (int i=0; i<docSets.length; i++) {
> - docSets[i] = new OpenBitSet(reader.maxDoc());
> + docSets[i] = new FixedBitSet(reader.maxDoc());
> }
> Bits liveDocs = reader.getLiveDocs();
>
> @@ -196,14 +197,14 @@ public class SolrIndexSplitter {
> docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
> for (;;) {
> int doc = docsEnum.nextDoc();
> - if (doc == DocsEnum.NO_MORE_DOCS) break;
> + if (doc == DocIdSetIterator.NO_MORE_DOCS) break;
> if (ranges == null) {
> - docSets[currPartition].fastSet(doc);
> + docSets[currPartition].set(doc);
> currPartition = (currPartition + 1) % numPieces;
> } else {
> for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
> if (rangesArr[i].includes(hash)) {
> - docSets[i].fastSet(doc);
> + docSets[i].set(doc);
> }
> }
> }
> @@ -232,13 +233,13 @@ public class SolrIndexSplitter {
>
> // change livedocs on the reader to delete those docs we don't want
> static class LiveDocsReader extends FilterAtomicReader {
> - final OpenBitSet liveDocs;
> + final FixedBitSet liveDocs;
> final int numDocs;
>
> - public LiveDocsReader(AtomicReaderContext context, OpenBitSet liveDocs) throws IOException {
> + public LiveDocsReader(AtomicReaderContext context, FixedBitSet liveDocs) throws IOException {
> super(context.reader());
> this.liveDocs = liveDocs;
> - this.numDocs = (int)liveDocs.cardinality();
> + this.numDocs = liveDocs.cardinality();
> }
>
> @Override
> @@ -253,6 +254,3 @@ public class SolrIndexSplitter {
> }
>
> }
> -
> -
> -
>
> Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java (original)
> +++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java Sun Feb 16 07:50:08 2014
> @@ -17,13 +17,10 @@
>
> package org.apache.solr.search;
>
> -import org.apache.solr.search.BitDocSet;
> -import org.apache.solr.search.HashDocSet;
> -import org.apache.solr.search.DocSet;
> -import org.apache.lucene.util.OpenBitSet;
> -
> -import java.util.Random;
> import java.util.BitSet;
> +import java.util.Random;
> +
> +import org.apache.lucene.util.FixedBitSet;
>
> /**
> */
> @@ -38,21 +35,20 @@ public class DocSetPerf {
>
> static Random rand = new Random();
>
> -
> - static OpenBitSet bs;
> + static FixedBitSet bs;
> static BitDocSet bds;
> static HashDocSet hds;
> static int[] ids; // not unique
>
> static void generate(int maxSize, int bitsToSet) {
> - bs = new OpenBitSet(maxSize);
> + bs = new FixedBitSet(maxSize);
> ids = new int[bitsToSet];
> int count=0;
> if (maxSize>0) {
> for (int i=0; i<bitsToSet; i++) {
> int id=rand.nextInt(maxSize);
> if (!bs.get(id)) {
> - bs.fastSet(id);
> + bs.set(id);
> ids[count++]=id;
> }
> }
> @@ -80,7 +76,7 @@ public class DocSetPerf {
>
> int ret=0;
>
> - OpenBitSet[] sets = new OpenBitSet[numSets];
> + FixedBitSet[] sets = new FixedBitSet[numSets];
> DocSet[] bset = new DocSet[numSets];
> DocSet[] hset = new DocSet[numSets];
> BitSet scratch=new BitSet();
> @@ -97,14 +93,14 @@ public class DocSetPerf {
> if ("test".equals(test)) {
> for (int it=0; it<iter; it++) {
> generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
> - OpenBitSet bs1=bs;
> + FixedBitSet bs1=bs;
> BitDocSet bds1=bds;
> HashDocSet hds1=hds;
> generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
>
> - OpenBitSet res = ((OpenBitSet)bs1.clone());
> + FixedBitSet res = bs1.clone();
> res.and(bs);
> - int icount = (int)res.cardinality();
> + int icount = res.cardinality();
>
> test(bds1.intersection(bds).size() == icount);
> test(bds1.intersectionSize(bds) == icount);
> @@ -167,8 +163,6 @@ public class DocSetPerf {
> }
> }
>
> -
> -
> long end = System.currentTimeMillis();
> System.out.println("TIME="+(end-start));
>
> @@ -176,6 +170,4 @@ public class DocSetPerf {
> System.out.println("ret="+ret);
> }
>
> -
> -
> }
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1568738 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/util/ lucene/grouping/ lucene/grouping/src/java/org/apache/lucene/search/grouping/ solr/ solr/core/ solr/core/src/java/org/ap...
Posted by Mark Miller <ma...@gmail.com>.
I guess the simplest test I see it in to look at is TestDistributedSearch.
The trace I’m seeing is:
Caused by: java.lang.AssertionError: index=31, numBits=31
at org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
at org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
at org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
... 27 more
Not code I’m super familiar with unfortunately...
- Mark
http://about.me/markrmiller
On Feb 16, 2014, at 3:07 PM, Shai Erera <se...@gmail.com> wrote:
> I ran Solr tests both on trunk and 4x several times before I committed. I'll dig.
>
> Shai
>
>
> On Sun, Feb 16, 2014 at 9:59 PM, Mark Miller <ma...@gmail.com> wrote:
> FYI, somehow this issue broke Solr tests I think:
>
> [junit4] 2> Caused by: java.lang.AssertionError: index=36, numBits=36
> [junit4] 2> at org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
> [junit4] 2> at org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
> [junit4] 2> at org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
> [junit4] 2> at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
> [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
> [junit4] 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
> [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
> [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
> [junit4] 2> ... 26 more
> [junit4] 2>
>
> [junit4] 2> 2818355 T8214 oejs.ServletHandler.doHandle WARN Error for /select java.lang.AssertionError: index=36, numBits=36
> [junit4] 2> at org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
> [junit4] 2> at org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
> [junit4] 2> at org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
> [junit4] 2> at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
> [junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
> [junit4] 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
> [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
> [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
> [junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
> [junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
> [junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:134)
>
> - Mark
>
> http://about.me/markrmiller
>
> On Feb 16, 2014, at 2:50 AM, shaie@apache.org wrote:
>
> > Author: shaie
> > Date: Sun Feb 16 07:50:08 2014
> > New Revision: 1568738
> >
> > URL: http://svn.apache.org/r1568738
> > Log:
> > LUCENE-5440: decouple OpenBitSet from DocSet and move to use FixedBitSet
> >
> > Modified:
> > lucene/dev/branches/branch_4x/ (props changed)
> > lucene/dev/branches/branch_4x/lucene/ (props changed)
> > lucene/dev/branches/branch_4x/lucene/core/ (props changed)
> > lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> > lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> > lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
> > lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> > lucene/dev/branches/branch_4x/solr/ (props changed)
> > lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
> > lucene/dev/branches/branch_4x/solr/core/ (props changed)
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> > lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> > lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> > lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestDocSet.java
> > lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestFiltering.java
> > lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestSort.java
> > lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/util/BitSetPerf.java
> >
> > Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
> > +++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Sun Feb 16 07:50:08 2014
> > @@ -28,6 +28,8 @@ import org.apache.lucene.search.DocIdSet
> > * long[], accessed with an int index, implementing {@link Bits} and
> > * {@link DocIdSet}. If you need to manage more than 2.1B bits, use
> > * {@link LongBitSet}.
> > + *
> > + * @lucene.internal
> > */
> > public final class FixedBitSet extends DocIdSet implements Bits {
> >
> > @@ -41,6 +43,12 @@ public final class FixedBitSet extends D
> > final long[] bits;
> > int doc = -1;
> >
> > + /** Creates an iterator over the given {@link FixedBitSet}. */
> > + public FixedBitSetIterator(FixedBitSet bits) {
> > + this(bits.bits, bits.numBits, bits.numWords);
> > + }
> > +
> > + /** Creates an iterator over the given array of bits. */
> > public FixedBitSetIterator(long[] bits, int numBits, int wordLength) {
> > this.bits = bits;
> > this.numBits = numBits;
> > @@ -48,7 +56,7 @@ public final class FixedBitSet extends D
> > }
> >
> > @Override
> > - public int nextDoc() throws IOException {
> > + public int nextDoc() {
> > if (doc == NO_MORE_DOCS || ++doc >= numBits) {
> > return doc = NO_MORE_DOCS;
> > }
> > @@ -81,7 +89,7 @@ public final class FixedBitSet extends D
> > }
> >
> > @Override
> > - public int advance(int target) throws IOException {
> > + public int advance(int target) {
> > if (doc == NO_MORE_DOCS || target >= numBits) {
> > return doc = NO_MORE_DOCS;
> > }
> > @@ -103,10 +111,6 @@ public final class FixedBitSet extends D
> > return doc = NO_MORE_DOCS;
> > }
> > }
> > -
> > - private final long[] bits;
> > - private final int numBits;
> > - private final int wordLength;
> >
> > /**
> > * If the given {@link FixedBitSet} is large enough to hold {@code numBits},
> > @@ -140,15 +144,53 @@ public final class FixedBitSet extends D
> > return numLong;
> > }
> >
> > + /**
> > + * Returns the popcount or cardinality of the intersection of the two sets.
> > + * Neither set is modified.
> > + */
> > + public static long intersectionCount(FixedBitSet a, FixedBitSet b) {
> > + return BitUtil.pop_intersect(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
> > + }
> > +
> > + /**
> > + * Returns the popcount or cardinality of the union of the two sets. Neither
> > + * set is modified.
> > + */
> > + public static long unionCount(FixedBitSet a, FixedBitSet b) {
> > + long tot = BitUtil.pop_union(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
> > + if (a.numWords < b.numWords) {
> > + tot += BitUtil.pop_array(b.bits, a.numWords, b.numWords - a.numWords);
> > + } else if (a.numWords > b.numWords) {
> > + tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
> > + }
> > + return tot;
> > + }
> > +
> > + /**
> > + * Returns the popcount or cardinality of "a and not b" or
> > + * "intersection(a, not(b))". Neither set is modified.
> > + */
> > + public static long andNotCount(FixedBitSet a, FixedBitSet b) {
> > + long tot = BitUtil.pop_andnot(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
> > + if (a.numWords > b.numWords) {
> > + tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
> > + }
> > + return tot;
> > + }
> > +
> > + final long[] bits;
> > + final int numBits;
> > + final int numWords;
> > +
> > public FixedBitSet(int numBits) {
> > this.numBits = numBits;
> > bits = new long[bits2words(numBits)];
> > - wordLength = bits.length;
> > + numWords = bits.length;
> > }
> >
> > public FixedBitSet(long[] storedBits, int numBits) {
> > - this.wordLength = bits2words(numBits);
> > - if (wordLength > storedBits.length) {
> > + this.numWords = bits2words(numBits);
> > + if (numWords > storedBits.length) {
> > throw new IllegalArgumentException("The given long array is too small to hold " + numBits + " bits");
> > }
> > this.numBits = numBits;
> > @@ -157,7 +199,7 @@ public final class FixedBitSet extends D
> >
> > @Override
> > public DocIdSetIterator iterator() {
> > - return new FixedBitSetIterator(bits, numBits, wordLength);
> > + return new FixedBitSetIterator(bits, numBits, numWords);
> > }
> >
> > @Override
> > @@ -190,7 +232,7 @@ public final class FixedBitSet extends D
> >
> > @Override
> > public boolean get(int index) {
> > - assert index >= 0 && index < numBits: "index=" + index;
> > + assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
> > int i = index >> 6; // div 64
> > // signed shift will keep a negative index and force an
> > // array-index-out-of-bounds-exception, removing the need for an explicit check.
> > @@ -200,7 +242,7 @@ public final class FixedBitSet extends D
> > }
> >
> > public void set(int index) {
> > - assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
> > + assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
> > int wordNum = index >> 6; // div 64
> > int bit = index & 0x3f; // mod 64
> > long bitmask = 1L << bit;
> > @@ -239,7 +281,7 @@ public final class FixedBitSet extends D
> > * -1 is returned if there are no more set bits.
> > */
> > public int nextSetBit(int index) {
> > - assert index >= 0 && index < numBits;
> > + assert index >= 0 && index < numBits : "index=" + index + ", numBits=" + numBits;
> > int i = index >> 6;
> > final int subIndex = index & 0x3f; // index within the word
> > long word = bits[i] >> subIndex; // skip all the bits to the right of index
> > @@ -248,7 +290,7 @@ public final class FixedBitSet extends D
> > return index + Long.numberOfTrailingZeros(word);
> > }
> >
> > - while(++i < wordLength) {
> > + while(++i < numWords) {
> > word = bits[i];
> > if (word != 0) {
> > return (i<<6) + Long.numberOfTrailingZeros(word);
> > @@ -306,12 +348,13 @@ public final class FixedBitSet extends D
> >
> > /** this = this OR other */
> > public void or(FixedBitSet other) {
> > - or(other.bits, other.wordLength);
> > + or(other.bits, other.numWords);
> > }
> >
> > - private void or(final long[] otherArr, final int otherLen) {
> > + private void or(final long[] otherArr, final int otherNumWords) {
> > + assert otherNumWords <= numWords : "numWords=" + numWords + ", otherNumWords=" + otherNumWords;
> > final long[] thisArr = this.bits;
> > - int pos = Math.min(wordLength, otherLen);
> > + int pos = Math.min(numWords, otherNumWords);
> > while (--pos >= 0) {
> > thisArr[pos] |= otherArr[pos];
> > }
> > @@ -319,9 +362,10 @@ public final class FixedBitSet extends D
> >
> > /** this = this XOR other */
> > public void xor(FixedBitSet other) {
> > + assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
> > final long[] thisBits = this.bits;
> > final long[] otherBits = other.bits;
> > - int pos = Math.min(wordLength, other.wordLength);
> > + int pos = Math.min(numWords, other.numWords);
> > while (--pos >= 0) {
> > thisBits[pos] ^= otherBits[pos];
> > }
> > @@ -366,7 +410,7 @@ public final class FixedBitSet extends D
> >
> > /** returns true if the sets have any elements in common */
> > public boolean intersects(FixedBitSet other) {
> > - int pos = Math.min(wordLength, other.wordLength);
> > + int pos = Math.min(numWords, other.numWords);
> > while (--pos>=0) {
> > if ((bits[pos] & other.bits[pos]) != 0) return true;
> > }
> > @@ -375,17 +419,17 @@ public final class FixedBitSet extends D
> >
> > /** this = this AND other */
> > public void and(FixedBitSet other) {
> > - and(other.bits, other.wordLength);
> > + and(other.bits, other.numWords);
> > }
> >
> > - private void and(final long[] otherArr, final int otherLen) {
> > + private void and(final long[] otherArr, final int otherNumWords) {
> > final long[] thisArr = this.bits;
> > - int pos = Math.min(this.wordLength, otherLen);
> > + int pos = Math.min(this.numWords, otherNumWords);
> > while(--pos >= 0) {
> > thisArr[pos] &= otherArr[pos];
> > }
> > - if (this.wordLength > otherLen) {
> > - Arrays.fill(thisArr, otherLen, this.wordLength, 0L);
> > + if (this.numWords > otherNumWords) {
> > + Arrays.fill(thisArr, otherNumWords, this.numWords, 0L);
> > }
> > }
> >
> > @@ -417,9 +461,9 @@ public final class FixedBitSet extends D
> > andNot(other.bits, other.bits.length);
> > }
> >
> > - private void andNot(final long[] otherArr, final int otherLen) {
> > + private void andNot(final long[] otherArr, final int otherNumWords) {
> > final long[] thisArr = this.bits;
> > - int pos = Math.min(this.wordLength, otherLen);
> > + int pos = Math.min(this.numWords, otherNumWords);
> > while(--pos >= 0) {
> > thisArr[pos] &= ~otherArr[pos];
> > }
> > @@ -502,8 +546,8 @@ public final class FixedBitSet extends D
> > * @param endIndex one-past the last bit to clear
> > */
> > public void clear(int startIndex, int endIndex) {
> > - assert startIndex >= 0 && startIndex < numBits;
> > - assert endIndex >= 0 && endIndex <= numBits;
> > + assert startIndex >= 0 && startIndex < numBits : "startIndex=" + startIndex + ", numBits=" + numBits;
> > + assert endIndex >= 0 && endIndex <= numBits : "endIndex=" + endIndex + ", numBits=" + numBits;
> > if (endIndex <= startIndex) {
> > return;
> > }
> > @@ -554,7 +598,7 @@ public final class FixedBitSet extends D
> > @Override
> > public int hashCode() {
> > long h = 0;
> > - for (int i = wordLength; --i>=0;) {
> > + for (int i = numWords; --i>=0;) {
> > h ^= bits[i];
> > h = (h << 1) | (h >>> 63); // rotate left
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java (original)
> > +++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java Sun Feb 16 07:50:08 2014
> > @@ -23,6 +23,8 @@ import java.util.Arrays;
> > * BitSet of fixed length (numBits), backed by accessible ({@link #getBits})
> > * long[], accessed with a long index. Use it only if you intend to store more
> > * than 2.1B bits, otherwise you should use {@link FixedBitSet}.
> > + *
> > + * @lucene.internal
> > */
> > public final class LongBitSet {
> >
> > @@ -188,6 +190,7 @@ public final class LongBitSet {
> >
> > /** this = this OR other */
> > public void or(LongBitSet other) {
> > + assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
> > int pos = Math.min(numWords, other.numWords);
> > while (--pos >= 0) {
> > bits[pos] |= other.bits[pos];
> > @@ -196,6 +199,7 @@ public final class LongBitSet {
> >
> > /** this = this XOR other */
> > public void xor(LongBitSet other) {
> > + assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
> > int pos = Math.min(numWords, other.numWords);
> > while (--pos >= 0) {
> > bits[pos] ^= other.bits[pos];
> >
> > Modified: lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java (original)
> > +++ lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java Sun Feb 16 07:50:08 2014
> > @@ -44,7 +44,7 @@ public abstract class AbstractAllGroupHe
> >
> > /**
> > * @param maxDoc The maxDoc of the top level {@link IndexReader}.
> > - * @return an {@link FixedBitSet} containing all group heads.
> > + * @return a {@link FixedBitSet} containing all group heads.
> > */
> > public FixedBitSet retrieveGroupHeads(int maxDoc) {
> > FixedBitSet bitSet = new FixedBitSet(maxDoc);
> >
> > Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
> > +++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sun Feb 16 07:50:08 2014
> > @@ -330,6 +330,10 @@ Optimizations
> > * SOLR-5624: Enable QueryResultCache for CollapsingQParserPlugin.
> > (David Boychuck, Joel Bernstein)
> >
> > +* LUCENE-5440: DocSet decoupled from OpenBitSet. DocSetBase moved to use
> > + FixedBitSet instead of OpenBitSet. As a result BitDocSet now only works
> > + with FixedBitSet. (Shai Erera)
> > +
> > Other Changes
> > ---------------------
> >
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java Sun Feb 16 07:50:08 2014
> > @@ -17,7 +17,17 @@
> >
> > package org.apache.solr.handler.component;
> >
> > -import org.apache.lucene.util.OpenBitSet;
> > +import java.io.IOException;
> > +import java.net.URL;
> > +import java.util.ArrayList;
> > +import java.util.Arrays;
> > +import java.util.Comparator;
> > +import java.util.HashMap;
> > +import java.util.LinkedHashMap;
> > +import java.util.List;
> > +import java.util.Map;
> > +
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.SolrException.ErrorCode;
> > import org.apache.solr.common.params.CommonParams;
> > @@ -35,10 +45,6 @@ import org.apache.solr.search.SyntaxErro
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.net.URL;
> > -import java.util.*;
> > -
> > /**
> > * TODO!
> > *
> > @@ -444,8 +450,8 @@ public class FacetComponent extends Sear
> > // and if it is >= smallestCount, then flag for refinement
> > long maxCount = sfc.count;
> > for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
> > - OpenBitSet obs = dff.counted[shardNum];
> > - if (obs!=null && !obs.get(sfc.termNum)) { // obs can be null if a shard request failed
> > + FixedBitSet fbs = dff.counted[shardNum];
> > + if (fbs!=null && !fbs.get(sfc.termNum)) { // fbs can be null if a shard request failed
> > // if missing from this shard, add the max it could be
> > maxCount += dff.maxPossible(sfc,shardNum);
> > }
> > @@ -459,8 +465,8 @@ public class FacetComponent extends Sear
> > if (needRefinement) {
> > // add a query for each shard missing the term that needs refinement
> > for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
> > - OpenBitSet obs = dff.counted[shardNum];
> > - if(obs!=null && !obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
> > + FixedBitSet fbs = dff.counted[shardNum];
> > + if(fbs!=null && !fbs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
> > dff.needRefinements = true;
> > List<String> lst = dff._toRefine[shardNum];
> > if (lst == null) {
> > @@ -759,7 +765,7 @@ public class FacetComponent extends Sear
> > public long missingMaxPossible;
> > // the max possible count for a missing term for each shard (indexed by shardNum)
> > public long[] missingMax;
> > - public OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
> > + public FixedBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
> > public HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
> > public int termNum;
> >
> > @@ -772,7 +778,7 @@ public class FacetComponent extends Sear
> > super(rb, facetStr);
> > // sf = rb.req.getSchema().getField(field);
> > missingMax = new long[rb.shards.length];
> > - counted = new OpenBitSet[rb.shards.length];
> > + counted = new FixedBitSet[rb.shards.length];
> > }
> >
> > void add(int shardNum, NamedList shardCounts, int numRequested) {
> > @@ -780,7 +786,7 @@ public class FacetComponent extends Sear
> > int sz = shardCounts == null ? 0 : shardCounts.size();
> > int numReceived = sz;
> >
> > - OpenBitSet terms = new OpenBitSet(termNum+sz);
> > + FixedBitSet terms = new FixedBitSet(termNum+sz);
> >
> > long last = 0;
> > for (int i=0; i<sz; i++) {
> > @@ -799,7 +805,7 @@ public class FacetComponent extends Sear
> > counts.put(name, sfc);
> > }
> > sfc.count += count;
> > - terms.fastSet(sfc.termNum);
> > + terms.set(sfc.termNum);
> > last = count;
> > }
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Sun Feb 16 07:50:08 2014
> > @@ -17,6 +17,26 @@
> >
> > package org.apache.solr.request;
> >
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.Collection;
> > +import java.util.Date;
> > +import java.util.EnumSet;
> > +import java.util.IdentityHashMap;
> > +import java.util.List;
> > +import java.util.Map;
> > +import java.util.Set;
> > +import java.util.concurrent.Callable;
> > +import java.util.concurrent.ExecutionException;
> > +import java.util.concurrent.Executor;
> > +import java.util.concurrent.Future;
> > +import java.util.concurrent.FutureTask;
> > +import java.util.concurrent.RunnableFuture;
> > +import java.util.concurrent.Semaphore;
> > +import java.util.concurrent.SynchronousQueue;
> > +import java.util.concurrent.ThreadPoolExecutor;
> > +import java.util.concurrent.TimeUnit;
> > +
> > import org.apache.lucene.index.AtomicReader;
> > import org.apache.lucene.index.DocsEnum;
> > import org.apache.lucene.index.Fields;
> > @@ -37,8 +57,6 @@ import org.apache.lucene.search.grouping
> > import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.CharsRef;
> > -import org.apache.lucene.util.FixedBitSet;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.lucene.util.StringHelper;
> > import org.apache.lucene.util.UnicodeUtil;
> > import org.apache.solr.common.SolrException;
> > @@ -80,26 +98,6 @@ import org.apache.solr.util.DateMathPars
> > import org.apache.solr.util.DefaultSolrThreadFactory;
> > import org.apache.solr.util.LongPriorityQueue;
> >
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.Collection;
> > -import java.util.Date;
> > -import java.util.EnumSet;
> > -import java.util.IdentityHashMap;
> > -import java.util.List;
> > -import java.util.Map;
> > -import java.util.Set;
> > -import java.util.concurrent.Callable;
> > -import java.util.concurrent.ExecutionException;
> > -import java.util.concurrent.Executor;
> > -import java.util.concurrent.Future;
> > -import java.util.concurrent.FutureTask;
> > -import java.util.concurrent.RunnableFuture;
> > -import java.util.concurrent.Semaphore;
> > -import java.util.concurrent.SynchronousQueue;
> > -import java.util.concurrent.ThreadPoolExecutor;
> > -import java.util.concurrent.TimeUnit;
> > -
> > /**
> > * A class that generates simple Facet information for a request.
> > *
> > @@ -230,10 +228,7 @@ public class SimpleFacets {
> > }
> > AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0).createAllGroupCollector();
> > searcher.search(new MatchAllDocsQuery(), base.getTopFilter(), allGroupHeadsCollector);
> > - int maxDoc = searcher.maxDoc();
> > - FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
> > - long[] bits = fixedBitSet.getBits();
> > - this.docs = new BitDocSet(new OpenBitSet(bits, bits.length));
> > + this.docs = new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
> > } else {
> > this.docs = base;
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Sun Feb 16 07:50:08 2014
> > @@ -27,12 +27,11 @@ import org.apache.lucene.index.DocTermOr
> > import org.apache.lucene.index.SortedDocValues;
> > import org.apache.lucene.index.Term;
> > import org.apache.lucene.index.TermsEnum;
> > -import org.apache.lucene.search.FieldCache;
> > import org.apache.lucene.search.TermQuery;
> > import org.apache.lucene.search.TermRangeQuery;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.CharsRef;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.lucene.util.UnicodeUtil;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.params.FacetParams;
> > @@ -44,7 +43,11 @@ import org.apache.solr.handler.component
> > import org.apache.solr.schema.FieldType;
> > import org.apache.solr.schema.SchemaField;
> > import org.apache.solr.schema.TrieField;
> > -import org.apache.solr.search.*;
> > +import org.apache.solr.search.BitDocSet;
> > +import org.apache.solr.search.DocIterator;
> > +import org.apache.solr.search.DocSet;
> > +import org.apache.solr.search.SolrCache;
> > +import org.apache.solr.search.SolrIndexSearcher;
> > import org.apache.solr.util.LongPriorityQueue;
> > import org.apache.solr.util.PrimUtils;
> >
> > @@ -269,7 +272,7 @@ public class UnInvertedField extends Doc
> > && docs instanceof BitDocSet;
> >
> > if (doNegative) {
> > - OpenBitSet bs = (OpenBitSet)((BitDocSet)docs).getBits().clone();
> > + FixedBitSet bs = ((BitDocSet)docs).getBits().clone();
> > bs.flip(0, maxDoc);
> > // TODO: when iterator across negative elements is available, use that
> > // instead of creating a new bitset and inverting.
> > @@ -512,7 +515,7 @@ public class UnInvertedField extends Doc
> > }
> >
> > if (doNegative) {
> > - OpenBitSet bs = (OpenBitSet) ((BitDocSet) docs).getBits().clone();
> > + FixedBitSet bs = ((BitDocSet) docs).getBits().clone();
> > bs.flip(0, maxDoc);
> > // TODO: when iterator across negative elements is available, use that
> > // instead of creating a new bitset and inverting.
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java Sun Feb 16 07:50:08 2014
> > @@ -21,38 +21,40 @@ import org.apache.lucene.index.AtomicRea
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.BitsFilteredDocIdSet;
> > import org.apache.lucene.search.DocIdSet;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.search.Filter;
> > import org.apache.lucene.util.Bits;
> > -import org.apache.lucene.util.OpenBitSet;
> > -import org.apache.lucene.util.OpenBitSetIterator;
> > -import org.apache.lucene.search.DocIdSetIterator;
> > +import org.apache.lucene.util.FixedBitSet;
> > +import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
> >
> > /**
> > * <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
> > * using a BitSet. A set bit represents inclusion in the set for that document.
> > *
> > - *
> > * @since solr 0.9
> > */
> > public class BitDocSet extends DocSetBase {
> > - final OpenBitSet bits;
> > + final FixedBitSet bits;
> > int size; // number of docs in the set (cached for perf)
> >
> > public BitDocSet() {
> > - bits = new OpenBitSet();
> > + bits = new FixedBitSet(64);
> > }
> >
> > - /** Construct a BitDocSet.
> > - * The capacity of the OpenBitSet should be at least maxDoc() */
> > - public BitDocSet(OpenBitSet bits) {
> > + /**
> > + * Construct a BitDocSet. The capacity of the {@link FixedBitSet} should be at
> > + * least maxDoc()
> > + */
> > + public BitDocSet(FixedBitSet bits) {
> > this.bits = bits;
> > size=-1;
> > }
> >
> > - /** Construct a BitDocSet, and provides the number of set bits.
> > - * The capacity of the OpenBitSet should be at least maxDoc()
> > + /**
> > + * Construct a BitDocSet, and provides the number of set bits. The capacity of
> > + * the {@link FixedBitSet} should be at least maxDoc()
> > */
> > - public BitDocSet(OpenBitSet bits, int size) {
> > + public BitDocSet(FixedBitSet bits, int size) {
> > this.bits = bits;
> > this.size = size;
> > }
> > @@ -89,7 +91,7 @@ public class BitDocSet extends DocSetBas
> > @Override
> > public DocIterator iterator() {
> > return new DocIterator() {
> > - private final OpenBitSetIterator iter = new OpenBitSetIterator(bits);
> > + private final FixedBitSetIterator iter = new FixedBitSetIterator(bits);
> > private int pos = iter.nextDoc();
> > @Override
> > public boolean hasNext() {
> > @@ -120,13 +122,11 @@ public class BitDocSet extends DocSetBas
> > };
> > }
> >
> > -
> > /**
> > - *
> > - * @return the <b>internal</b> OpenBitSet that should <b>not</b> be modified.
> > + * @return the <b>internal</b> {@link FixedBitSet} that should <b>not</b> be modified.
> > */
> > @Override
> > - public OpenBitSet getBits() {
> > + public FixedBitSet getBits() {
> > return bits;
> > }
> >
> > @@ -145,7 +145,7 @@ public class BitDocSet extends DocSetBas
> > @Override
> > public int size() {
> > if (size!=-1) return size;
> > - return size=(int)bits.cardinality();
> > + return size = bits.cardinality();
> > }
> >
> > /**
> > @@ -156,18 +156,19 @@ public class BitDocSet extends DocSetBas
> > size=-1;
> > }
> >
> > - /** Returns true of the doc exists in the set.
> > - * Should only be called when doc < OpenBitSet.size()
> > + /**
> > + * Returns true of the doc exists in the set. Should only be called when doc <
> > + * {@link FixedBitSet#length()}.
> > */
> > @Override
> > public boolean exists(int doc) {
> > - return bits.fastGet(doc);
> > + return bits.get(doc);
> > }
> >
> > @Override
> > public int intersectionSize(DocSet other) {
> > if (other instanceof BitDocSet) {
> > - return (int)OpenBitSet.intersectionCount(this.bits, ((BitDocSet)other).bits);
> > + return (int) FixedBitSet.intersectionCount(this.bits, ((BitDocSet) other).bits);
> > } else {
> > // they had better not call us back!
> > return other.intersectionSize(this);
> > @@ -189,7 +190,7 @@ public class BitDocSet extends DocSetBas
> > if (other instanceof BitDocSet) {
> > // if we don't know our current size, this is faster than
> > // size + other.size - intersection_size
> > - return (int)OpenBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
> > + return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
> > } else {
> > // they had better not call us back!
> > return other.unionSize(this);
> > @@ -201,42 +202,56 @@ public class BitDocSet extends DocSetBas
> > if (other instanceof BitDocSet) {
> > // if we don't know our current size, this is faster than
> > // size - intersection_size
> > - return (int)OpenBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
> > + return (int) FixedBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
> > } else {
> > return super.andNotSize(other);
> > }
> > }
> >
> > @Override
> > - public void setBitsOn(OpenBitSet target) {
> > - target.union(bits);
> > + public void addAllTo(DocSet target) {
> > + if (target instanceof BitDocSet) {
> > + ((BitDocSet) target).bits.or(bits);
> > + } else {
> > + super.addAllTo(target);
> > + }
> > }
> >
> > @Override
> > - public DocSet andNot(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(bits.clone());
> > - if (other instanceof BitDocSet) {
> > - newbits.andNot(((BitDocSet)other).bits);
> > - } else {
> > - DocIterator iter = other.iterator();
> > - while (iter.hasNext()) newbits.clear(iter.nextDoc());
> > - }
> > - return new BitDocSet(newbits);
> > + public DocSet andNot(DocSet other) {
> > + FixedBitSet newbits = bits.clone();
> > + if (other instanceof BitDocSet) {
> > + newbits.andNot(((BitDocSet) other).bits);
> > + } else {
> > + DocIterator iter = other.iterator();
> > + while (iter.hasNext()) {
> > + int doc = iter.nextDoc();
> > + if (doc < newbits.length()) {
> > + newbits.clear(doc);
> > + }
> > + }
> > + }
> > + return new BitDocSet(newbits);
> > }
> > -
> > +
> > @Override
> > - public DocSet union(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(bits.clone());
> > - if (other instanceof BitDocSet) {
> > - newbits.union(((BitDocSet)other).bits);
> > - } else {
> > - DocIterator iter = other.iterator();
> > - while (iter.hasNext()) newbits.set(iter.nextDoc());
> > - }
> > - return new BitDocSet(newbits);
> > + public DocSet union(DocSet other) {
> > + FixedBitSet newbits = bits.clone();
> > + if (other instanceof BitDocSet) {
> > + BitDocSet otherDocSet = (BitDocSet) other;
> > + newbits = FixedBitSet.ensureCapacity(newbits, otherDocSet.bits.length());
> > + newbits.or(otherDocSet.bits);
> > + } else {
> > + DocIterator iter = other.iterator();
> > + while (iter.hasNext()) {
> > + int doc = iter.nextDoc();
> > + newbits = FixedBitSet.ensureCapacity(newbits, doc);
> > + newbits.set(doc);
> > + }
> > + }
> > + return new BitDocSet(newbits);
> > }
> > -
> > -
> > +
> > @Override
> > public long memSize() {
> > return (bits.getBits().length << 3) + 16;
> > @@ -244,12 +259,12 @@ public class BitDocSet extends DocSetBas
> >
> > @Override
> > protected BitDocSet clone() {
> > - return new BitDocSet((OpenBitSet)bits.clone(), size);
> > + return new BitDocSet(bits.clone(), size);
> > }
> >
> > @Override
> > public Filter getTopFilter() {
> > - final OpenBitSet bs = bits;
> > + final FixedBitSet bs = bits;
> > // TODO: if cardinality isn't cached, do a quick measure of sparseness
> > // and return null from bits() if too sparse.
> >
> > @@ -272,8 +287,8 @@ public class BitDocSet extends DocSetBas
> > @Override
> > public DocIdSetIterator iterator() {
> > return new DocIdSetIterator() {
> > - int pos=base-1;
> > - int adjustedDoc=-1;
> > + int pos = base - 1;
> > + int adjustedDoc = -1;
> >
> > @Override
> > public int docID() {
> > @@ -282,15 +297,24 @@ public class BitDocSet extends DocSetBas
> >
> > @Override
> > public int nextDoc() {
> > - pos = bs.nextSetBit(pos+1);
> > - return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
> > + if (pos >= bs.length() - 1) {
> > + return adjustedDoc = NO_MORE_DOCS;
> > + } else {
> > + pos = bs.nextSetBit(pos + 1);
> > + return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
> > + }
> > }
> >
> > @Override
> > public int advance(int target) {
> > - if (target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;
> > - pos = bs.nextSetBit(target+base);
> > - return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
> > + if (target == NO_MORE_DOCS) return adjustedDoc = NO_MORE_DOCS;
> > + int adjusted = target + base;
> > + if (adjusted >= bs.length()) {
> > + return adjustedDoc = NO_MORE_DOCS;
> > + } else {
> > + pos = bs.nextSetBit(adjusted);
> > + return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
> > + }
> > }
> >
> > @Override
> > @@ -298,7 +322,7 @@ public class BitDocSet extends DocSetBas
> > // we don't want to actually compute cardinality, but
> > // if its already been computed, we use it (pro-rated for the segment)
> > if (size != -1) {
> > - return (long)(size * ((OpenBitSet.bits2words(maxDoc)<<6) / (float)bs.capacity()));
> > + return (long)(size * ((FixedBitSet.bits2words(maxDoc)<<6) / (float)bs.length()));
> > } else {
> > return maxDoc;
> > }
> > @@ -316,7 +340,7 @@ public class BitDocSet extends DocSetBas
> > return new Bits() {
> > @Override
> > public boolean get(int index) {
> > - return bs.fastGet(index + base);
> > + return bs.get(index + base);
> > }
> >
> > @Override
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java Sun Feb 16 07:50:08 2014
> > @@ -17,45 +17,49 @@
> >
> > package org.apache.solr.search;
> >
> > +import java.io.IOException;
> > +import java.util.Arrays;
> > +import java.util.HashSet;
> > +import java.util.Iterator;
> > +import java.util.List;
> > +import java.util.Map;
> > +import java.util.Set;
> > +
> > +import org.apache.lucene.index.AtomicReader;
> > +import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.index.DocsEnum;
> > +import org.apache.lucene.index.SortedDocValues;
> > +import org.apache.lucene.index.Terms;
> > +import org.apache.lucene.index.TermsEnum;
> > import org.apache.lucene.queries.function.FunctionQuery;
> > import org.apache.lucene.queries.function.FunctionValues;
> > import org.apache.lucene.queries.function.ValueSource;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > +import org.apache.lucene.search.FieldCache;
> > +import org.apache.lucene.search.IndexSearcher;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.Scorer;
> > +import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.params.ModifiableSolrParams;
> > +import org.apache.solr.common.params.SolrParams;
> > +import org.apache.solr.common.util.NamedList;
> > +import org.apache.solr.handler.component.QueryElevationComponent;
> > import org.apache.solr.request.LocalSolrQueryRequest;
> > +import org.apache.solr.request.SolrQueryRequest;
> > import org.apache.solr.request.SolrRequestInfo;
> > -import org.apache.solr.schema.TrieFloatField;
> > -import org.apache.solr.schema.TrieIntField;
> > -import org.apache.solr.schema.TrieLongField;
> > import org.apache.solr.schema.FieldType;
> > -import org.apache.solr.handler.component.QueryElevationComponent;
> > -import org.apache.lucene.index.AtomicReaderContext;
> > -import org.apache.lucene.index.AtomicReader;
> > -import org.apache.lucene.index.SortedDocValues;
> > -import org.apache.lucene.index.Terms;
> > -import org.apache.lucene.index.TermsEnum;
> > -import org.apache.lucene.index.DocsEnum;
> > -import org.apache.lucene.search.*;
> > -import org.apache.lucene.util.OpenBitSet;
> > -import org.apache.lucene.util.Bits;
> > -import org.apache.solr.common.params.SolrParams;
> > -import org.apache.solr.request.SolrQueryRequest;
> > -import org.apache.solr.common.util.NamedList;
> > import org.apache.solr.schema.IndexSchema;
> > import org.apache.solr.schema.SchemaField;
> > +import org.apache.solr.schema.TrieFloatField;
> > +import org.apache.solr.schema.TrieIntField;
> > +import org.apache.solr.schema.TrieLongField;
> >
> > import com.carrotsearch.hppc.FloatArrayList;
> > import com.carrotsearch.hppc.IntOpenHashSet;
> > import com.carrotsearch.hppc.cursors.IntCursor;
> >
> > -import java.io.IOException;
> > -import java.util.Arrays;
> > -import java.util.Map;
> > -import java.util.Set;
> > -import java.util.HashSet;
> > -import java.util.List;
> > -import java.util.Iterator;
> > -
> > /**
> >
> > The <b>CollapsingQParserPlugin</b> is a PostFilter that performs field collapsing.
> > @@ -419,7 +423,7 @@ public class CollapsingQParserPlugin ext
> > private class CollapsingScoreCollector extends DelegatingCollector {
> >
> > private AtomicReaderContext[] contexts;
> > - private OpenBitSet collapsedSet;
> > + private FixedBitSet collapsedSet;
> > private SortedDocValues values;
> > private int[] ords;
> > private float[] scores;
> > @@ -438,14 +442,14 @@ public class CollapsingQParserPlugin ext
> > IntOpenHashSet boostDocs) {
> > this.maxDoc = maxDoc;
> > this.contexts = new AtomicReaderContext[segments];
> > - this.collapsedSet = new OpenBitSet(maxDoc);
> > + this.collapsedSet = new FixedBitSet(maxDoc);
> > this.boostDocs = boostDocs;
> > if(this.boostDocs != null) {
> > //Set the elevated docs now.
> > Iterator<IntCursor> it = this.boostDocs.iterator();
> > while(it.hasNext()) {
> > IntCursor cursor = it.next();
> > - this.collapsedSet.fastSet(cursor.value);
> > + this.collapsedSet.set(cursor.value);
> > }
> > }
> > this.values = values;
> > @@ -460,16 +464,19 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> >
> > + @Override
> > public boolean acceptsDocsOutOfOrder() {
> > //Documents must be sent in order to this collector.
> > return false;
> > }
> >
> > + @Override
> > public void setNextReader(AtomicReaderContext context) throws IOException {
> > this.contexts[context.ord] = context;
> > this.docBase = context.docBase;
> > }
> >
> > + @Override
> > public void collect(int docId) throws IOException {
> > int globalDoc = docId+this.docBase;
> > int ord = values.getOrd(globalDoc);
> > @@ -479,7 +486,7 @@ public class CollapsingQParserPlugin ext
> > ords[ord] = globalDoc;
> > scores[ord] = score;
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //The doc is elevated so score does not matter
> > //We just want to be sure it doesn't fall into the null policy
> > } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > @@ -489,24 +496,25 @@ public class CollapsingQParserPlugin ext
> > nullDoc = globalDoc;
> > }
> > } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - collapsedSet.fastSet(globalDoc);
> > + collapsedSet.set(globalDoc);
> > nullScores.add(scorer.score());
> > }
> > }
> >
> > + @Override
> > public void finish() throws IOException {
> > if(contexts.length == 0) {
> > return;
> > }
> >
> > if(nullScore > 0) {
> > - this.collapsedSet.fastSet(nullDoc);
> > + this.collapsedSet.set(nullDoc);
> > }
> >
> > for(int i=0; i<ords.length; i++) {
> > int doc = ords[i];
> > if(doc > -1) {
> > - collapsedSet.fastSet(doc);
> > + collapsedSet.set(doc);
> > }
> > }
> >
> > @@ -677,7 +685,7 @@ public class CollapsingQParserPlugin ext
> > protected FloatArrayList nullScores;
> > protected float nullScore;
> > protected float[] scores;
> > - protected OpenBitSet collapsedSet;
> > + protected FixedBitSet collapsedSet;
> > protected IntOpenHashSet boostDocs;
> > protected int nullDoc = -1;
> > protected boolean needsScores;
> > @@ -697,26 +705,26 @@ public class CollapsingQParserPlugin ext
> > this.nullPolicy = nullPolicy;
> > this.max = max;
> > this.needsScores = needsScores;
> > - this.collapsedSet = new OpenBitSet(maxDoc);
> > + this.collapsedSet = new FixedBitSet(maxDoc);
> > this.boostDocs = boostDocs;
> > if(this.boostDocs != null) {
> > Iterator<IntCursor> it = boostDocs.iterator();
> > while(it.hasNext()) {
> > IntCursor cursor = it.next();
> > - this.collapsedSet.fastSet(cursor.value);
> > + this.collapsedSet.set(cursor.value);
> > }
> > }
> > }
> >
> > - public OpenBitSet getCollapsedSet() {
> > + public FixedBitSet getCollapsedSet() {
> > if(nullDoc > -1) {
> > - this.collapsedSet.fastSet(nullDoc);
> > + this.collapsedSet.set(nullDoc);
> > }
> >
> > for(int i=0; i<ords.length; i++) {
> > int doc = ords[i];
> > if(doc > -1) {
> > - collapsedSet.fastSet(doc);
> > + collapsedSet.set(doc);
> > }
> > }
> >
> > @@ -790,7 +798,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = scorer.score();
> > }
> > }
> > - } else if(this.collapsedSet.fastGet(globalDoc)) {
> > + } else if(this.collapsedSet.get(globalDoc)) {
> > // Elevated doc so do nothing.
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -801,7 +809,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(scorer.score());
> > }
> > @@ -858,7 +866,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = scorer.score();
> > }
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //Elevated doc so do nothing
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -869,7 +877,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(scorer.score());
> > }
> > @@ -927,7 +935,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = scorer.score();
> > }
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //Elevated doc so do nothing
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -938,7 +946,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(scorer.score());
> > }
> > @@ -1015,7 +1023,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = score;
> > }
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //Elevated doc so do nothing
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -1026,7 +1034,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(score);
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java Sun Feb 16 07:50:08 2014
> > @@ -18,7 +18,6 @@
> > package org.apache.solr.search;
> >
> > import org.apache.lucene.search.Filter;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.solr.common.SolrException;
> >
> > /**
> > @@ -29,7 +28,6 @@ import org.apache.solr.common.SolrExcept
> > * a cache and could be shared.
> > * </p>
> > *
> > - *
> > * @since solr 0.9
> > */
> > public interface DocSet /* extends Collection<Integer> */ {
> > @@ -78,16 +76,6 @@ public interface DocSet /* extends Colle
> > public DocIterator iterator();
> >
> > /**
> > - * Returns a BitSet view of the DocSet. Any changes to this BitSet <b>may</b>
> > - * be reflected in the DocSet, hence if the DocSet is shared or was returned from
> > - * a SolrIndexSearcher method, it's not safe to modify the BitSet.
> > - *
> > - * @return
> > - * An OpenBitSet with the bit number of every docid set in the set.
> > - */
> > - public OpenBitSet getBits();
> > -
> > - /**
> > * Returns the approximate amount of memory taken by this DocSet.
> > * This is only an approximation and doesn't take into account java object overhead.
> > *
> > @@ -145,10 +133,11 @@ public interface DocSet /* extends Colle
> > public Filter getTopFilter();
> >
> > /**
> > - * Takes the docs from this set and sets those bits on the target OpenBitSet.
> > - * The target should be sized large enough to accommodate all of the documents before calling this method.
> > + * Adds all the docs from this set to the target set. The target should be
> > + * sized large enough to accommodate all of the documents before calling this
> > + * method.
> > */
> > - public void setBitsOn(OpenBitSet target);
> > + public void addAllTo(DocSet target);
> >
> > public static DocSet EMPTY = new SortedIntDocSet(new int[0], 0);
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java Sun Feb 16 07:50:08 2014
> > @@ -18,18 +18,32 @@
> > package org.apache.solr.search;
> >
> > import org.apache.lucene.index.AtomicReader;
> > -import org.apache.solr.common.SolrException;
> > -import org.apache.lucene.util.Bits;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.search.BitsFilteredDocIdSet;
> > import org.apache.lucene.search.DocIdSet;
> > -import org.apache.lucene.search.Filter;
> > import org.apache.lucene.search.DocIdSetIterator;
> > -import org.apache.lucene.search.BitsFilteredDocIdSet;
> > -import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.util.Bits;
> > +import org.apache.lucene.util.FixedBitSet;
> > +import org.apache.solr.common.SolrException;
> >
> > /** A base class that may be usefull for implementing DocSets */
> > abstract class DocSetBase implements DocSet {
> >
> > + public static FixedBitSet toBitSet(DocSet set) {
> > + if (set instanceof DocSetBase) {
> > + return ((DocSetBase) set).getBits();
> > + } else {
> > + FixedBitSet bits = new FixedBitSet(64);
> > + for (DocIterator iter = set.iterator(); iter.hasNext();) {
> > + int nextDoc = iter.nextDoc();
> > + bits = FixedBitSet.ensureCapacity(bits, nextDoc);
> > + bits.set(nextDoc);
> > + }
> > + return bits;
> > + }
> > + }
> > +
> > // Not implemented efficiently... for testing purposes only
> > @Override
> > public boolean equals(Object obj) {
> > @@ -49,7 +63,7 @@ abstract class DocSetBase implements Doc
> > }
> >
> > // if (this.size() != other.size()) return false;
> > - return this.getBits().equals(other.getBits());
> > + return this.getBits().equals(toBitSet(other));
> > }
> >
> > /**
> > @@ -69,18 +83,20 @@ abstract class DocSetBase implements Doc
> > }
> >
> > /**
> > - * Inefficient base implementation.
> > - *
> > - * @see BitDocSet#getBits
> > + * Return a {@link FixedBitSet} with a bit set for every document in this
> > + * {@link DocSet}. The default implementation iterates on all docs and sets
> > + * the relevant bits. You should override if you can provide a more efficient
> > + * implementation.
> > */
> > - @Override
> > - public OpenBitSet getBits() {
> > - OpenBitSet bits = new OpenBitSet();
> > + protected FixedBitSet getBits() {
> > + FixedBitSet bits = new FixedBitSet(64);
> > for (DocIterator iter = iterator(); iter.hasNext();) {
> > - bits.set(iter.nextDoc());
> > + int nextDoc = iter.nextDoc();
> > + bits = FixedBitSet.ensureCapacity(bits, nextDoc);
> > + bits.set(nextDoc);
> > }
> > return bits;
> > - };
> > + }
> >
> > @Override
> > public DocSet intersection(DocSet other) {
> > @@ -91,8 +107,8 @@ abstract class DocSetBase implements Doc
> > }
> >
> > // Default... handle with bitsets.
> > - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> > - newbits.and(other.getBits());
> > + FixedBitSet newbits = getBits().clone();
> > + newbits.and(toBitSet(other));
> > return new BitDocSet(newbits);
> > }
> >
> > @@ -107,11 +123,11 @@ abstract class DocSetBase implements Doc
> > return intersectionSize(other) > 0;
> > }
> >
> > -
> > @Override
> > public DocSet union(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> > - newbits.or(other.getBits());
> > + FixedBitSet otherBits = toBitSet(other);
> > + FixedBitSet newbits = FixedBitSet.ensureCapacity(getBits().clone(), otherBits.length());
> > + newbits.or(otherBits);
> > return new BitDocSet(newbits);
> > }
> >
> > @@ -133,8 +149,8 @@ abstract class DocSetBase implements Doc
> >
> > @Override
> > public DocSet andNot(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> > - newbits.andNot(other.getBits());
> > + FixedBitSet newbits = getBits().clone();
> > + newbits.andNot(toBitSet(other));
> > return new BitDocSet(newbits);
> > }
> >
> > @@ -145,7 +161,7 @@ abstract class DocSetBase implements Doc
> >
> > @Override
> > public Filter getTopFilter() {
> > - final OpenBitSet bs = getBits();
> > + final FixedBitSet bs = getBits();
> >
> > return new Filter() {
> > @Override
> > @@ -189,7 +205,7 @@ abstract class DocSetBase implements Doc
> >
> > @Override
> > public long cost() {
> > - return bs.capacity();
> > + return bs.length();
> > }
> > };
> > }
> > @@ -211,10 +227,10 @@ abstract class DocSetBase implements Doc
> > }
> >
> > @Override
> > - public void setBitsOn(OpenBitSet target) {
> > + public void addAllTo(DocSet target) {
> > DocIterator iter = iterator();
> > while (iter.hasNext()) {
> > - target.fastSet(iter.nextDoc());
> > + target.add(iter.nextDoc());
> > }
> > }
> >
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,12 @@ package org.apache.solr.search;
> > * limitations under the License.
> > */
> >
> > -import org.apache.lucene.index.IndexReader;
> > +import java.io.IOException;
> > +
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.Collector;
> > import org.apache.lucene.search.Scorer;
> > -import org.apache.lucene.util.OpenBitSet;
> > -
> > -import java.io.IOException;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > *
> > @@ -31,7 +30,7 @@ import java.io.IOException;
> >
> > public class DocSetCollector extends Collector {
> > int pos=0;
> > - OpenBitSet bits;
> > + FixedBitSet bits;
> > final int maxDoc;
> > final int smallSetSize;
> > int base;
> > @@ -62,8 +61,8 @@ public class DocSetCollector extends Col
> > } else {
> > // this conditional could be removed if BitSet was preallocated, but that
> > // would take up more memory, and add more GC time...
> > - if (bits==null) bits = new OpenBitSet(maxDoc);
> > - bits.fastSet(doc);
> > + if (bits==null) bits = new FixedBitSet(maxDoc);
> > + bits.set(doc);
> > }
> >
> > pos++;
> > @@ -75,7 +74,7 @@ public class DocSetCollector extends Col
> > return new SortedIntDocSet(scratch, pos);
> > } else {
> > // set the bits for ids that were collected in the array
> > - for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
> > + for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
> > return new BitDocSet(bits,pos);
> > }
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,12 @@ package org.apache.solr.search;
> > * limitations under the License.
> > */
> >
> > -import org.apache.lucene.index.IndexReader;
> > +import java.io.IOException;
> > +
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.Collector;
> > import org.apache.lucene.search.Scorer;
> > -import org.apache.lucene.util.OpenBitSet;
> > -
> > -import java.io.IOException;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > *
> > @@ -53,8 +52,8 @@ public class DocSetDelegateCollector ext
> > } else {
> > // this conditional could be removed if BitSet was preallocated, but that
> > // would take up more memory, and add more GC time...
> > - if (bits==null) bits = new OpenBitSet(maxDoc);
> > - bits.fastSet(doc);
> > + if (bits==null) bits = new FixedBitSet(maxDoc);
> > + bits.set(doc);
> > }
> >
> > pos++;
> > @@ -67,7 +66,7 @@ public class DocSetDelegateCollector ext
> > return new SortedIntDocSet(scratch, pos);
> > } else {
> > // set the bits for ids that were collected in the array
> > - for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
> > + for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
> > return new BitDocSet(bits,pos);
> > }
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,38 @@
> >
> > package org.apache.solr.search;
> >
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.Collection;
> > +import java.util.LinkedHashSet;
> > +import java.util.List;
> > +import java.util.Locale;
> > +import java.util.Map;
> > +import java.util.Set;
> > +
> > import org.apache.commons.lang.ArrayUtils;
> > import org.apache.lucene.index.IndexableField;
> > import org.apache.lucene.queries.function.FunctionQuery;
> > import org.apache.lucene.queries.function.ValueSource;
> > import org.apache.lucene.queries.function.valuesource.QueryValueSource;
> > -import org.apache.lucene.search.*;
> > -import org.apache.lucene.search.grouping.*;
> > +import org.apache.lucene.search.CachingCollector;
> > +import org.apache.lucene.search.Collector;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.search.MultiCollector;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.ScoreDoc;
> > +import org.apache.lucene.search.Sort;
> > +import org.apache.lucene.search.SortField;
> > +import org.apache.lucene.search.TimeLimitingCollector;
> > +import org.apache.lucene.search.TopDocs;
> > +import org.apache.lucene.search.TopDocsCollector;
> > +import org.apache.lucene.search.TopFieldCollector;
> > +import org.apache.lucene.search.TopScoreDocCollector;
> > +import org.apache.lucene.search.TotalHitCountCollector;
> > +import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
> > +import org.apache.lucene.search.grouping.GroupDocs;
> > +import org.apache.lucene.search.grouping.SearchGroup;
> > +import org.apache.lucene.search.grouping.TopGroups;
> > import org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
> > import org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector;
> > import org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector;
> > @@ -33,8 +58,6 @@ import org.apache.lucene.search.grouping
> > import org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector;
> > import org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector;
> > import org.apache.lucene.util.BytesRef;
> > -import org.apache.lucene.util.FixedBitSet;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.lucene.util.mutable.MutableValue;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.util.NamedList;
> > @@ -47,9 +70,6 @@ import org.apache.solr.search.grouping.c
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.util.*;
> > -
> > /**
> > * Basic Solr Grouping infrastructure.
> > * Warning NOT thread save!
> > @@ -353,10 +373,7 @@ public class Grouping {
> > }
> >
> > if (getGroupedDocSet && allGroupHeadsCollector != null) {
> > - FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
> > - long[] bits = fixedBitSet.getBits();
> > - OpenBitSet openBitSet = new OpenBitSet(bits, bits.length);
> > - qr.setDocSet(new BitDocSet(openBitSet));
> > + qr.setDocSet(new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(maxDoc)));
> > } else if (getDocSet) {
> > qr.setDocSet(setCollector.getDocSet());
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java Sun Feb 16 07:50:08 2014
> > @@ -16,12 +16,33 @@
> > */
> > package org.apache.solr.search;
> >
> > -import org.apache.lucene.index.*;
> > -import org.apache.lucene.search.*;
> > +import java.io.Closeable;
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.Arrays;
> > +import java.util.List;
> > +import java.util.Set;
> > +
> > +import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.index.DocsEnum;
> > +import org.apache.lucene.index.Fields;
> > +import org.apache.lucene.index.IndexReader;
> > +import org.apache.lucene.index.MultiDocsEnum;
> > +import org.apache.lucene.index.Terms;
> > +import org.apache.lucene.index.TermsEnum;
> > +import org.apache.lucene.search.ComplexExplanation;
> > +import org.apache.lucene.search.DocIdSet;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > +import org.apache.lucene.search.Explanation;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.search.IndexSearcher;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.Scorer;
> > +import org.apache.lucene.search.Weight;
> > import org.apache.lucene.search.similarities.Similarity;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.lucene.util.StringHelper;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.params.SolrParams;
> > @@ -36,13 +57,6 @@ import org.apache.solr.request.SolrReque
> > import org.apache.solr.schema.TrieField;
> > import org.apache.solr.util.RefCounted;
> >
> > -import java.io.Closeable;
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.Arrays;
> > -import java.util.List;
> > -import java.util.Set;
> > -
> >
> > public class JoinQParserPlugin extends QParserPlugin {
> > public static final String NAME = "join";
> > @@ -270,7 +284,7 @@ class JoinQuery extends Query {
> >
> >
> > public DocSet getDocSet() throws IOException {
> > - OpenBitSet resultBits = null;
> > + FixedBitSet resultBits = null;
> >
> > // minimum docFreq to use the cache
> > int minDocFreqFrom = Math.max(5, fromSearcher.maxDoc() >> 13);
> > @@ -387,7 +401,7 @@ class JoinQuery extends Query {
> > int df = toTermsEnum.docFreq();
> > toTermHitsTotalDf += df;
> > if (resultBits==null && df + resultListDocs > maxSortedIntSize && resultList.size() > 0) {
> > - resultBits = new OpenBitSet(toSearcher.maxDoc());
> > + resultBits = new FixedBitSet(toSearcher.maxDoc());
> > }
> >
> > // if we don't have a bitset yet, or if the resulting set will be too large
> > @@ -397,10 +411,10 @@ class JoinQuery extends Query {
> > DocSet toTermSet = toSearcher.getDocSet(toDeState);
> > resultListDocs += toTermSet.size();
> > if (resultBits != null) {
> > - toTermSet.setBitsOn(resultBits);
> > + toTermSet.addAllTo(new BitDocSet(resultBits));
> > } else {
> > if (toTermSet instanceof BitDocSet) {
> > - resultBits = (OpenBitSet)((BitDocSet)toTermSet).bits.clone();
> > + resultBits = ((BitDocSet)toTermSet).bits.clone();
> > } else {
> > resultList.add(toTermSet);
> > }
> > @@ -422,14 +436,14 @@ class JoinQuery extends Query {
> > int docid;
> > while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> > resultListDocs++;
> > - resultBits.fastSet(docid + base);
> > + resultBits.set(docid + base);
> > }
> > }
> > } else {
> > int docid;
> > while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> > resultListDocs++;
> > - resultBits.fastSet(docid);
> > + resultBits.set(docid);
> > }
> > }
> > }
> > @@ -443,10 +457,11 @@ class JoinQuery extends Query {
> > smallSetsDeferred = resultList.size();
> >
> > if (resultBits != null) {
> > + BitDocSet bitSet = new BitDocSet(resultBits);
> > for (DocSet set : resultList) {
> > - set.setBitsOn(resultBits);
> > + set.addAllTo(bitSet);
> > }
> > - return new BitDocSet(resultBits);
> > + return bitSet;
> > }
> >
> > if (resultList.size()==0) {
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sun Feb 16 07:50:08 2014
> > @@ -65,11 +65,11 @@ import org.apache.lucene.search.Constant
> > import org.apache.lucene.search.DocIdSet;
> > import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.search.Explanation;
> > +import org.apache.lucene.search.FieldDoc;
> > import org.apache.lucene.search.Filter;
> > import org.apache.lucene.search.IndexSearcher;
> > import org.apache.lucene.search.MatchAllDocsQuery;
> > import org.apache.lucene.search.Query;
> > -import org.apache.lucene.search.FieldDoc;
> > import org.apache.lucene.search.ScoreDoc;
> > import org.apache.lucene.search.Scorer;
> > import org.apache.lucene.search.Sort;
> > @@ -77,16 +77,16 @@ import org.apache.lucene.search.SortFiel
> > import org.apache.lucene.search.TermQuery;
> > import org.apache.lucene.search.TimeLimitingCollector;
> > import org.apache.lucene.search.TopDocs;
> > -import org.apache.lucene.search.TopFieldDocs;
> > import org.apache.lucene.search.TopDocsCollector;
> > import org.apache.lucene.search.TopFieldCollector;
> > +import org.apache.lucene.search.TopFieldDocs;
> > import org.apache.lucene.search.TopScoreDocCollector;
> > import org.apache.lucene.search.TotalHitCountCollector;
> > import org.apache.lucene.search.Weight;
> > import org.apache.lucene.store.Directory;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.SolrException.ErrorCode;
> > import org.apache.solr.common.params.ModifiableSolrParams;
> > @@ -104,7 +104,6 @@ import org.apache.solr.request.UnInverte
> > import org.apache.solr.response.SolrQueryResponse;
> > import org.apache.solr.schema.IndexSchema;
> > import org.apache.solr.schema.SchemaField;
> > -import org.apache.solr.spelling.QueryConverter;
> > import org.apache.solr.update.SolrIndexConfig;
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> > @@ -1087,7 +1086,7 @@ public class SolrIndexSearcher extends I
> > final int[] docs = deState.scratch;
> > int upto = 0;
> > int bitsSet = 0;
> > - OpenBitSet obs = null;
> > + FixedBitSet fbs = null;
> >
> > DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, DocsEnum.FLAG_NONE);
> > if (deState.docsEnum == null) {
> > @@ -1104,9 +1103,9 @@ public class SolrIndexSearcher extends I
> > int docid;
> >
> > if (largestPossible > docs.length) {
> > - if (obs == null) obs = new OpenBitSet(maxDoc());
> > + if (fbs == null) fbs = new FixedBitSet(maxDoc());
> > while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> > - obs.fastSet(docid + base);
> > + fbs.set(docid + base);
> > bitsSet++;
> > }
> > } else {
> > @@ -1118,9 +1117,9 @@ public class SolrIndexSearcher extends I
> > } else {
> > int docid;
> > if (largestPossible > docs.length) {
> > - if (obs == null) obs = new OpenBitSet(maxDoc());
> > + if (fbs == null) fbs = new FixedBitSet(maxDoc());
> > while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
> > - obs.fastSet(docid);
> > + fbs.set(docid);
> > bitsSet++;
> > }
> > } else {
> > @@ -1131,12 +1130,12 @@ public class SolrIndexSearcher extends I
> > }
> >
> > DocSet result;
> > - if (obs != null) {
> > + if (fbs != null) {
> > for (int i=0; i<upto; i++) {
> > - obs.fastSet(docs[i]);
> > + fbs.set(docs[i]);
> > }
> > bitsSet += upto;
> > - result = new BitDocSet(obs, bitsSet);
> > + result = new BitDocSet(fbs, bitsSet);
> > } else {
> > result = upto==0 ? DocSet.EMPTY : new SortedIntDocSet(Arrays.copyOf(docs, upto));
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java Sun Feb 16 07:50:08 2014
> > @@ -18,13 +18,13 @@
> > package org.apache.solr.search;
> >
> > import org.apache.lucene.index.AtomicReader;
> > -import org.apache.lucene.util.Bits;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.BitsFilteredDocIdSet;
> > import org.apache.lucene.search.DocIdSet;
> > import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.search.Filter;
> > -import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.util.Bits;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > * <code>SortedIntDocSet</code> represents a sorted set of Lucene Document Ids.
> > @@ -563,13 +563,12 @@ public class SortedIntDocSet extends Doc
> > }
> >
> > @Override
> > - public void setBitsOn(OpenBitSet target) {
> > + public void addAllTo(DocSet target) {
> > for (int doc : docs) {
> > - target.fastSet(doc);
> > + target.add(doc);
> > }
> > }
> >
> > -
> > @Override
> > public boolean exists(int doc) {
> > // this could be faster by estimating where in the list the doc is likely to appear,
> > @@ -630,16 +629,15 @@ public class SortedIntDocSet extends Doc
> > }
> >
> > @Override
> > - public OpenBitSet getBits() {
> > + public FixedBitSet getBits() {
> > int maxDoc = size() > 0 ? docs[size()-1] : 0;
> > - OpenBitSet bs = new OpenBitSet(maxDoc+1);
> > + FixedBitSet bs = new FixedBitSet(maxDoc+1);
> > for (int doc : docs) {
> > - bs.fastSet(doc);
> > + bs.set(doc);
> > }
> > return bs;
> > }
> >
> > -
> > public static int findIndex(int[] arr, int value, int low, int high) {
> > // binary search
> > while (low <= high) {
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java Sun Feb 16 07:50:08 2014
> > @@ -17,22 +17,31 @@ package org.apache.solr.search.grouping;
> > * limitations under the License.
> > */
> >
> > -import org.apache.lucene.search.*;
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.List;
> > +
> > +import org.apache.lucene.search.Collector;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.search.MultiCollector;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.TimeLimitingCollector;
> > +import org.apache.lucene.search.TotalHitCountCollector;
> > import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
> > import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.util.NamedList;
> > -import org.apache.solr.search.*;
> > -import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
> > +import org.apache.solr.search.BitDocSet;
> > +import org.apache.solr.search.DocSet;
> > +import org.apache.solr.search.DocSetCollector;
> > +import org.apache.solr.search.DocSetDelegateCollector;
> > import org.apache.solr.search.QueryUtils;
> > +import org.apache.solr.search.SolrIndexSearcher;
> > +import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
> > import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.List;
> > -
> > /**
> > * Responsible for executing a search with a number of {@link Command} instances.
> > * A typical search can have more then one {@link Command} instances.
> > @@ -159,9 +168,7 @@ public class CommandHandler {
> > searchWithTimeLimiter(query, filter, MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])));
> > }
> >
> > - int maxDoc = searcher.maxDoc();
> > - long[] bits = termAllGroupHeadsCollector.retrieveGroupHeads(maxDoc).getBits();
> > - return new BitDocSet(new OpenBitSet(bits, bits.length));
> > + return new BitDocSet(termAllGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
> > }
> >
> > private DocSet computeDocSet(Query query, ProcessedFilter filter, List<Collector> collectors) throws IOException {
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java Sun Feb 16 07:50:08 2014
> > @@ -17,30 +17,30 @@
> >
> > package org.apache.solr.search.join;
> >
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > class BitSetSlice {
> > - private final OpenBitSet obs;
> > + private final FixedBitSet fbs;
> > private final int off;
> > private final int len;
> >
> > - BitSetSlice(OpenBitSet obs, int off, int len) {
> > - this.obs = obs;
> > + BitSetSlice(FixedBitSet fbs, int off, int len) {
> > + this.fbs = fbs;
> > this.off = off;
> > this.len = len;
> > }
> >
> > public boolean get(int pos) {
> > - return obs.get(pos + off);
> > + return fbs.get(pos + off);
> > }
> >
> > public int prevSetBit(int pos) {
> > - int result = obs.prevSetBit(pos + off) - off;
> > + int result = fbs.prevSetBit(pos + off) - off;
> > return (result < 0) ? -1 : result;
> > }
> >
> > public int nextSetBit(int pos) {
> > - int result = obs.nextSetBit(pos + off) - off;
> > + int result = fbs.nextSetBit(pos + off) - off;
> > return (result < 0 || result >= len) ? -1 : result;
> > }
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java Sun Feb 16 07:50:08 2014
> > @@ -19,7 +19,7 @@ package org.apache.solr.store.blockcache
> >
> > import java.util.concurrent.atomic.AtomicLongArray;
> >
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.LongBitSet;
> >
> > public class BlockLocks {
> >
> > @@ -27,7 +27,7 @@ public class BlockLocks {
> > private int wlen;
> >
> > public BlockLocks(long numBits) {
> > - int length = OpenBitSet.bits2words(numBits);
> > + int length = LongBitSet.bits2words(numBits);
> > bits = new AtomicLongArray(length);
> > wlen = length;
> > }
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Sun Feb 16 07:50:08 2014
> > @@ -17,6 +17,10 @@
> >
> > package org.apache.solr.update;
> >
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.List;
> > +
> > import org.apache.lucene.index.AtomicReader;
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.index.DocsEnum;
> > @@ -26,11 +30,12 @@ import org.apache.lucene.index.IndexRead
> > import org.apache.lucene.index.IndexWriter;
> > import org.apache.lucene.index.Terms;
> > import org.apache.lucene.index.TermsEnum;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.CharsRef;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.lucene.util.IOUtils;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.solr.common.cloud.CompositeIdRouter;
> > import org.apache.solr.common.cloud.DocRouter;
> > import org.apache.solr.common.cloud.HashBasedRouter;
> > @@ -41,10 +46,6 @@ import org.apache.solr.util.RefCounted;
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.List;
> > -
> > public class SolrIndexSplitter {
> > public static Logger log = LoggerFactory.getLogger(SolrIndexSplitter.class);
> >
> > @@ -89,13 +90,13 @@ public class SolrIndexSplitter {
> > public void split() throws IOException {
> >
> > List<AtomicReaderContext> leaves = searcher.getTopReaderContext().leaves();
> > - List<OpenBitSet[]> segmentDocSets = new ArrayList<OpenBitSet[]>(leaves.size());
> > + List<FixedBitSet[]> segmentDocSets = new ArrayList<FixedBitSet[]>(leaves.size());
> >
> > log.info("SolrIndexSplitter: partitions=" + numPieces + " segments="+leaves.size());
> >
> > for (AtomicReaderContext readerContext : leaves) {
> > assert readerContext.ordInParent == segmentDocSets.size(); // make sure we're going in order
> > - OpenBitSet[] docSets = split(readerContext);
> > + FixedBitSet[] docSets = split(readerContext);
> > segmentDocSets.add( docSets );
> > }
> >
> > @@ -150,11 +151,11 @@ public class SolrIndexSplitter {
> >
> >
> >
> > - OpenBitSet[] split(AtomicReaderContext readerContext) throws IOException {
> > + FixedBitSet[] split(AtomicReaderContext readerContext) throws IOException {
> > AtomicReader reader = readerContext.reader();
> > - OpenBitSet[] docSets = new OpenBitSet[numPieces];
> > + FixedBitSet[] docSets = new FixedBitSet[numPieces];
> > for (int i=0; i<docSets.length; i++) {
> > - docSets[i] = new OpenBitSet(reader.maxDoc());
> > + docSets[i] = new FixedBitSet(reader.maxDoc());
> > }
> > Bits liveDocs = reader.getLiveDocs();
> >
> > @@ -196,14 +197,14 @@ public class SolrIndexSplitter {
> > docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
> > for (;;) {
> > int doc = docsEnum.nextDoc();
> > - if (doc == DocsEnum.NO_MORE_DOCS) break;
> > + if (doc == DocIdSetIterator.NO_MORE_DOCS) break;
> > if (ranges == null) {
> > - docSets[currPartition].fastSet(doc);
> > + docSets[currPartition].set(doc);
> > currPartition = (currPartition + 1) % numPieces;
> > } else {
> > for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
> > if (rangesArr[i].includes(hash)) {
> > - docSets[i].fastSet(doc);
> > + docSets[i].set(doc);
> > }
> > }
> > }
> > @@ -232,13 +233,13 @@ public class SolrIndexSplitter {
> >
> > // change livedocs on the reader to delete those docs we don't want
> > static class LiveDocsReader extends FilterAtomicReader {
> > - final OpenBitSet liveDocs;
> > + final FixedBitSet liveDocs;
> > final int numDocs;
> >
> > - public LiveDocsReader(AtomicReaderContext context, OpenBitSet liveDocs) throws IOException {
> > + public LiveDocsReader(AtomicReaderContext context, FixedBitSet liveDocs) throws IOException {
> > super(context.reader());
> > this.liveDocs = liveDocs;
> > - this.numDocs = (int)liveDocs.cardinality();
> > + this.numDocs = liveDocs.cardinality();
> > }
> >
> > @Override
> > @@ -253,6 +254,3 @@ public class SolrIndexSplitter {
> > }
> >
> > }
> > -
> > -
> > -
> >
> > Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> > URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> > ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java (original)
> > +++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,10 @@
> >
> > package org.apache.solr.search;
> >
> > -import org.apache.solr.search.BitDocSet;
> > -import org.apache.solr.search.HashDocSet;
> > -import org.apache.solr.search.DocSet;
> > -import org.apache.lucene.util.OpenBitSet;
> > -
> > -import java.util.Random;
> > import java.util.BitSet;
> > +import java.util.Random;
> > +
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > */
> > @@ -38,21 +35,20 @@ public class DocSetPerf {
> >
> > static Random rand = new Random();
> >
> > -
> > - static OpenBitSet bs;
> > + static FixedBitSet bs;
> > static BitDocSet bds;
> > static HashDocSet hds;
> > static int[] ids; // not unique
> >
> > static void generate(int maxSize, int bitsToSet) {
> > - bs = new OpenBitSet(maxSize);
> > + bs = new FixedBitSet(maxSize);
> > ids = new int[bitsToSet];
> > int count=0;
> > if (maxSize>0) {
> > for (int i=0; i<bitsToSet; i++) {
> > int id=rand.nextInt(maxSize);
> > if (!bs.get(id)) {
> > - bs.fastSet(id);
> > + bs.set(id);
> > ids[count++]=id;
> > }
> > }
> > @@ -80,7 +76,7 @@ public class DocSetPerf {
> >
> > int ret=0;
> >
> > - OpenBitSet[] sets = new OpenBitSet[numSets];
> > + FixedBitSet[] sets = new FixedBitSet[numSets];
> > DocSet[] bset = new DocSet[numSets];
> > DocSet[] hset = new DocSet[numSets];
> > BitSet scratch=new BitSet();
> > @@ -97,14 +93,14 @@ public class DocSetPerf {
> > if ("test".equals(test)) {
> > for (int it=0; it<iter; it++) {
> > generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
> > - OpenBitSet bs1=bs;
> > + FixedBitSet bs1=bs;
> > BitDocSet bds1=bds;
> > HashDocSet hds1=hds;
> > generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
> >
> > - OpenBitSet res = ((OpenBitSet)bs1.clone());
> > + FixedBitSet res = bs1.clone();
> > res.and(bs);
> > - int icount = (int)res.cardinality();
> > + int icount = res.cardinality();
> >
> > test(bds1.intersection(bds).size() == icount);
> > test(bds1.intersectionSize(bds) == icount);
> > @@ -167,8 +163,6 @@ public class DocSetPerf {
> > }
> > }
> >
> > -
> > -
> > long end = System.currentTimeMillis();
> > System.out.println("TIME="+(end-start));
> >
> > @@ -176,6 +170,4 @@ public class DocSetPerf {
> > System.out.println("ret="+ret);
> > }
> >
> > -
> > -
> > }
> >
> >
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1568738 [1/2] - in /lucene/dev/branches/branch_4x:
./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/util/
lucene/grouping/ lucene/grouping/src/java/org/apache/lucene/search/grouping/
solr/ solr/core/ solr/core/src/java/org/ap...
Posted by Shai Erera <se...@gmail.com>.
I ran Solr tests both on trunk and 4x several times before I committed.
I'll dig.
Shai
On Sun, Feb 16, 2014 at 9:59 PM, Mark Miller <ma...@gmail.com> wrote:
> FYI, somehow this issue broke Solr tests I think:
>
> [junit4] 2> Caused by: java.lang.AssertionError: index=36,
> numBits=36
> [junit4] 2> at
> org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
> [junit4] 2> at
> org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
> [junit4] 2> at
> org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
> [junit4] 2> at
> org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
> [junit4] 2> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
> [junit4] 2> at
> org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
> [junit4] 2> ... 26 more
> [junit4] 2>
>
> [junit4] 2> 2818355 T8214 oejs.ServletHandler.doHandle WARN Error for
> /select java.lang.AssertionError: index=36, numBits=36
> [junit4] 2> at
> org.apache.lucene.util.FixedBitSet.get(FixedBitSet.java:235)
> [junit4] 2> at
> org.apache.solr.handler.component.FacetComponent.countFacets(FacetComponent.java:469)
> [junit4] 2> at
> org.apache.solr.handler.component.FacetComponent.handleResponses(FacetComponent.java:283)
> [junit4] 2> at
> org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:322)
> [junit4] 2> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
> [junit4] 2> at
> org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
> [junit4] 2> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
> [junit4] 2> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
> [junit4] 2> at
> org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:134)
>
> - Mark
>
> http://about.me/markrmiller
>
> On Feb 16, 2014, at 2:50 AM, shaie@apache.org wrote:
>
> > Author: shaie
> > Date: Sun Feb 16 07:50:08 2014
> > New Revision: 1568738
> >
> > URL: http://svn.apache.org/r1568738
> > Log:
> > LUCENE-5440: decouple OpenBitSet from DocSet and move to use FixedBitSet
> >
> > Modified:
> > lucene/dev/branches/branch_4x/ (props changed)
> > lucene/dev/branches/branch_4x/lucene/ (props changed)
> > lucene/dev/branches/branch_4x/lucene/core/ (props changed)
> >
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> >
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> > lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
> >
> lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> > lucene/dev/branches/branch_4x/solr/ (props changed)
> > lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props
> changed)
> > lucene/dev/branches/branch_4x/solr/core/ (props changed)
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestDocSet.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestFiltering.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestSort.java
> >
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/util/BitSetPerf.java
> >
> > Modified:
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
> Sun Feb 16 07:50:08 2014
> > @@ -28,6 +28,8 @@ import org.apache.lucene.search.DocIdSet
> > * long[], accessed with an int index, implementing {@link Bits} and
> > * {@link DocIdSet}. If you need to manage more than 2.1B bits, use
> > * {@link LongBitSet}.
> > + *
> > + * @lucene.internal
> > */
> > public final class FixedBitSet extends DocIdSet implements Bits {
> >
> > @@ -41,6 +43,12 @@ public final class FixedBitSet extends D
> > final long[] bits;
> > int doc = -1;
> >
> > + /** Creates an iterator over the given {@link FixedBitSet}. */
> > + public FixedBitSetIterator(FixedBitSet bits) {
> > + this(bits.bits, bits.numBits, bits.numWords);
> > + }
> > +
> > + /** Creates an iterator over the given array of bits. */
> > public FixedBitSetIterator(long[] bits, int numBits, int wordLength)
> {
> > this.bits = bits;
> > this.numBits = numBits;
> > @@ -48,7 +56,7 @@ public final class FixedBitSet extends D
> > }
> >
> > @Override
> > - public int nextDoc() throws IOException {
> > + public int nextDoc() {
> > if (doc == NO_MORE_DOCS || ++doc >= numBits) {
> > return doc = NO_MORE_DOCS;
> > }
> > @@ -81,7 +89,7 @@ public final class FixedBitSet extends D
> > }
> >
> > @Override
> > - public int advance(int target) throws IOException {
> > + public int advance(int target) {
> > if (doc == NO_MORE_DOCS || target >= numBits) {
> > return doc = NO_MORE_DOCS;
> > }
> > @@ -103,10 +111,6 @@ public final class FixedBitSet extends D
> > return doc = NO_MORE_DOCS;
> > }
> > }
> > -
> > - private final long[] bits;
> > - private final int numBits;
> > - private final int wordLength;
> >
> > /**
> > * If the given {@link FixedBitSet} is large enough to hold {@code
> numBits},
> > @@ -140,15 +144,53 @@ public final class FixedBitSet extends D
> > return numLong;
> > }
> >
> > + /**
> > + * Returns the popcount or cardinality of the intersection of the two
> sets.
> > + * Neither set is modified.
> > + */
> > + public static long intersectionCount(FixedBitSet a, FixedBitSet b) {
> > + return BitUtil.pop_intersect(a.bits, b.bits, 0,
> Math.min(a.numWords, b.numWords));
> > + }
> > +
> > + /**
> > + * Returns the popcount or cardinality of the union of the two sets.
> Neither
> > + * set is modified.
> > + */
> > + public static long unionCount(FixedBitSet a, FixedBitSet b) {
> > + long tot = BitUtil.pop_union(a.bits, b.bits, 0,
> Math.min(a.numWords, b.numWords));
> > + if (a.numWords < b.numWords) {
> > + tot += BitUtil.pop_array(b.bits, a.numWords, b.numWords -
> a.numWords);
> > + } else if (a.numWords > b.numWords) {
> > + tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords -
> b.numWords);
> > + }
> > + return tot;
> > + }
> > +
> > + /**
> > + * Returns the popcount or cardinality of "a and not b" or
> > + * "intersection(a, not(b))". Neither set is modified.
> > + */
> > + public static long andNotCount(FixedBitSet a, FixedBitSet b) {
> > + long tot = BitUtil.pop_andnot(a.bits, b.bits, 0,
> Math.min(a.numWords, b.numWords));
> > + if (a.numWords > b.numWords) {
> > + tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords -
> b.numWords);
> > + }
> > + return tot;
> > + }
> > +
> > + final long[] bits;
> > + final int numBits;
> > + final int numWords;
> > +
> > public FixedBitSet(int numBits) {
> > this.numBits = numBits;
> > bits = new long[bits2words(numBits)];
> > - wordLength = bits.length;
> > + numWords = bits.length;
> > }
> >
> > public FixedBitSet(long[] storedBits, int numBits) {
> > - this.wordLength = bits2words(numBits);
> > - if (wordLength > storedBits.length) {
> > + this.numWords = bits2words(numBits);
> > + if (numWords > storedBits.length) {
> > throw new IllegalArgumentException("The given long array is too
> small to hold " + numBits + " bits");
> > }
> > this.numBits = numBits;
> > @@ -157,7 +199,7 @@ public final class FixedBitSet extends D
> >
> > @Override
> > public DocIdSetIterator iterator() {
> > - return new FixedBitSetIterator(bits, numBits, wordLength);
> > + return new FixedBitSetIterator(bits, numBits, numWords);
> > }
> >
> > @Override
> > @@ -190,7 +232,7 @@ public final class FixedBitSet extends D
> >
> > @Override
> > public boolean get(int index) {
> > - assert index >= 0 && index < numBits: "index=" + index;
> > + assert index >= 0 && index < numBits: "index=" + index + ",
> numBits=" + numBits;
> > int i = index >> 6; // div 64
> > // signed shift will keep a negative index and force an
> > // array-index-out-of-bounds-exception, removing the need for an
> explicit check.
> > @@ -200,7 +242,7 @@ public final class FixedBitSet extends D
> > }
> >
> > public void set(int index) {
> > - assert index >= 0 && index < numBits: "index=" + index + "
> numBits=" + numBits;
> > + assert index >= 0 && index < numBits: "index=" + index + ",
> numBits=" + numBits;
> > int wordNum = index >> 6; // div 64
> > int bit = index & 0x3f; // mod 64
> > long bitmask = 1L << bit;
> > @@ -239,7 +281,7 @@ public final class FixedBitSet extends D
> > * -1 is returned if there are no more set bits.
> > */
> > public int nextSetBit(int index) {
> > - assert index >= 0 && index < numBits;
> > + assert index >= 0 && index < numBits : "index=" + index + ",
> numBits=" + numBits;
> > int i = index >> 6;
> > final int subIndex = index & 0x3f; // index within the word
> > long word = bits[i] >> subIndex; // skip all the bits to the right
> of index
> > @@ -248,7 +290,7 @@ public final class FixedBitSet extends D
> > return index + Long.numberOfTrailingZeros(word);
> > }
> >
> > - while(++i < wordLength) {
> > + while(++i < numWords) {
> > word = bits[i];
> > if (word != 0) {
> > return (i<<6) + Long.numberOfTrailingZeros(word);
> > @@ -306,12 +348,13 @@ public final class FixedBitSet extends D
> >
> > /** this = this OR other */
> > public void or(FixedBitSet other) {
> > - or(other.bits, other.wordLength);
> > + or(other.bits, other.numWords);
> > }
> >
> > - private void or(final long[] otherArr, final int otherLen) {
> > + private void or(final long[] otherArr, final int otherNumWords) {
> > + assert otherNumWords <= numWords : "numWords=" + numWords + ",
> otherNumWords=" + otherNumWords;
> > final long[] thisArr = this.bits;
> > - int pos = Math.min(wordLength, otherLen);
> > + int pos = Math.min(numWords, otherNumWords);
> > while (--pos >= 0) {
> > thisArr[pos] |= otherArr[pos];
> > }
> > @@ -319,9 +362,10 @@ public final class FixedBitSet extends D
> >
> > /** this = this XOR other */
> > public void xor(FixedBitSet other) {
> > + assert other.numWords <= numWords : "numWords=" + numWords + ",
> other.numWords=" + other.numWords;
> > final long[] thisBits = this.bits;
> > final long[] otherBits = other.bits;
> > - int pos = Math.min(wordLength, other.wordLength);
> > + int pos = Math.min(numWords, other.numWords);
> > while (--pos >= 0) {
> > thisBits[pos] ^= otherBits[pos];
> > }
> > @@ -366,7 +410,7 @@ public final class FixedBitSet extends D
> >
> > /** returns true if the sets have any elements in common */
> > public boolean intersects(FixedBitSet other) {
> > - int pos = Math.min(wordLength, other.wordLength);
> > + int pos = Math.min(numWords, other.numWords);
> > while (--pos>=0) {
> > if ((bits[pos] & other.bits[pos]) != 0) return true;
> > }
> > @@ -375,17 +419,17 @@ public final class FixedBitSet extends D
> >
> > /** this = this AND other */
> > public void and(FixedBitSet other) {
> > - and(other.bits, other.wordLength);
> > + and(other.bits, other.numWords);
> > }
> >
> > - private void and(final long[] otherArr, final int otherLen) {
> > + private void and(final long[] otherArr, final int otherNumWords) {
> > final long[] thisArr = this.bits;
> > - int pos = Math.min(this.wordLength, otherLen);
> > + int pos = Math.min(this.numWords, otherNumWords);
> > while(--pos >= 0) {
> > thisArr[pos] &= otherArr[pos];
> > }
> > - if (this.wordLength > otherLen) {
> > - Arrays.fill(thisArr, otherLen, this.wordLength, 0L);
> > + if (this.numWords > otherNumWords) {
> > + Arrays.fill(thisArr, otherNumWords, this.numWords, 0L);
> > }
> > }
> >
> > @@ -417,9 +461,9 @@ public final class FixedBitSet extends D
> > andNot(other.bits, other.bits.length);
> > }
> >
> > - private void andNot(final long[] otherArr, final int otherLen) {
> > + private void andNot(final long[] otherArr, final int otherNumWords) {
> > final long[] thisArr = this.bits;
> > - int pos = Math.min(this.wordLength, otherLen);
> > + int pos = Math.min(this.numWords, otherNumWords);
> > while(--pos >= 0) {
> > thisArr[pos] &= ~otherArr[pos];
> > }
> > @@ -502,8 +546,8 @@ public final class FixedBitSet extends D
> > * @param endIndex one-past the last bit to clear
> > */
> > public void clear(int startIndex, int endIndex) {
> > - assert startIndex >= 0 && startIndex < numBits;
> > - assert endIndex >= 0 && endIndex <= numBits;
> > + assert startIndex >= 0 && startIndex < numBits : "startIndex=" +
> startIndex + ", numBits=" + numBits;
> > + assert endIndex >= 0 && endIndex <= numBits : "endIndex=" +
> endIndex + ", numBits=" + numBits;
> > if (endIndex <= startIndex) {
> > return;
> > }
> > @@ -554,7 +598,7 @@ public final class FixedBitSet extends D
> > @Override
> > public int hashCode() {
> > long h = 0;
> > - for (int i = wordLength; --i>=0;) {
> > + for (int i = numWords; --i>=0;) {
> > h ^= bits[i];
> > h = (h << 1) | (h >>> 63); // rotate left
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
> Sun Feb 16 07:50:08 2014
> > @@ -23,6 +23,8 @@ import java.util.Arrays;
> > * BitSet of fixed length (numBits), backed by accessible ({@link
> #getBits})
> > * long[], accessed with a long index. Use it only if you intend to
> store more
> > * than 2.1B bits, otherwise you should use {@link FixedBitSet}.
> > + *
> > + * @lucene.internal
> > */
> > public final class LongBitSet {
> >
> > @@ -188,6 +190,7 @@ public final class LongBitSet {
> >
> > /** this = this OR other */
> > public void or(LongBitSet other) {
> > + assert other.numWords <= numWords : "numWords=" + numWords + ",
> other.numWords=" + other.numWords;
> > int pos = Math.min(numWords, other.numWords);
> > while (--pos >= 0) {
> > bits[pos] |= other.bits[pos];
> > @@ -196,6 +199,7 @@ public final class LongBitSet {
> >
> > /** this = this XOR other */
> > public void xor(LongBitSet other) {
> > + assert other.numWords <= numWords : "numWords=" + numWords + ",
> other.numWords=" + other.numWords;
> > int pos = Math.min(numWords, other.numWords);
> > while (--pos >= 0) {
> > bits[pos] ^= other.bits[pos];
> >
> > Modified:
> lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
> Sun Feb 16 07:50:08 2014
> > @@ -44,7 +44,7 @@ public abstract class AbstractAllGroupHe
> >
> > /**
> > * @param maxDoc The maxDoc of the top level {@link IndexReader}.
> > - * @return an {@link FixedBitSet} containing all group heads.
> > + * @return a {@link FixedBitSet} containing all group heads.
> > */
> > public FixedBitSet retrieveGroupHeads(int maxDoc) {
> > FixedBitSet bitSet = new FixedBitSet(maxDoc);
> >
> > Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > --- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
> > +++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sun Feb 16 07:50:08
> 2014
> > @@ -330,6 +330,10 @@ Optimizations
> > * SOLR-5624: Enable QueryResultCache for CollapsingQParserPlugin.
> > (David Boychuck, Joel Bernstein)
> >
> > +* LUCENE-5440: DocSet decoupled from OpenBitSet. DocSetBase moved to use
> > + FixedBitSet instead of OpenBitSet. As a result BitDocSet now only
> works
> > + with FixedBitSet. (Shai Erera)
> > +
> > Other Changes
> > ---------------------
> >
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,7 +17,17 @@
> >
> > package org.apache.solr.handler.component;
> >
> > -import org.apache.lucene.util.OpenBitSet;
> > +import java.io.IOException;
> > +import java.net.URL;
> > +import java.util.ArrayList;
> > +import java.util.Arrays;
> > +import java.util.Comparator;
> > +import java.util.HashMap;
> > +import java.util.LinkedHashMap;
> > +import java.util.List;
> > +import java.util.Map;
> > +
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.SolrException.ErrorCode;
> > import org.apache.solr.common.params.CommonParams;
> > @@ -35,10 +45,6 @@ import org.apache.solr.search.SyntaxErro
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.net.URL;
> > -import java.util.*;
> > -
> > /**
> > * TODO!
> > *
> > @@ -444,8 +450,8 @@ public class FacetComponent extends Sear
> > // and if it is >= smallestCount, then flag for refinement
> > long maxCount = sfc.count;
> > for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
> > - OpenBitSet obs = dff.counted[shardNum];
> > - if (obs!=null && !obs.get(sfc.termNum)) { // obs can be
> null if a shard request failed
> > + FixedBitSet fbs = dff.counted[shardNum];
> > + if (fbs!=null && !fbs.get(sfc.termNum)) { // fbs can be
> null if a shard request failed
> > // if missing from this shard, add the max it could be
> > maxCount += dff.maxPossible(sfc,shardNum);
> > }
> > @@ -459,8 +465,8 @@ public class FacetComponent extends Sear
> > if (needRefinement) {
> > // add a query for each shard missing the term that needs
> refinement
> > for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
> > - OpenBitSet obs = dff.counted[shardNum];
> > - if(obs!=null && !obs.get(sfc.termNum) &&
> dff.maxPossible(sfc,shardNum)>0) {
> > + FixedBitSet fbs = dff.counted[shardNum];
> > + if(fbs!=null && !fbs.get(sfc.termNum) &&
> dff.maxPossible(sfc,shardNum)>0) {
> > dff.needRefinements = true;
> > List<String> lst = dff._toRefine[shardNum];
> > if (lst == null) {
> > @@ -759,7 +765,7 @@ public class FacetComponent extends Sear
> > public long missingMaxPossible;
> > // the max possible count for a missing term for each shard (indexed
> by shardNum)
> > public long[] missingMax;
> > - public OpenBitSet[] counted; // a bitset for each shard, keeping
> track of which terms seen
> > + public FixedBitSet[] counted; // a bitset for each shard, keeping
> track of which terms seen
> > public HashMap<String,ShardFacetCount> counts = new
> HashMap<String,ShardFacetCount>(128);
> > public int termNum;
> >
> > @@ -772,7 +778,7 @@ public class FacetComponent extends Sear
> > super(rb, facetStr);
> > // sf = rb.req.getSchema().getField(field);
> > missingMax = new long[rb.shards.length];
> > - counted = new OpenBitSet[rb.shards.length];
> > + counted = new FixedBitSet[rb.shards.length];
> > }
> >
> > void add(int shardNum, NamedList shardCounts, int numRequested) {
> > @@ -780,7 +786,7 @@ public class FacetComponent extends Sear
> > int sz = shardCounts == null ? 0 : shardCounts.size();
> > int numReceived = sz;
> >
> > - OpenBitSet terms = new OpenBitSet(termNum+sz);
> > + FixedBitSet terms = new FixedBitSet(termNum+sz);
> >
> > long last = 0;
> > for (int i=0; i<sz; i++) {
> > @@ -799,7 +805,7 @@ public class FacetComponent extends Sear
> > counts.put(name, sfc);
> > }
> > sfc.count += count;
> > - terms.fastSet(sfc.termNum);
> > + terms.set(sfc.termNum);
> > last = count;
> > }
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,6 +17,26 @@
> >
> > package org.apache.solr.request;
> >
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.Collection;
> > +import java.util.Date;
> > +import java.util.EnumSet;
> > +import java.util.IdentityHashMap;
> > +import java.util.List;
> > +import java.util.Map;
> > +import java.util.Set;
> > +import java.util.concurrent.Callable;
> > +import java.util.concurrent.ExecutionException;
> > +import java.util.concurrent.Executor;
> > +import java.util.concurrent.Future;
> > +import java.util.concurrent.FutureTask;
> > +import java.util.concurrent.RunnableFuture;
> > +import java.util.concurrent.Semaphore;
> > +import java.util.concurrent.SynchronousQueue;
> > +import java.util.concurrent.ThreadPoolExecutor;
> > +import java.util.concurrent.TimeUnit;
> > +
> > import org.apache.lucene.index.AtomicReader;
> > import org.apache.lucene.index.DocsEnum;
> > import org.apache.lucene.index.Fields;
> > @@ -37,8 +57,6 @@ import org.apache.lucene.search.grouping
> > import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.CharsRef;
> > -import org.apache.lucene.util.FixedBitSet;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.lucene.util.StringHelper;
> > import org.apache.lucene.util.UnicodeUtil;
> > import org.apache.solr.common.SolrException;
> > @@ -80,26 +98,6 @@ import org.apache.solr.util.DateMathPars
> > import org.apache.solr.util.DefaultSolrThreadFactory;
> > import org.apache.solr.util.LongPriorityQueue;
> >
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.Collection;
> > -import java.util.Date;
> > -import java.util.EnumSet;
> > -import java.util.IdentityHashMap;
> > -import java.util.List;
> > -import java.util.Map;
> > -import java.util.Set;
> > -import java.util.concurrent.Callable;
> > -import java.util.concurrent.ExecutionException;
> > -import java.util.concurrent.Executor;
> > -import java.util.concurrent.Future;
> > -import java.util.concurrent.FutureTask;
> > -import java.util.concurrent.RunnableFuture;
> > -import java.util.concurrent.Semaphore;
> > -import java.util.concurrent.SynchronousQueue;
> > -import java.util.concurrent.ThreadPoolExecutor;
> > -import java.util.concurrent.TimeUnit;
> > -
> > /**
> > * A class that generates simple Facet information for a request.
> > *
> > @@ -230,10 +228,7 @@ public class SimpleFacets {
> > }
> > AbstractAllGroupHeadsCollector allGroupHeadsCollector =
> grouping.getCommands().get(0).createAllGroupCollector();
> > searcher.search(new MatchAllDocsQuery(), base.getTopFilter(),
> allGroupHeadsCollector);
> > - int maxDoc = searcher.maxDoc();
> > - FixedBitSet fixedBitSet =
> allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
> > - long[] bits = fixedBitSet.getBits();
> > - this.docs = new BitDocSet(new OpenBitSet(bits, bits.length));
> > + this.docs = new
> BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
> > } else {
> > this.docs = base;
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
> Sun Feb 16 07:50:08 2014
> > @@ -27,12 +27,11 @@ import org.apache.lucene.index.DocTermOr
> > import org.apache.lucene.index.SortedDocValues;
> > import org.apache.lucene.index.Term;
> > import org.apache.lucene.index.TermsEnum;
> > -import org.apache.lucene.search.FieldCache;
> > import org.apache.lucene.search.TermQuery;
> > import org.apache.lucene.search.TermRangeQuery;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.CharsRef;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.lucene.util.UnicodeUtil;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.params.FacetParams;
> > @@ -44,7 +43,11 @@ import org.apache.solr.handler.component
> > import org.apache.solr.schema.FieldType;
> > import org.apache.solr.schema.SchemaField;
> > import org.apache.solr.schema.TrieField;
> > -import org.apache.solr.search.*;
> > +import org.apache.solr.search.BitDocSet;
> > +import org.apache.solr.search.DocIterator;
> > +import org.apache.solr.search.DocSet;
> > +import org.apache.solr.search.SolrCache;
> > +import org.apache.solr.search.SolrIndexSearcher;
> > import org.apache.solr.util.LongPriorityQueue;
> > import org.apache.solr.util.PrimUtils;
> >
> > @@ -269,7 +272,7 @@ public class UnInvertedField extends Doc
> > && docs instanceof BitDocSet;
> >
> > if (doNegative) {
> > - OpenBitSet bs = (OpenBitSet)((BitDocSet)docs).getBits().clone();
> > + FixedBitSet bs = ((BitDocSet)docs).getBits().clone();
> > bs.flip(0, maxDoc);
> > // TODO: when iterator across negative elements is available,
> use that
> > // instead of creating a new bitset and inverting.
> > @@ -512,7 +515,7 @@ public class UnInvertedField extends Doc
> > }
> >
> > if (doNegative) {
> > - OpenBitSet bs = (OpenBitSet) ((BitDocSet) docs).getBits().clone();
> > + FixedBitSet bs = ((BitDocSet) docs).getBits().clone();
> > bs.flip(0, maxDoc);
> > // TODO: when iterator across negative elements is available, use
> that
> > // instead of creating a new bitset and inverting.
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/BitDocSet.java
> Sun Feb 16 07:50:08 2014
> > @@ -21,38 +21,40 @@ import org.apache.lucene.index.AtomicRea
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.BitsFilteredDocIdSet;
> > import org.apache.lucene.search.DocIdSet;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.search.Filter;
> > import org.apache.lucene.util.Bits;
> > -import org.apache.lucene.util.OpenBitSet;
> > -import org.apache.lucene.util.OpenBitSetIterator;
> > -import org.apache.lucene.search.DocIdSetIterator;
> > +import org.apache.lucene.util.FixedBitSet;
> > +import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
> >
> > /**
> > * <code>BitDocSet</code> represents an unordered set of Lucene Document
> Ids
> > * using a BitSet. A set bit represents inclusion in the set for that
> document.
> > *
> > - *
> > * @since solr 0.9
> > */
> > public class BitDocSet extends DocSetBase {
> > - final OpenBitSet bits;
> > + final FixedBitSet bits;
> > int size; // number of docs in the set (cached for perf)
> >
> > public BitDocSet() {
> > - bits = new OpenBitSet();
> > + bits = new FixedBitSet(64);
> > }
> >
> > - /** Construct a BitDocSet.
> > - * The capacity of the OpenBitSet should be at least maxDoc() */
> > - public BitDocSet(OpenBitSet bits) {
> > + /**
> > + * Construct a BitDocSet. The capacity of the {@link FixedBitSet}
> should be at
> > + * least maxDoc()
> > + */
> > + public BitDocSet(FixedBitSet bits) {
> > this.bits = bits;
> > size=-1;
> > }
> >
> > - /** Construct a BitDocSet, and provides the number of set bits.
> > - * The capacity of the OpenBitSet should be at least maxDoc()
> > + /**
> > + * Construct a BitDocSet, and provides the number of set bits. The
> capacity of
> > + * the {@link FixedBitSet} should be at least maxDoc()
> > */
> > - public BitDocSet(OpenBitSet bits, int size) {
> > + public BitDocSet(FixedBitSet bits, int size) {
> > this.bits = bits;
> > this.size = size;
> > }
> > @@ -89,7 +91,7 @@ public class BitDocSet extends DocSetBas
> > @Override
> > public DocIterator iterator() {
> > return new DocIterator() {
> > - private final OpenBitSetIterator iter = new
> OpenBitSetIterator(bits);
> > + private final FixedBitSetIterator iter = new
> FixedBitSetIterator(bits);
> > private int pos = iter.nextDoc();
> > @Override
> > public boolean hasNext() {
> > @@ -120,13 +122,11 @@ public class BitDocSet extends DocSetBas
> > };
> > }
> >
> > -
> > /**
> > - *
> > - * @return the <b>internal</b> OpenBitSet that should <b>not</b> be
> modified.
> > + * @return the <b>internal</b> {@link FixedBitSet} that should
> <b>not</b> be modified.
> > */
> > @Override
> > - public OpenBitSet getBits() {
> > + public FixedBitSet getBits() {
> > return bits;
> > }
> >
> > @@ -145,7 +145,7 @@ public class BitDocSet extends DocSetBas
> > @Override
> > public int size() {
> > if (size!=-1) return size;
> > - return size=(int)bits.cardinality();
> > + return size = bits.cardinality();
> > }
> >
> > /**
> > @@ -156,18 +156,19 @@ public class BitDocSet extends DocSetBas
> > size=-1;
> > }
> >
> > - /** Returns true of the doc exists in the set.
> > - * Should only be called when doc < OpenBitSet.size()
> > + /**
> > + * Returns true of the doc exists in the set. Should only be called
> when doc <
> > + * {@link FixedBitSet#length()}.
> > */
> > @Override
> > public boolean exists(int doc) {
> > - return bits.fastGet(doc);
> > + return bits.get(doc);
> > }
> >
> > @Override
> > public int intersectionSize(DocSet other) {
> > if (other instanceof BitDocSet) {
> > - return (int)OpenBitSet.intersectionCount(this.bits,
> ((BitDocSet)other).bits);
> > + return (int) FixedBitSet.intersectionCount(this.bits,
> ((BitDocSet) other).bits);
> > } else {
> > // they had better not call us back!
> > return other.intersectionSize(this);
> > @@ -189,7 +190,7 @@ public class BitDocSet extends DocSetBas
> > if (other instanceof BitDocSet) {
> > // if we don't know our current size, this is faster than
> > // size + other.size - intersection_size
> > - return (int)OpenBitSet.unionCount(this.bits,
> ((BitDocSet)other).bits);
> > + return (int) FixedBitSet.unionCount(this.bits,
> ((BitDocSet)other).bits);
> > } else {
> > // they had better not call us back!
> > return other.unionSize(this);
> > @@ -201,42 +202,56 @@ public class BitDocSet extends DocSetBas
> > if (other instanceof BitDocSet) {
> > // if we don't know our current size, this is faster than
> > // size - intersection_size
> > - return (int)OpenBitSet.andNotCount(this.bits,
> ((BitDocSet)other).bits);
> > + return (int) FixedBitSet.andNotCount(this.bits,
> ((BitDocSet)other).bits);
> > } else {
> > return super.andNotSize(other);
> > }
> > }
> >
> > @Override
> > - public void setBitsOn(OpenBitSet target) {
> > - target.union(bits);
> > + public void addAllTo(DocSet target) {
> > + if (target instanceof BitDocSet) {
> > + ((BitDocSet) target).bits.or(bits);
> > + } else {
> > + super.addAllTo(target);
> > + }
> > }
> >
> > @Override
> > - public DocSet andNot(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(bits.clone());
> > - if (other instanceof BitDocSet) {
> > - newbits.andNot(((BitDocSet)other).bits);
> > - } else {
> > - DocIterator iter = other.iterator();
> > - while (iter.hasNext()) newbits.clear(iter.nextDoc());
> > - }
> > - return new BitDocSet(newbits);
> > + public DocSet andNot(DocSet other) {
> > + FixedBitSet newbits = bits.clone();
> > + if (other instanceof BitDocSet) {
> > + newbits.andNot(((BitDocSet) other).bits);
> > + } else {
> > + DocIterator iter = other.iterator();
> > + while (iter.hasNext()) {
> > + int doc = iter.nextDoc();
> > + if (doc < newbits.length()) {
> > + newbits.clear(doc);
> > + }
> > + }
> > + }
> > + return new BitDocSet(newbits);
> > }
> > -
> > +
> > @Override
> > - public DocSet union(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(bits.clone());
> > - if (other instanceof BitDocSet) {
> > - newbits.union(((BitDocSet)other).bits);
> > - } else {
> > - DocIterator iter = other.iterator();
> > - while (iter.hasNext()) newbits.set(iter.nextDoc());
> > - }
> > - return new BitDocSet(newbits);
> > + public DocSet union(DocSet other) {
> > + FixedBitSet newbits = bits.clone();
> > + if (other instanceof BitDocSet) {
> > + BitDocSet otherDocSet = (BitDocSet) other;
> > + newbits = FixedBitSet.ensureCapacity(newbits,
> otherDocSet.bits.length());
> > + newbits.or(otherDocSet.bits);
> > + } else {
> > + DocIterator iter = other.iterator();
> > + while (iter.hasNext()) {
> > + int doc = iter.nextDoc();
> > + newbits = FixedBitSet.ensureCapacity(newbits, doc);
> > + newbits.set(doc);
> > + }
> > + }
> > + return new BitDocSet(newbits);
> > }
> > -
> > -
> > +
> > @Override
> > public long memSize() {
> > return (bits.getBits().length << 3) + 16;
> > @@ -244,12 +259,12 @@ public class BitDocSet extends DocSetBas
> >
> > @Override
> > protected BitDocSet clone() {
> > - return new BitDocSet((OpenBitSet)bits.clone(), size);
> > + return new BitDocSet(bits.clone(), size);
> > }
> >
> > @Override
> > public Filter getTopFilter() {
> > - final OpenBitSet bs = bits;
> > + final FixedBitSet bs = bits;
> > // TODO: if cardinality isn't cached, do a quick measure of
> sparseness
> > // and return null from bits() if too sparse.
> >
> > @@ -272,8 +287,8 @@ public class BitDocSet extends DocSetBas
> > @Override
> > public DocIdSetIterator iterator() {
> > return new DocIdSetIterator() {
> > - int pos=base-1;
> > - int adjustedDoc=-1;
> > + int pos = base - 1;
> > + int adjustedDoc = -1;
> >
> > @Override
> > public int docID() {
> > @@ -282,15 +297,24 @@ public class BitDocSet extends DocSetBas
> >
> > @Override
> > public int nextDoc() {
> > - pos = bs.nextSetBit(pos+1);
> > - return adjustedDoc = (pos>=0 && pos<max) ? pos-base :
> NO_MORE_DOCS;
> > + if (pos >= bs.length() - 1) {
> > + return adjustedDoc = NO_MORE_DOCS;
> > + } else {
> > + pos = bs.nextSetBit(pos + 1);
> > + return adjustedDoc = (pos >= 0 && pos < max) ? pos -
> base : NO_MORE_DOCS;
> > + }
> > }
> >
> > @Override
> > public int advance(int target) {
> > - if (target==NO_MORE_DOCS) return
> adjustedDoc=NO_MORE_DOCS;
> > - pos = bs.nextSetBit(target+base);
> > - return adjustedDoc = (pos>=0 && pos<max) ? pos-base :
> NO_MORE_DOCS;
> > + if (target == NO_MORE_DOCS) return adjustedDoc =
> NO_MORE_DOCS;
> > + int adjusted = target + base;
> > + if (adjusted >= bs.length()) {
> > + return adjustedDoc = NO_MORE_DOCS;
> > + } else {
> > + pos = bs.nextSetBit(adjusted);
> > + return adjustedDoc = (pos >= 0 && pos < max) ? pos -
> base : NO_MORE_DOCS;
> > + }
> > }
> >
> > @Override
> > @@ -298,7 +322,7 @@ public class BitDocSet extends DocSetBas
> > // we don't want to actually compute cardinality, but
> > // if its already been computed, we use it (pro-rated
> for the segment)
> > if (size != -1) {
> > - return (long)(size *
> ((OpenBitSet.bits2words(maxDoc)<<6) / (float)bs.capacity()));
> > + return (long)(size *
> ((FixedBitSet.bits2words(maxDoc)<<6) / (float)bs.length()));
> > } else {
> > return maxDoc;
> > }
> > @@ -316,7 +340,7 @@ public class BitDocSet extends DocSetBas
> > return new Bits() {
> > @Override
> > public boolean get(int index) {
> > - return bs.fastGet(index + base);
> > + return bs.get(index + base);
> > }
> >
> > @Override
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,45 +17,49 @@
> >
> > package org.apache.solr.search;
> >
> > +import java.io.IOException;
> > +import java.util.Arrays;
> > +import java.util.HashSet;
> > +import java.util.Iterator;
> > +import java.util.List;
> > +import java.util.Map;
> > +import java.util.Set;
> > +
> > +import org.apache.lucene.index.AtomicReader;
> > +import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.index.DocsEnum;
> > +import org.apache.lucene.index.SortedDocValues;
> > +import org.apache.lucene.index.Terms;
> > +import org.apache.lucene.index.TermsEnum;
> > import org.apache.lucene.queries.function.FunctionQuery;
> > import org.apache.lucene.queries.function.FunctionValues;
> > import org.apache.lucene.queries.function.ValueSource;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > +import org.apache.lucene.search.FieldCache;
> > +import org.apache.lucene.search.IndexSearcher;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.Scorer;
> > +import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.params.ModifiableSolrParams;
> > +import org.apache.solr.common.params.SolrParams;
> > +import org.apache.solr.common.util.NamedList;
> > +import org.apache.solr.handler.component.QueryElevationComponent;
> > import org.apache.solr.request.LocalSolrQueryRequest;
> > +import org.apache.solr.request.SolrQueryRequest;
> > import org.apache.solr.request.SolrRequestInfo;
> > -import org.apache.solr.schema.TrieFloatField;
> > -import org.apache.solr.schema.TrieIntField;
> > -import org.apache.solr.schema.TrieLongField;
> > import org.apache.solr.schema.FieldType;
> > -import org.apache.solr.handler.component.QueryElevationComponent;
> > -import org.apache.lucene.index.AtomicReaderContext;
> > -import org.apache.lucene.index.AtomicReader;
> > -import org.apache.lucene.index.SortedDocValues;
> > -import org.apache.lucene.index.Terms;
> > -import org.apache.lucene.index.TermsEnum;
> > -import org.apache.lucene.index.DocsEnum;
> > -import org.apache.lucene.search.*;
> > -import org.apache.lucene.util.OpenBitSet;
> > -import org.apache.lucene.util.Bits;
> > -import org.apache.solr.common.params.SolrParams;
> > -import org.apache.solr.request.SolrQueryRequest;
> > -import org.apache.solr.common.util.NamedList;
> > import org.apache.solr.schema.IndexSchema;
> > import org.apache.solr.schema.SchemaField;
> > +import org.apache.solr.schema.TrieFloatField;
> > +import org.apache.solr.schema.TrieIntField;
> > +import org.apache.solr.schema.TrieLongField;
> >
> > import com.carrotsearch.hppc.FloatArrayList;
> > import com.carrotsearch.hppc.IntOpenHashSet;
> > import com.carrotsearch.hppc.cursors.IntCursor;
> >
> > -import java.io.IOException;
> > -import java.util.Arrays;
> > -import java.util.Map;
> > -import java.util.Set;
> > -import java.util.HashSet;
> > -import java.util.List;
> > -import java.util.Iterator;
> > -
> > /**
> >
> > The <b>CollapsingQParserPlugin</b> is a PostFilter that performs field
> collapsing.
> > @@ -419,7 +423,7 @@ public class CollapsingQParserPlugin ext
> > private class CollapsingScoreCollector extends DelegatingCollector {
> >
> > private AtomicReaderContext[] contexts;
> > - private OpenBitSet collapsedSet;
> > + private FixedBitSet collapsedSet;
> > private SortedDocValues values;
> > private int[] ords;
> > private float[] scores;
> > @@ -438,14 +442,14 @@ public class CollapsingQParserPlugin ext
> > IntOpenHashSet boostDocs) {
> > this.maxDoc = maxDoc;
> > this.contexts = new AtomicReaderContext[segments];
> > - this.collapsedSet = new OpenBitSet(maxDoc);
> > + this.collapsedSet = new FixedBitSet(maxDoc);
> > this.boostDocs = boostDocs;
> > if(this.boostDocs != null) {
> > //Set the elevated docs now.
> > Iterator<IntCursor> it = this.boostDocs.iterator();
> > while(it.hasNext()) {
> > IntCursor cursor = it.next();
> > - this.collapsedSet.fastSet(cursor.value);
> > + this.collapsedSet.set(cursor.value);
> > }
> > }
> > this.values = values;
> > @@ -460,16 +464,19 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> >
> > + @Override
> > public boolean acceptsDocsOutOfOrder() {
> > //Documents must be sent in order to this collector.
> > return false;
> > }
> >
> > + @Override
> > public void setNextReader(AtomicReaderContext context) throws
> IOException {
> > this.contexts[context.ord] = context;
> > this.docBase = context.docBase;
> > }
> >
> > + @Override
> > public void collect(int docId) throws IOException {
> > int globalDoc = docId+this.docBase;
> > int ord = values.getOrd(globalDoc);
> > @@ -479,7 +486,7 @@ public class CollapsingQParserPlugin ext
> > ords[ord] = globalDoc;
> > scores[ord] = score;
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //The doc is elevated so score does not matter
> > //We just want to be sure it doesn't fall into the null policy
> > } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE)
> {
> > @@ -489,24 +496,25 @@ public class CollapsingQParserPlugin ext
> > nullDoc = globalDoc;
> > }
> > } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - collapsedSet.fastSet(globalDoc);
> > + collapsedSet.set(globalDoc);
> > nullScores.add(scorer.score());
> > }
> > }
> >
> > + @Override
> > public void finish() throws IOException {
> > if(contexts.length == 0) {
> > return;
> > }
> >
> > if(nullScore > 0) {
> > - this.collapsedSet.fastSet(nullDoc);
> > + this.collapsedSet.set(nullDoc);
> > }
> >
> > for(int i=0; i<ords.length; i++) {
> > int doc = ords[i];
> > if(doc > -1) {
> > - collapsedSet.fastSet(doc);
> > + collapsedSet.set(doc);
> > }
> > }
> >
> > @@ -677,7 +685,7 @@ public class CollapsingQParserPlugin ext
> > protected FloatArrayList nullScores;
> > protected float nullScore;
> > protected float[] scores;
> > - protected OpenBitSet collapsedSet;
> > + protected FixedBitSet collapsedSet;
> > protected IntOpenHashSet boostDocs;
> > protected int nullDoc = -1;
> > protected boolean needsScores;
> > @@ -697,26 +705,26 @@ public class CollapsingQParserPlugin ext
> > this.nullPolicy = nullPolicy;
> > this.max = max;
> > this.needsScores = needsScores;
> > - this.collapsedSet = new OpenBitSet(maxDoc);
> > + this.collapsedSet = new FixedBitSet(maxDoc);
> > this.boostDocs = boostDocs;
> > if(this.boostDocs != null) {
> > Iterator<IntCursor> it = boostDocs.iterator();
> > while(it.hasNext()) {
> > IntCursor cursor = it.next();
> > - this.collapsedSet.fastSet(cursor.value);
> > + this.collapsedSet.set(cursor.value);
> > }
> > }
> > }
> >
> > - public OpenBitSet getCollapsedSet() {
> > + public FixedBitSet getCollapsedSet() {
> > if(nullDoc > -1) {
> > - this.collapsedSet.fastSet(nullDoc);
> > + this.collapsedSet.set(nullDoc);
> > }
> >
> > for(int i=0; i<ords.length; i++) {
> > int doc = ords[i];
> > if(doc > -1) {
> > - collapsedSet.fastSet(doc);
> > + collapsedSet.set(doc);
> > }
> > }
> >
> > @@ -790,7 +798,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = scorer.score();
> > }
> > }
> > - } else if(this.collapsedSet.fastGet(globalDoc)) {
> > + } else if(this.collapsedSet.get(globalDoc)) {
> > // Elevated doc so do nothing.
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -801,7 +809,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(scorer.score());
> > }
> > @@ -858,7 +866,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = scorer.score();
> > }
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //Elevated doc so do nothing
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -869,7 +877,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(scorer.score());
> > }
> > @@ -927,7 +935,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = scorer.score();
> > }
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //Elevated doc so do nothing
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -938,7 +946,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(scorer.score());
> > }
> > @@ -1015,7 +1023,7 @@ public class CollapsingQParserPlugin ext
> > scores[ord] = score;
> > }
> > }
> > - } else if (this.collapsedSet.fastGet(globalDoc)) {
> > + } else if (this.collapsedSet.get(globalDoc)) {
> > //Elevated doc so do nothing
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
> > if(comp.test(val, nullVal)) {
> > @@ -1026,7 +1034,7 @@ public class CollapsingQParserPlugin ext
> > }
> > }
> > } else if(this.nullPolicy ==
> CollapsingPostFilter.NULL_POLICY_EXPAND) {
> > - this.collapsedSet.fastSet(globalDoc);
> > + this.collapsedSet.set(globalDoc);
> > if(needsScores) {
> > nullScores.add(score);
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSet.java
> Sun Feb 16 07:50:08 2014
> > @@ -18,7 +18,6 @@
> > package org.apache.solr.search;
> >
> > import org.apache.lucene.search.Filter;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.solr.common.SolrException;
> >
> > /**
> > @@ -29,7 +28,6 @@ import org.apache.solr.common.SolrExcept
> > * a cache and could be shared.
> > * </p>
> > *
> > - *
> > * @since solr 0.9
> > */
> > public interface DocSet /* extends Collection<Integer> */ {
> > @@ -78,16 +76,6 @@ public interface DocSet /* extends Colle
> > public DocIterator iterator();
> >
> > /**
> > - * Returns a BitSet view of the DocSet. Any changes to this BitSet
> <b>may</b>
> > - * be reflected in the DocSet, hence if the DocSet is shared or was
> returned from
> > - * a SolrIndexSearcher method, it's not safe to modify the BitSet.
> > - *
> > - * @return
> > - * An OpenBitSet with the bit number of every docid set in the set.
> > - */
> > - public OpenBitSet getBits();
> > -
> > - /**
> > * Returns the approximate amount of memory taken by this DocSet.
> > * This is only an approximation and doesn't take into account java
> object overhead.
> > *
> > @@ -145,10 +133,11 @@ public interface DocSet /* extends Colle
> > public Filter getTopFilter();
> >
> > /**
> > - * Takes the docs from this set and sets those bits on the target
> OpenBitSet.
> > - * The target should be sized large enough to accommodate all of the
> documents before calling this method.
> > + * Adds all the docs from this set to the target set. The target
> should be
> > + * sized large enough to accommodate all of the documents before
> calling this
> > + * method.
> > */
> > - public void setBitsOn(OpenBitSet target);
> > + public void addAllTo(DocSet target);
> >
> > public static DocSet EMPTY = new SortedIntDocSet(new int[0], 0);
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetBase.java
> Sun Feb 16 07:50:08 2014
> > @@ -18,18 +18,32 @@
> > package org.apache.solr.search;
> >
> > import org.apache.lucene.index.AtomicReader;
> > -import org.apache.solr.common.SolrException;
> > -import org.apache.lucene.util.Bits;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.search.BitsFilteredDocIdSet;
> > import org.apache.lucene.search.DocIdSet;
> > -import org.apache.lucene.search.Filter;
> > import org.apache.lucene.search.DocIdSetIterator;
> > -import org.apache.lucene.search.BitsFilteredDocIdSet;
> > -import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.util.Bits;
> > +import org.apache.lucene.util.FixedBitSet;
> > +import org.apache.solr.common.SolrException;
> >
> > /** A base class that may be usefull for implementing DocSets */
> > abstract class DocSetBase implements DocSet {
> >
> > + public static FixedBitSet toBitSet(DocSet set) {
> > + if (set instanceof DocSetBase) {
> > + return ((DocSetBase) set).getBits();
> > + } else {
> > + FixedBitSet bits = new FixedBitSet(64);
> > + for (DocIterator iter = set.iterator(); iter.hasNext();) {
> > + int nextDoc = iter.nextDoc();
> > + bits = FixedBitSet.ensureCapacity(bits, nextDoc);
> > + bits.set(nextDoc);
> > + }
> > + return bits;
> > + }
> > + }
> > +
> > // Not implemented efficiently... for testing purposes only
> > @Override
> > public boolean equals(Object obj) {
> > @@ -49,7 +63,7 @@ abstract class DocSetBase implements Doc
> > }
> >
> > // if (this.size() != other.size()) return false;
> > - return this.getBits().equals(other.getBits());
> > + return this.getBits().equals(toBitSet(other));
> > }
> >
> > /**
> > @@ -69,18 +83,20 @@ abstract class DocSetBase implements Doc
> > }
> >
> > /**
> > - * Inefficient base implementation.
> > - *
> > - * @see BitDocSet#getBits
> > + * Return a {@link FixedBitSet} with a bit set for every document in
> this
> > + * {@link DocSet}. The default implementation iterates on all docs
> and sets
> > + * the relevant bits. You should override if you can provide a more
> efficient
> > + * implementation.
> > */
> > - @Override
> > - public OpenBitSet getBits() {
> > - OpenBitSet bits = new OpenBitSet();
> > + protected FixedBitSet getBits() {
> > + FixedBitSet bits = new FixedBitSet(64);
> > for (DocIterator iter = iterator(); iter.hasNext();) {
> > - bits.set(iter.nextDoc());
> > + int nextDoc = iter.nextDoc();
> > + bits = FixedBitSet.ensureCapacity(bits, nextDoc);
> > + bits.set(nextDoc);
> > }
> > return bits;
> > - };
> > + }
> >
> > @Override
> > public DocSet intersection(DocSet other) {
> > @@ -91,8 +107,8 @@ abstract class DocSetBase implements Doc
> > }
> >
> > // Default... handle with bitsets.
> > - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> > - newbits.and(other.getBits());
> > + FixedBitSet newbits = getBits().clone();
> > + newbits.and(toBitSet(other));
> > return new BitDocSet(newbits);
> > }
> >
> > @@ -107,11 +123,11 @@ abstract class DocSetBase implements Doc
> > return intersectionSize(other) > 0;
> > }
> >
> > -
> > @Override
> > public DocSet union(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> > - newbits.or(other.getBits());
> > + FixedBitSet otherBits = toBitSet(other);
> > + FixedBitSet newbits = FixedBitSet.ensureCapacity(getBits().clone(),
> otherBits.length());
> > + newbits.or(otherBits);
> > return new BitDocSet(newbits);
> > }
> >
> > @@ -133,8 +149,8 @@ abstract class DocSetBase implements Doc
> >
> > @Override
> > public DocSet andNot(DocSet other) {
> > - OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
> > - newbits.andNot(other.getBits());
> > + FixedBitSet newbits = getBits().clone();
> > + newbits.andNot(toBitSet(other));
> > return new BitDocSet(newbits);
> > }
> >
> > @@ -145,7 +161,7 @@ abstract class DocSetBase implements Doc
> >
> > @Override
> > public Filter getTopFilter() {
> > - final OpenBitSet bs = getBits();
> > + final FixedBitSet bs = getBits();
> >
> > return new Filter() {
> > @Override
> > @@ -189,7 +205,7 @@ abstract class DocSetBase implements Doc
> >
> > @Override
> > public long cost() {
> > - return bs.capacity();
> > + return bs.length();
> > }
> > };
> > }
> > @@ -211,10 +227,10 @@ abstract class DocSetBase implements Doc
> > }
> >
> > @Override
> > - public void setBitsOn(OpenBitSet target) {
> > + public void addAllTo(DocSet target) {
> > DocIterator iter = iterator();
> > while (iter.hasNext()) {
> > - target.fastSet(iter.nextDoc());
> > + target.add(iter.nextDoc());
> > }
> > }
> >
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,12 @@ package org.apache.solr.search;
> > * limitations under the License.
> > */
> >
> > -import org.apache.lucene.index.IndexReader;
> > +import java.io.IOException;
> > +
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.Collector;
> > import org.apache.lucene.search.Scorer;
> > -import org.apache.lucene.util.OpenBitSet;
> > -
> > -import java.io.IOException;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > *
> > @@ -31,7 +30,7 @@ import java.io.IOException;
> >
> > public class DocSetCollector extends Collector {
> > int pos=0;
> > - OpenBitSet bits;
> > + FixedBitSet bits;
> > final int maxDoc;
> > final int smallSetSize;
> > int base;
> > @@ -62,8 +61,8 @@ public class DocSetCollector extends Col
> > } else {
> > // this conditional could be removed if BitSet was preallocated,
> but that
> > // would take up more memory, and add more GC time...
> > - if (bits==null) bits = new OpenBitSet(maxDoc);
> > - bits.fastSet(doc);
> > + if (bits==null) bits = new FixedBitSet(maxDoc);
> > + bits.set(doc);
> > }
> >
> > pos++;
> > @@ -75,7 +74,7 @@ public class DocSetCollector extends Col
> > return new SortedIntDocSet(scratch, pos);
> > } else {
> > // set the bits for ids that were collected in the array
> > - for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
> > + for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
> > return new BitDocSet(bits,pos);
> > }
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,12 @@ package org.apache.solr.search;
> > * limitations under the License.
> > */
> >
> > -import org.apache.lucene.index.IndexReader;
> > +import java.io.IOException;
> > +
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.Collector;
> > import org.apache.lucene.search.Scorer;
> > -import org.apache.lucene.util.OpenBitSet;
> > -
> > -import java.io.IOException;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > *
> > @@ -53,8 +52,8 @@ public class DocSetDelegateCollector ext
> > } else {
> > // this conditional could be removed if BitSet was preallocated,
> but that
> > // would take up more memory, and add more GC time...
> > - if (bits==null) bits = new OpenBitSet(maxDoc);
> > - bits.fastSet(doc);
> > + if (bits==null) bits = new FixedBitSet(maxDoc);
> > + bits.set(doc);
> > }
> >
> > pos++;
> > @@ -67,7 +66,7 @@ public class DocSetDelegateCollector ext
> > return new SortedIntDocSet(scratch, pos);
> > } else {
> > // set the bits for ids that were collected in the array
> > - for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
> > + for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
> > return new BitDocSet(bits,pos);
> > }
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/Grouping.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,38 @@
> >
> > package org.apache.solr.search;
> >
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.Collection;
> > +import java.util.LinkedHashSet;
> > +import java.util.List;
> > +import java.util.Locale;
> > +import java.util.Map;
> > +import java.util.Set;
> > +
> > import org.apache.commons.lang.ArrayUtils;
> > import org.apache.lucene.index.IndexableField;
> > import org.apache.lucene.queries.function.FunctionQuery;
> > import org.apache.lucene.queries.function.ValueSource;
> > import org.apache.lucene.queries.function.valuesource.QueryValueSource;
> > -import org.apache.lucene.search.*;
> > -import org.apache.lucene.search.grouping.*;
> > +import org.apache.lucene.search.CachingCollector;
> > +import org.apache.lucene.search.Collector;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.search.MultiCollector;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.ScoreDoc;
> > +import org.apache.lucene.search.Sort;
> > +import org.apache.lucene.search.SortField;
> > +import org.apache.lucene.search.TimeLimitingCollector;
> > +import org.apache.lucene.search.TopDocs;
> > +import org.apache.lucene.search.TopDocsCollector;
> > +import org.apache.lucene.search.TopFieldCollector;
> > +import org.apache.lucene.search.TopScoreDocCollector;
> > +import org.apache.lucene.search.TotalHitCountCollector;
> > +import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
> > +import org.apache.lucene.search.grouping.GroupDocs;
> > +import org.apache.lucene.search.grouping.SearchGroup;
> > +import org.apache.lucene.search.grouping.TopGroups;
> > import
> org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
> > import
> org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector;
> > import
> org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector;
> > @@ -33,8 +58,6 @@ import org.apache.lucene.search.grouping
> > import
> org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector;
> > import
> org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector;
> > import org.apache.lucene.util.BytesRef;
> > -import org.apache.lucene.util.FixedBitSet;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.lucene.util.mutable.MutableValue;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.util.NamedList;
> > @@ -47,9 +70,6 @@ import org.apache.solr.search.grouping.c
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.util.*;
> > -
> > /**
> > * Basic Solr Grouping infrastructure.
> > * Warning NOT thread save!
> > @@ -353,10 +373,7 @@ public class Grouping {
> > }
> >
> > if (getGroupedDocSet && allGroupHeadsCollector != null) {
> > - FixedBitSet fixedBitSet =
> allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
> > - long[] bits = fixedBitSet.getBits();
> > - OpenBitSet openBitSet = new OpenBitSet(bits, bits.length);
> > - qr.setDocSet(new BitDocSet(openBitSet));
> > + qr.setDocSet(new
> BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(maxDoc)));
> > } else if (getDocSet) {
> > qr.setDocSet(setCollector.getDocSet());
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
> Sun Feb 16 07:50:08 2014
> > @@ -16,12 +16,33 @@
> > */
> > package org.apache.solr.search;
> >
> > -import org.apache.lucene.index.*;
> > -import org.apache.lucene.search.*;
> > +import java.io.Closeable;
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.Arrays;
> > +import java.util.List;
> > +import java.util.Set;
> > +
> > +import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.index.DocsEnum;
> > +import org.apache.lucene.index.Fields;
> > +import org.apache.lucene.index.IndexReader;
> > +import org.apache.lucene.index.MultiDocsEnum;
> > +import org.apache.lucene.index.Terms;
> > +import org.apache.lucene.index.TermsEnum;
> > +import org.apache.lucene.search.ComplexExplanation;
> > +import org.apache.lucene.search.DocIdSet;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > +import org.apache.lucene.search.Explanation;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.search.IndexSearcher;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.Scorer;
> > +import org.apache.lucene.search.Weight;
> > import org.apache.lucene.search.similarities.Similarity;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.lucene.util.StringHelper;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.params.SolrParams;
> > @@ -36,13 +57,6 @@ import org.apache.solr.request.SolrReque
> > import org.apache.solr.schema.TrieField;
> > import org.apache.solr.util.RefCounted;
> >
> > -import java.io.Closeable;
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.Arrays;
> > -import java.util.List;
> > -import java.util.Set;
> > -
> >
> > public class JoinQParserPlugin extends QParserPlugin {
> > public static final String NAME = "join";
> > @@ -270,7 +284,7 @@ class JoinQuery extends Query {
> >
> >
> > public DocSet getDocSet() throws IOException {
> > - OpenBitSet resultBits = null;
> > + FixedBitSet resultBits = null;
> >
> > // minimum docFreq to use the cache
> > int minDocFreqFrom = Math.max(5, fromSearcher.maxDoc() >> 13);
> > @@ -387,7 +401,7 @@ class JoinQuery extends Query {
> > int df = toTermsEnum.docFreq();
> > toTermHitsTotalDf += df;
> > if (resultBits==null && df + resultListDocs >
> maxSortedIntSize && resultList.size() > 0) {
> > - resultBits = new OpenBitSet(toSearcher.maxDoc());
> > + resultBits = new FixedBitSet(toSearcher.maxDoc());
> > }
> >
> > // if we don't have a bitset yet, or if the resulting set
> will be too large
> > @@ -397,10 +411,10 @@ class JoinQuery extends Query {
> > DocSet toTermSet = toSearcher.getDocSet(toDeState);
> > resultListDocs += toTermSet.size();
> > if (resultBits != null) {
> > - toTermSet.setBitsOn(resultBits);
> > + toTermSet.addAllTo(new BitDocSet(resultBits));
> > } else {
> > if (toTermSet instanceof BitDocSet) {
> > - resultBits =
> (OpenBitSet)((BitDocSet)toTermSet).bits.clone();
> > + resultBits = ((BitDocSet)toTermSet).bits.clone();
> > } else {
> > resultList.add(toTermSet);
> > }
> > @@ -422,14 +436,14 @@ class JoinQuery extends Query {
> > int docid;
> > while ((docid = sub.docsEnum.nextDoc()) !=
> DocIdSetIterator.NO_MORE_DOCS) {
> > resultListDocs++;
> > - resultBits.fastSet(docid + base);
> > + resultBits.set(docid + base);
> > }
> > }
> > } else {
> > int docid;
> > while ((docid = docsEnum.nextDoc()) !=
> DocIdSetIterator.NO_MORE_DOCS) {
> > resultListDocs++;
> > - resultBits.fastSet(docid);
> > + resultBits.set(docid);
> > }
> > }
> > }
> > @@ -443,10 +457,11 @@ class JoinQuery extends Query {
> > smallSetsDeferred = resultList.size();
> >
> > if (resultBits != null) {
> > + BitDocSet bitSet = new BitDocSet(resultBits);
> > for (DocSet set : resultList) {
> > - set.setBitsOn(resultBits);
> > + set.addAllTo(bitSet);
> > }
> > - return new BitDocSet(resultBits);
> > + return bitSet;
> > }
> >
> > if (resultList.size()==0) {
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
> Sun Feb 16 07:50:08 2014
> > @@ -65,11 +65,11 @@ import org.apache.lucene.search.Constant
> > import org.apache.lucene.search.DocIdSet;
> > import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.search.Explanation;
> > +import org.apache.lucene.search.FieldDoc;
> > import org.apache.lucene.search.Filter;
> > import org.apache.lucene.search.IndexSearcher;
> > import org.apache.lucene.search.MatchAllDocsQuery;
> > import org.apache.lucene.search.Query;
> > -import org.apache.lucene.search.FieldDoc;
> > import org.apache.lucene.search.ScoreDoc;
> > import org.apache.lucene.search.Scorer;
> > import org.apache.lucene.search.Sort;
> > @@ -77,16 +77,16 @@ import org.apache.lucene.search.SortFiel
> > import org.apache.lucene.search.TermQuery;
> > import org.apache.lucene.search.TimeLimitingCollector;
> > import org.apache.lucene.search.TopDocs;
> > -import org.apache.lucene.search.TopFieldDocs;
> > import org.apache.lucene.search.TopDocsCollector;
> > import org.apache.lucene.search.TopFieldCollector;
> > +import org.apache.lucene.search.TopFieldDocs;
> > import org.apache.lucene.search.TopScoreDocCollector;
> > import org.apache.lucene.search.TotalHitCountCollector;
> > import org.apache.lucene.search.Weight;
> > import org.apache.lucene.store.Directory;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.SolrException;
> > import org.apache.solr.common.SolrException.ErrorCode;
> > import org.apache.solr.common.params.ModifiableSolrParams;
> > @@ -104,7 +104,6 @@ import org.apache.solr.request.UnInverte
> > import org.apache.solr.response.SolrQueryResponse;
> > import org.apache.solr.schema.IndexSchema;
> > import org.apache.solr.schema.SchemaField;
> > -import org.apache.solr.spelling.QueryConverter;
> > import org.apache.solr.update.SolrIndexConfig;
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> > @@ -1087,7 +1086,7 @@ public class SolrIndexSearcher extends I
> > final int[] docs = deState.scratch;
> > int upto = 0;
> > int bitsSet = 0;
> > - OpenBitSet obs = null;
> > + FixedBitSet fbs = null;
> >
> > DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs,
> deState.docsEnum, DocsEnum.FLAG_NONE);
> > if (deState.docsEnum == null) {
> > @@ -1104,9 +1103,9 @@ public class SolrIndexSearcher extends I
> > int docid;
> >
> > if (largestPossible > docs.length) {
> > - if (obs == null) obs = new OpenBitSet(maxDoc());
> > + if (fbs == null) fbs = new FixedBitSet(maxDoc());
> > while ((docid = sub.docsEnum.nextDoc()) !=
> DocIdSetIterator.NO_MORE_DOCS) {
> > - obs.fastSet(docid + base);
> > + fbs.set(docid + base);
> > bitsSet++;
> > }
> > } else {
> > @@ -1118,9 +1117,9 @@ public class SolrIndexSearcher extends I
> > } else {
> > int docid;
> > if (largestPossible > docs.length) {
> > - if (obs == null) obs = new OpenBitSet(maxDoc());
> > + if (fbs == null) fbs = new FixedBitSet(maxDoc());
> > while ((docid = docsEnum.nextDoc()) !=
> DocIdSetIterator.NO_MORE_DOCS) {
> > - obs.fastSet(docid);
> > + fbs.set(docid);
> > bitsSet++;
> > }
> > } else {
> > @@ -1131,12 +1130,12 @@ public class SolrIndexSearcher extends I
> > }
> >
> > DocSet result;
> > - if (obs != null) {
> > + if (fbs != null) {
> > for (int i=0; i<upto; i++) {
> > - obs.fastSet(docs[i]);
> > + fbs.set(docs[i]);
> > }
> > bitsSet += upto;
> > - result = new BitDocSet(obs, bitsSet);
> > + result = new BitDocSet(fbs, bitsSet);
> > } else {
> > result = upto==0 ? DocSet.EMPTY : new
> SortedIntDocSet(Arrays.copyOf(docs, upto));
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
> Sun Feb 16 07:50:08 2014
> > @@ -18,13 +18,13 @@
> > package org.apache.solr.search;
> >
> > import org.apache.lucene.index.AtomicReader;
> > -import org.apache.lucene.util.Bits;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.search.BitsFilteredDocIdSet;
> > import org.apache.lucene.search.DocIdSet;
> > import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.search.Filter;
> > -import org.apache.lucene.index.AtomicReaderContext;
> > +import org.apache.lucene.util.Bits;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > * <code>SortedIntDocSet</code> represents a sorted set of Lucene
> Document Ids.
> > @@ -563,13 +563,12 @@ public class SortedIntDocSet extends Doc
> > }
> >
> > @Override
> > - public void setBitsOn(OpenBitSet target) {
> > + public void addAllTo(DocSet target) {
> > for (int doc : docs) {
> > - target.fastSet(doc);
> > + target.add(doc);
> > }
> > }
> >
> > -
> > @Override
> > public boolean exists(int doc) {
> > // this could be faster by estimating where in the list the doc is
> likely to appear,
> > @@ -630,16 +629,15 @@ public class SortedIntDocSet extends Doc
> > }
> >
> > @Override
> > - public OpenBitSet getBits() {
> > + public FixedBitSet getBits() {
> > int maxDoc = size() > 0 ? docs[size()-1] : 0;
> > - OpenBitSet bs = new OpenBitSet(maxDoc+1);
> > + FixedBitSet bs = new FixedBitSet(maxDoc+1);
> > for (int doc : docs) {
> > - bs.fastSet(doc);
> > + bs.set(doc);
> > }
> > return bs;
> > }
> >
> > -
> > public static int findIndex(int[] arr, int value, int low, int high) {
> > // binary search
> > while (low <= high) {
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,22 +17,31 @@ package org.apache.solr.search.grouping;
> > * limitations under the License.
> > */
> >
> > -import org.apache.lucene.search.*;
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.List;
> > +
> > +import org.apache.lucene.search.Collector;
> > +import org.apache.lucene.search.Filter;
> > +import org.apache.lucene.search.MultiCollector;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.TimeLimitingCollector;
> > +import org.apache.lucene.search.TotalHitCountCollector;
> > import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
> > import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.solr.common.util.NamedList;
> > -import org.apache.solr.search.*;
> > -import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
> > +import org.apache.solr.search.BitDocSet;
> > +import org.apache.solr.search.DocSet;
> > +import org.apache.solr.search.DocSetCollector;
> > +import org.apache.solr.search.DocSetDelegateCollector;
> > import org.apache.solr.search.QueryUtils;
> > +import org.apache.solr.search.SolrIndexSearcher;
> > +import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
> > import
> org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.List;
> > -
> > /**
> > * Responsible for executing a search with a number of {@link Command}
> instances.
> > * A typical search can have more then one {@link Command} instances.
> > @@ -159,9 +168,7 @@ public class CommandHandler {
> > searchWithTimeLimiter(query, filter,
> MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])));
> > }
> >
> > - int maxDoc = searcher.maxDoc();
> > - long[] bits =
> termAllGroupHeadsCollector.retrieveGroupHeads(maxDoc).getBits();
> > - return new BitDocSet(new OpenBitSet(bits, bits.length));
> > + return new
> BitDocSet(termAllGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
> > }
> >
> > private DocSet computeDocSet(Query query, ProcessedFilter filter,
> List<Collector> collectors) throws IOException {
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,30 +17,30 @@
> >
> > package org.apache.solr.search.join;
> >
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > class BitSetSlice {
> > - private final OpenBitSet obs;
> > + private final FixedBitSet fbs;
> > private final int off;
> > private final int len;
> >
> > - BitSetSlice(OpenBitSet obs, int off, int len) {
> > - this.obs = obs;
> > + BitSetSlice(FixedBitSet fbs, int off, int len) {
> > + this.fbs = fbs;
> > this.off = off;
> > this.len = len;
> > }
> >
> > public boolean get(int pos) {
> > - return obs.get(pos + off);
> > + return fbs.get(pos + off);
> > }
> >
> > public int prevSetBit(int pos) {
> > - int result = obs.prevSetBit(pos + off) - off;
> > + int result = fbs.prevSetBit(pos + off) - off;
> > return (result < 0) ? -1 : result;
> > }
> >
> > public int nextSetBit(int pos) {
> > - int result = obs.nextSetBit(pos + off) - off;
> > + int result = fbs.nextSetBit(pos + off) - off;
> > return (result < 0 || result >= len) ? -1 : result;
> > }
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
> Sun Feb 16 07:50:08 2014
> > @@ -19,7 +19,7 @@ package org.apache.solr.store.blockcache
> >
> > import java.util.concurrent.atomic.AtomicLongArray;
> >
> > -import org.apache.lucene.util.OpenBitSet;
> > +import org.apache.lucene.util.LongBitSet;
> >
> > public class BlockLocks {
> >
> > @@ -27,7 +27,7 @@ public class BlockLocks {
> > private int wlen;
> >
> > public BlockLocks(long numBits) {
> > - int length = OpenBitSet.bits2words(numBits);
> > + int length = LongBitSet.bits2words(numBits);
> > bits = new AtomicLongArray(length);
> > wlen = length;
> > }
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,6 +17,10 @@
> >
> > package org.apache.solr.update;
> >
> > +import java.io.IOException;
> > +import java.util.ArrayList;
> > +import java.util.List;
> > +
> > import org.apache.lucene.index.AtomicReader;
> > import org.apache.lucene.index.AtomicReaderContext;
> > import org.apache.lucene.index.DocsEnum;
> > @@ -26,11 +30,12 @@ import org.apache.lucene.index.IndexRead
> > import org.apache.lucene.index.IndexWriter;
> > import org.apache.lucene.index.Terms;
> > import org.apache.lucene.index.TermsEnum;
> > +import org.apache.lucene.search.DocIdSetIterator;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.CharsRef;
> > +import org.apache.lucene.util.FixedBitSet;
> > import org.apache.lucene.util.IOUtils;
> > -import org.apache.lucene.util.OpenBitSet;
> > import org.apache.solr.common.cloud.CompositeIdRouter;
> > import org.apache.solr.common.cloud.DocRouter;
> > import org.apache.solr.common.cloud.HashBasedRouter;
> > @@ -41,10 +46,6 @@ import org.apache.solr.util.RefCounted;
> > import org.slf4j.Logger;
> > import org.slf4j.LoggerFactory;
> >
> > -import java.io.IOException;
> > -import java.util.ArrayList;
> > -import java.util.List;
> > -
> > public class SolrIndexSplitter {
> > public static Logger log =
> LoggerFactory.getLogger(SolrIndexSplitter.class);
> >
> > @@ -89,13 +90,13 @@ public class SolrIndexSplitter {
> > public void split() throws IOException {
> >
> > List<AtomicReaderContext> leaves =
> searcher.getTopReaderContext().leaves();
> > - List<OpenBitSet[]> segmentDocSets = new
> ArrayList<OpenBitSet[]>(leaves.size());
> > + List<FixedBitSet[]> segmentDocSets = new
> ArrayList<FixedBitSet[]>(leaves.size());
> >
> > log.info("SolrIndexSplitter: partitions=" + numPieces + "
> segments="+leaves.size());
> >
> > for (AtomicReaderContext readerContext : leaves) {
> > assert readerContext.ordInParent == segmentDocSets.size(); //
> make sure we're going in order
> > - OpenBitSet[] docSets = split(readerContext);
> > + FixedBitSet[] docSets = split(readerContext);
> > segmentDocSets.add( docSets );
> > }
> >
> > @@ -150,11 +151,11 @@ public class SolrIndexSplitter {
> >
> >
> >
> > - OpenBitSet[] split(AtomicReaderContext readerContext) throws
> IOException {
> > + FixedBitSet[] split(AtomicReaderContext readerContext) throws
> IOException {
> > AtomicReader reader = readerContext.reader();
> > - OpenBitSet[] docSets = new OpenBitSet[numPieces];
> > + FixedBitSet[] docSets = new FixedBitSet[numPieces];
> > for (int i=0; i<docSets.length; i++) {
> > - docSets[i] = new OpenBitSet(reader.maxDoc());
> > + docSets[i] = new FixedBitSet(reader.maxDoc());
> > }
> > Bits liveDocs = reader.getLiveDocs();
> >
> > @@ -196,14 +197,14 @@ public class SolrIndexSplitter {
> > docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
> > for (;;) {
> > int doc = docsEnum.nextDoc();
> > - if (doc == DocsEnum.NO_MORE_DOCS) break;
> > + if (doc == DocIdSetIterator.NO_MORE_DOCS) break;
> > if (ranges == null) {
> > - docSets[currPartition].fastSet(doc);
> > + docSets[currPartition].set(doc);
> > currPartition = (currPartition + 1) % numPieces;
> > } else {
> > for (int i=0; i<rangesArr.length; i++) { // inner-loop:
> use array here for extra speed.
> > if (rangesArr[i].includes(hash)) {
> > - docSets[i].fastSet(doc);
> > + docSets[i].set(doc);
> > }
> > }
> > }
> > @@ -232,13 +233,13 @@ public class SolrIndexSplitter {
> >
> > // change livedocs on the reader to delete those docs we don't want
> > static class LiveDocsReader extends FilterAtomicReader {
> > - final OpenBitSet liveDocs;
> > + final FixedBitSet liveDocs;
> > final int numDocs;
> >
> > - public LiveDocsReader(AtomicReaderContext context, OpenBitSet
> liveDocs) throws IOException {
> > + public LiveDocsReader(AtomicReaderContext context, FixedBitSet
> liveDocs) throws IOException {
> > super(context.reader());
> > this.liveDocs = liveDocs;
> > - this.numDocs = (int)liveDocs.cardinality();
> > + this.numDocs = liveDocs.cardinality();
> > }
> >
> > @Override
> > @@ -253,6 +254,3 @@ public class SolrIndexSplitter {
> > }
> >
> > }
> > -
> > -
> > -
> >
> > Modified:
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java?rev=1568738&r1=1568737&r2=1568738&view=diff
> >
> ==============================================================================
> > ---
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> (original)
> > +++
> lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
> Sun Feb 16 07:50:08 2014
> > @@ -17,13 +17,10 @@
> >
> > package org.apache.solr.search;
> >
> > -import org.apache.solr.search.BitDocSet;
> > -import org.apache.solr.search.HashDocSet;
> > -import org.apache.solr.search.DocSet;
> > -import org.apache.lucene.util.OpenBitSet;
> > -
> > -import java.util.Random;
> > import java.util.BitSet;
> > +import java.util.Random;
> > +
> > +import org.apache.lucene.util.FixedBitSet;
> >
> > /**
> > */
> > @@ -38,21 +35,20 @@ public class DocSetPerf {
> >
> > static Random rand = new Random();
> >
> > -
> > - static OpenBitSet bs;
> > + static FixedBitSet bs;
> > static BitDocSet bds;
> > static HashDocSet hds;
> > static int[] ids; // not unique
> >
> > static void generate(int maxSize, int bitsToSet) {
> > - bs = new OpenBitSet(maxSize);
> > + bs = new FixedBitSet(maxSize);
> > ids = new int[bitsToSet];
> > int count=0;
> > if (maxSize>0) {
> > for (int i=0; i<bitsToSet; i++) {
> > int id=rand.nextInt(maxSize);
> > if (!bs.get(id)) {
> > - bs.fastSet(id);
> > + bs.set(id);
> > ids[count++]=id;
> > }
> > }
> > @@ -80,7 +76,7 @@ public class DocSetPerf {
> >
> > int ret=0;
> >
> > - OpenBitSet[] sets = new OpenBitSet[numSets];
> > + FixedBitSet[] sets = new FixedBitSet[numSets];
> > DocSet[] bset = new DocSet[numSets];
> > DocSet[] hset = new DocSet[numSets];
> > BitSet scratch=new BitSet();
> > @@ -97,14 +93,14 @@ public class DocSetPerf {
> > if ("test".equals(test)) {
> > for (int it=0; it<iter; it++) {
> > generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize,
> numBitsSet);
> > - OpenBitSet bs1=bs;
> > + FixedBitSet bs1=bs;
> > BitDocSet bds1=bds;
> > HashDocSet hds1=hds;
> > generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize,
> numBitsSet);
> >
> > - OpenBitSet res = ((OpenBitSet)bs1.clone());
> > + FixedBitSet res = bs1.clone();
> > res.and(bs);
> > - int icount = (int)res.cardinality();
> > + int icount = res.cardinality();
> >
> > test(bds1.intersection(bds).size() == icount);
> > test(bds1.intersectionSize(bds) == icount);
> > @@ -167,8 +163,6 @@ public class DocSetPerf {
> > }
> > }
> >
> > -
> > -
> > long end = System.currentTimeMillis();
> > System.out.println("TIME="+(end-start));
> >
> > @@ -176,6 +170,4 @@ public class DocSetPerf {
> > System.out.println("ret="+ret);
> > }
> >
> > -
> > -
> > }
> >
> >
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>