You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by is...@apache.org on 2024/02/05 19:47:06 UTC
(solr) branch jira/solr-13350 created (now 74c05e520ab)
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a change to branch jira/solr-13350
in repository https://gitbox.apache.org/repos/asf/solr.git
at 74c05e520ab SOLR-13350: Fix tests and precommit
This branch includes the following new commits:
new 3cee3e59199 SOLR-13350: Multithreaded search using CollectorManager
new 74c05e520ab SOLR-13350: Fix tests and precommit
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(solr) 02/02: SOLR-13350: Fix tests and precommit
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-13350
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 74c05e520abe85f056fe440976dbbd6a812b6386
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Tue May 23 03:21:59 2023 +0530
SOLR-13350: Fix tests and precommit
---
.../java/org/apache/solr/core/CoreContainer.java | 13 +-
.../java/org/apache/solr/search/QueryResult.java | 2 +-
.../org/apache/solr/search/SolrIndexSearcher.java | 344 +++++-----
.../solr/search/SolrMultiCollectorManager.java | 27 +-
.../org/apache/solr/search/ThreadSafeBitSet.java | 714 +++++++++++----------
.../solr/search/ThreadSafeBitSetCollector.java | 31 +-
.../org/apache/solr/TestDistributedSearch.java | 2 +-
.../test/org/apache/solr/search/TestFiltering.java | 21 +-
8 files changed, 603 insertions(+), 551 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 012c31633b8..9146177e2b1 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -177,11 +177,11 @@ public class CoreContainer {
final SolrCores solrCores;
- public Executor getCollectorExecutor() {
- return collectorExecutor;
- }
+ public Executor getCollectorExecutor() {
+ return collectorExecutor;
+ }
- public static class CoreLoadFailure {
+ public static class CoreLoadFailure {
public final CoreDescriptor cd;
public final Exception exception;
@@ -440,8 +440,9 @@ public class CoreContainer {
this.allowListUrlChecker = AllowListUrlChecker.create(config);
- this.collectorExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(6,
- new SolrNamedThreadFactory("searcherCollector"));
+ this.collectorExecutor =
+ ExecutorUtil.newMDCAwareCachedThreadPool(
+ 6, new SolrNamedThreadFactory("searcherCollector"));
}
@SuppressWarnings({"unchecked"})
diff --git a/solr/core/src/java/org/apache/solr/search/QueryResult.java b/solr/core/src/java/org/apache/solr/search/QueryResult.java
index 5d4500a4878..25399cb336c 100755
--- a/solr/core/src/java/org/apache/solr/search/QueryResult.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryResult.java
@@ -46,7 +46,7 @@ public class QueryResult {
docListAndSet = new DocListAndSet();
}
docListAndSet.docSet = set;
- // log.error("set docset {}", docListAndSet.docSet.getBits().length());
+ // log.error("set docset {}", docListAndSet.docSet.getBits().length());
}
public boolean isPartialResults() {
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index a8c11016848..23df0a8b296 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -1873,46 +1873,53 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
qr.setNextCursorMark(cmd.getCursorMark());
hitsRelation = Relation.EQUAL_TO;
} else {
- log.info("calling from 2, query: "+query.getClass()); // nocommit
+ if (log.isInfoEnabled()) {
+ log.info("calling from 2, query: {}", query.getClass());
+ }
MTCollectorQueryCheck allowMT = new MTCollectorQueryCheck();
query.visit(allowMT);
TopDocs topDocs;
- if (pf.postFilter != null || cmd.getSegmentTerminateEarly() || cmd.getTimeAllowed() > 0
- || !allowMT.allowed()) {
- log.debug("skipping collector manager");
- final TopDocsCollector<?> topCollector = buildTopDocsCollector(len, cmd);
- MaxScoreCollector maxScoreCollector = null;
- Collector collector = topCollector;
- if (needScores) {
- maxScoreCollector = new MaxScoreCollector();
- collector = MultiCollector.wrap(topCollector, maxScoreCollector);
- }
- ScoreMode scoreModeUsed =
- buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter).scoreMode();
+ if (pf.postFilter != null
+ || cmd.getSegmentTerminateEarly()
+ || cmd.getTimeAllowed() > 0
+ || !allowMT.allowed()) {
+ if (log.isInfoEnabled()) {
+ log.info("skipping collector manager");
+ }
+ final TopDocsCollector<?> topCollector = buildTopDocsCollector(len, cmd);
+ MaxScoreCollector maxScoreCollector = null;
+ Collector collector = topCollector;
+ if (needScores) {
+ maxScoreCollector = new MaxScoreCollector();
+ collector = MultiCollector.wrap(topCollector, maxScoreCollector);
+ }
+ ScoreMode scoreModeUsed =
+ buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter).scoreMode();
- totalHits = topCollector.getTotalHits();
- topDocs = topCollector.topDocs(0, len);
- if (scoreModeUsed == ScoreMode.COMPLETE || scoreModeUsed == ScoreMode.COMPLETE_NO_SCORES) {
- hitsRelation = TotalHits.Relation.EQUAL_TO;
- } else {
- hitsRelation = topDocs.totalHits.relation;
- }
- if (cmd.getSort() != null
- && cmd.getQuery() instanceof RankQuery == false
- && needScores) {
- TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
- }
- populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
+ totalHits = topCollector.getTotalHits();
+ topDocs = topCollector.topDocs(0, len);
+ if (scoreModeUsed == ScoreMode.COMPLETE || scoreModeUsed == ScoreMode.COMPLETE_NO_SCORES) {
+ hitsRelation = TotalHits.Relation.EQUAL_TO;
+ } else {
+ hitsRelation = topDocs.totalHits.relation;
+ }
+ if (cmd.getSort() != null && cmd.getQuery() instanceof RankQuery == false && needScores) {
+ TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
+ }
+ populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
- maxScore =
- totalHits > 0
- ? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore())
- : 0.0f;
- nDocsReturned = topDocs.scoreDocs.length;
+ maxScore =
+ totalHits > 0
+ ? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore())
+ : 0.0f;
+ nDocsReturned = topDocs.scoreDocs.length;
} else {
- log.info("using CollectorManager");
- SearchResult searchResult = searchCollectorManagers(len, cmd, query, true, needScores, false);
+ if (log.isInfoEnabled()) {
+ log.info("using CollectorManager");
+ }
+ SearchResult searchResult =
+ searchCollectorManagers(len, cmd, query, true, needScores, false);
Object[] res = searchResult.result;
TopDocsResult result = (TopDocsResult) res[0];
@@ -1937,7 +1944,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
} else {
hitsRelation = topDocs.totalHits.relation;
}
-
}
ids = new int[nDocsReturned];
@@ -1954,8 +1960,14 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
qr.setDocList(new DocSlice(0, sliceLen, ids, scores, totalHits, maxScore, hitsRelation));
}
- SearchResult searchCollectorManagers(int len, QueryCommand cmd, Query query,
- boolean needTopDocs, boolean needMaxScore, boolean needDocSet) throws IOException {
+ SearchResult searchCollectorManagers(
+ int len,
+ QueryCommand cmd,
+ Query query,
+ boolean needTopDocs,
+ boolean needMaxScore,
+ boolean needDocSet)
+ throws IOException {
Collection<CollectorManager<Collector, Object>> collectors = new ArrayList<>();
ScoreMode scoreMode = null;
@@ -1963,75 +1975,80 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
if (needTopDocs) {
- collectors.add(new CollectorManager<>() {
- @Override
- public Collector newCollector() throws IOException {
- @SuppressWarnings("rawtypes")
- TopDocsCollector collector = buildTopDocsCollector(len, cmd);
- if (firstCollectors[0] == null) {
- firstCollectors[0] = collector;
- }
- return collector;
- }
+ collectors.add(
+ new CollectorManager<>() {
+ @Override
+ public Collector newCollector() throws IOException {
+ @SuppressWarnings("rawtypes")
+ TopDocsCollector collector = buildTopDocsCollector(len, cmd);
+ if (firstCollectors[0] == null) {
+ firstCollectors[0] = collector;
+ }
+ return collector;
+ }
- @Override
- @SuppressWarnings("rawtypes")
- public Object reduce(Collection collectors) throws IOException {
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Object reduce(Collection collectors) throws IOException {
- TopDocs[] topDocs = new TopDocs[collectors.size()];
+ TopDocs[] topDocs = new TopDocs[collectors.size()];
- int totalHits = -1;
- int i = 0;
+ int totalHits = -1;
+ int i = 0;
- Collector collector;
- for (Object o : collectors) {
- collector = (Collector) o;
- if (collector instanceof TopDocsCollector) {
- TopDocs td = ((TopDocsCollector) collector).topDocs(0, len);
- assert td != null : Arrays.asList(topDocs);
- topDocs[i++] = td;
- }
- }
+ Collector collector;
+ for (Object o : collectors) {
+ collector = (Collector) o;
+ if (collector instanceof TopDocsCollector) {
+ TopDocs td = ((TopDocsCollector) collector).topDocs(0, len);
+ assert td != null : Arrays.asList(topDocs);
+ topDocs[i++] = td;
+ }
+ }
- TopDocs mergedTopDocs = null;
+ TopDocs mergedTopDocs = null;
- if (topDocs.length > 0 && topDocs[0] != null) {
- if (topDocs[0] instanceof TopFieldDocs) {
- TopFieldDocs[] topFieldDocs = Arrays.copyOf(topDocs, topDocs.length, TopFieldDocs[].class);
- mergedTopDocs = TopFieldDocs.merge(weightSort(cmd.getSort()), len, topFieldDocs);
- } else {
- mergedTopDocs = TopDocs.merge(0, len, topDocs);
+ if (topDocs.length > 0 && topDocs[0] != null) {
+ if (topDocs[0] instanceof TopFieldDocs) {
+ TopFieldDocs[] topFieldDocs =
+ Arrays.copyOf(topDocs, topDocs.length, TopFieldDocs[].class);
+ mergedTopDocs = TopFieldDocs.merge(weightSort(cmd.getSort()), len, topFieldDocs);
+ } else {
+ mergedTopDocs = TopDocs.merge(0, len, topDocs);
+ }
+ totalHits = (int) mergedTopDocs.totalHits.value;
+ }
+ return new TopDocsResult(mergedTopDocs, totalHits);
}
- totalHits = (int) mergedTopDocs.totalHits.value;
- }
- return new TopDocsResult(mergedTopDocs, totalHits);
- }
- });
+ });
}
if (needMaxScore) {
- collectors.add(new CollectorManager<>() {
- @Override
- public Collector newCollector() throws IOException {
- MaxScoreCollector collector = new MaxScoreCollector();
- if (firstCollectors[1] == null) {
- firstCollectors[1] = collector;
- }
- return collector;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Object reduce(Collection collectors) throws IOException {
+ collectors.add(
+ new CollectorManager<>() {
+ @Override
+ public Collector newCollector() throws IOException {
+ MaxScoreCollector collector = new MaxScoreCollector();
+ if (firstCollectors[1] == null) {
+ firstCollectors[1] = collector;
+ }
+ return collector;
+ }
- MaxScoreCollector collector;
- float maxScore = 0.0f;
- for (Iterator var4 = collectors.iterator(); var4.hasNext(); maxScore = Math.max(maxScore, collector.getMaxScore())) {
- collector = (MaxScoreCollector) var4.next();
- }
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Object reduce(Collection collectors) throws IOException {
+
+ MaxScoreCollector collector;
+ float maxScore = 0.0f;
+ for (Iterator var4 = collectors.iterator();
+ var4.hasNext();
+ maxScore = Math.max(maxScore, collector.getMaxScore())) {
+ collector = (MaxScoreCollector) var4.next();
+ }
- return new MaxScoreResult(maxScore);
- }
- });
+ return new MaxScoreResult(maxScore);
+ }
+ });
}
if (needDocSet) {
int maxDoc = rawReader.maxDoc();
@@ -2040,37 +2057,38 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
LeafSlice[] leaves = getSlices();
int[] docBase = new int[1];
- // DocSetCollector collector = new DocSetCollector(maxDoc);
+ // DocSetCollector collector = new DocSetCollector(maxDoc);
ThreadSafeBitSet bits = new ThreadSafeBitSet(14, 2);
+ collectors.add(
+ new CollectorManager<>() {
+ @Override
+ public Collector newCollector() throws IOException {
+ int numDocs = 0;
- collectors.add(new CollectorManager<>() {
- @Override
- public Collector newCollector() throws IOException {
- int numDocs = 0;
+ if (leaves != null) {
+ LeafSlice leaf = leaves[docBase[0]++];
- if (leaves != null) {
- LeafSlice leaf = leaves[docBase[0]++];
+ for (LeafReaderContext reader : leaf.leaves) {
+ numDocs += reader.reader().maxDoc();
+ }
+ } else {
+ numDocs = maxDoc();
+ }
+ log.error("new docset collector for {} max={}", numDocs, maxDoc());
- for (LeafReaderContext reader : leaf.leaves) {
- numDocs += reader.reader().maxDoc();
+ return new ThreadSafeBitSetCollector(bits, maxDoc);
}
- } else {
- numDocs = maxDoc();
- }
- log.error("new docset collector for {} max={}", numDocs, maxDoc());
- return new ThreadSafeBitSetCollector(bits, maxDoc);
- }
-
- @Override
- @SuppressWarnings({"rawtypes"})
- public Object reduce(Collection collectors) throws IOException {
+ @Override
+ @SuppressWarnings({"rawtypes"})
+ public Object reduce(Collection collectors) throws IOException {
- return new DocSetResult(((ThreadSafeBitSetCollector)collectors.iterator().next()).getDocSet());
- }
- });
+ return new DocSetResult(
+ ((ThreadSafeBitSetCollector) collectors.iterator().next()).getDocSet());
+ }
+ });
}
for (Collector collector : firstCollectors) {
if (collector != null) {
@@ -2089,16 +2107,17 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
try {
ret = super.search(query, manager);
} catch (Exception ex) {
- if (ex instanceof RuntimeException &&
- ex.getCause() != null & ex.getCause() instanceof ExecutionException
- && ex.getCause().getCause() != null && ex.getCause().getCause() instanceof RuntimeException) {
+ if (ex instanceof RuntimeException
+ && ex.getCause() != null
+ && ex.getCause() instanceof ExecutionException
+ && ex.getCause().getCause() != null
+ && ex.getCause().getCause() instanceof RuntimeException) {
throw (RuntimeException) ex.getCause().getCause();
} else {
throw ex;
}
}
-
-
+
return new SearchResult(scoreMode, ret);
}
@@ -2137,7 +2156,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
this.result = result;
}
}
-
+
// any DocSet returned is for the query only, without any filtering... that way it may
// be cached if desired.
private DocSet getDocListAndSetNC(QueryResult qr, QueryCommand cmd) throws IOException {
@@ -2210,44 +2229,47 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
MTCollectorQueryCheck allowMT = new MTCollectorQueryCheck();
query.visit(allowMT);
TopDocs topDocs;
- if (pf.postFilter != null || cmd.getSegmentTerminateEarly() || cmd.getTimeAllowed() > 0
- || !allowMT.allowed()) {
- @SuppressWarnings({"rawtypes"})
- final TopDocsCollector<? extends ScoreDoc> topCollector = buildTopDocsCollector(len, cmd);
- DocSetCollector setCollector = new DocSetCollector(maxDoc);
- MaxScoreCollector maxScoreCollector = null;
- List<Collector> collectors = new ArrayList<>(Arrays.asList(topCollector, setCollector));
-
- if ((cmd.getFlags() & GET_SCORES) != 0) {
- maxScoreCollector = new MaxScoreCollector();
- collectors.add(maxScoreCollector);
- }
+ if (pf.postFilter != null
+ || cmd.getSegmentTerminateEarly()
+ || cmd.getTimeAllowed() > 0
+ || !allowMT.allowed()) {
+ @SuppressWarnings({"rawtypes"})
+ final TopDocsCollector<? extends ScoreDoc> topCollector = buildTopDocsCollector(len, cmd);
+ DocSetCollector setCollector = new DocSetCollector(maxDoc);
+ MaxScoreCollector maxScoreCollector = null;
+ List<Collector> collectors = new ArrayList<>(Arrays.asList(topCollector, setCollector));
+
+ if ((cmd.getFlags() & GET_SCORES) != 0) {
+ maxScoreCollector = new MaxScoreCollector();
+ collectors.add(maxScoreCollector);
+ }
- Collector collector = MultiCollector.wrap(collectors);
+ Collector collector = MultiCollector.wrap(collectors);
- buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter);
+ buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter);
- set = DocSetUtil.getDocSet(setCollector, this);
+ set = DocSetUtil.getDocSet(setCollector, this);
- totalHits = topCollector.getTotalHits();
- assert (totalHits == set.size()) || qr.isPartialResults();
+ totalHits = topCollector.getTotalHits();
+ assert (totalHits == set.size()) || qr.isPartialResults();
- topDocs = topCollector.topDocs(0, len);
- if (cmd.getSort() != null
- && !(cmd.getQuery() instanceof RankQuery)
- && (cmd.getFlags() & GET_SCORES) != 0) {
- TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
- }
- populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
- maxScore =
- totalHits > 0
- ? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore())
- : 0.0f;
+ topDocs = topCollector.topDocs(0, len);
+ if (cmd.getSort() != null
+ && !(cmd.getQuery() instanceof RankQuery)
+ && (cmd.getFlags() & GET_SCORES) != 0) {
+ TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
+ }
+ populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
+ maxScore =
+ totalHits > 0
+ ? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore())
+ : 0.0f;
} else {
log.debug("using CollectorManager");
boolean needMaxScore = (cmd.getFlags() & GET_SCORES) != 0;
- SearchResult searchResult = searchCollectorManagers(len, cmd, query, true, needMaxScore, true);
+ SearchResult searchResult =
+ searchCollectorManagers(len, cmd, query, true, needMaxScore, true);
Object[] res = searchResult.result;
TopDocsResult result = (TopDocsResult) res[0];
totalHits = result.totalHits;
@@ -2269,12 +2291,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}
populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
-// if (cmd.getSort() != null && !(cmd.getQuery() instanceof RankQuery) && (cmd.getFlags() & GET_SCORES) != 0) {
-// TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
-// }
- // nDocsReturned = topDocs.scoreDocs.length;
- //TODO: Is this correct?
- //hitsRelation = topDocs.totalHits.relation;
+ // if (cmd.getSort() != null && !(cmd.getQuery() instanceof RankQuery) &&
+ // (cmd.getFlags() & GET_SCORES) != 0) {
+ // TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
+ // }
+ // nDocsReturned = topDocs.scoreDocs.length;
+ // TODO: Is this correct?
+ // hitsRelation = topDocs.totalHits.relation;
// } else {
// hitsRelation = Relation.EQUAL_TO;
// }
@@ -2905,14 +2928,16 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
return true;
}
+ @Override
public void consumeTerms(Query query, Term... terms) {
if (!allowMt(query)) {
subVisitor = EMPTY_VISITOR;
}
}
+ @Override
public void consumeTermsMatching(
- Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ Query query, String field, Supplier<ByteRunAutomaton> automaton) {
if (!allowMt(query)) {
subVisitor = EMPTY_VISITOR;
} else {
@@ -2920,12 +2945,14 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}
}
+ @Override
public void visitLeaf(Query query) {
if (!allowMt(query)) {
subVisitor = EMPTY_VISITOR;
}
}
+ @Override
public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) {
return subVisitor;
}
@@ -2934,5 +2961,4 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
return subVisitor != EMPTY_VISITOR;
}
}
-
}
diff --git a/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java b/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java
index 0af33da4a63..df44db8d337 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java
@@ -21,7 +21,13 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.CollectorManager;
+import org.apache.lucene.search.FilterScorable;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
/**
* A {@link CollectorManager} implements which wrap a set of {@link CollectorManager} as {@link
@@ -100,7 +106,8 @@ public class SolrMultiCollectorManager
private final LeafCollector[] leafCollectors;
private final boolean skipNonCompetitiveScores;
- private LeafCollectors(final LeafReaderContext context, boolean skipNonCompetitiveScores) throws IOException {
+ private LeafCollectors(final LeafReaderContext context, boolean skipNonCompetitiveScores)
+ throws IOException {
this.skipNonCompetitiveScores = skipNonCompetitiveScores;
leafCollectors = new LeafCollector[collectors.length];
for (int i = 0; i < collectors.length; i++)
@@ -114,14 +121,14 @@ public class SolrMultiCollectorManager
if (leafCollector != null) leafCollector.setScorer(scorer);
} else {
FilterScorable fScorer =
- new FilterScorable(scorer) {
- @Override
- public void setMinCompetitiveScore(float minScore) throws IOException {
- // Ignore calls to setMinCompetitiveScore so that if we wrap two
- // collectors and one of them wants to skip low-scoring hits, then
- // the other collector still sees all hits.
- }
- };
+ new FilterScorable(scorer) {
+ @Override
+ public void setMinCompetitiveScore(float minScore) throws IOException {
+ // Ignore calls to setMinCompetitiveScore so that if we wrap two
+ // collectors and one of them wants to skip low-scoring hits, then
+ // the other collector still sees all hits.
+ }
+ };
for (LeafCollector leafCollector : leafCollectors) {
if (leafCollector != null) {
leafCollector.setScorer(fScorer);
diff --git a/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java
index 8384b25b9a2..5285da44b9e 100644
--- a/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java
+++ b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java
@@ -24,418 +24,442 @@ import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReference;
/**
- * This is a lock-free, thread-safe version of a {@link java.util.BitSet}.<p>
+ * This is a lock-free, thread-safe version of a {@link java.util.BitSet}.
*
- * Instead of a long array to hold the bits, this implementation uses an AtomicLongArray, then
+ * <p>Instead of a long array to hold the bits, this implementation uses an AtomicLongArray, then
* does the appropriate compare-and-swap operations when setting the bits.
- *
- * @author dkoszewnik
- *
*/
public class ThreadSafeBitSet {
- public static final int DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS = 14;
-
- private final int numLongsPerSegment;
- private final int log2SegmentSize;
- private final int segmentMask;
- private final AtomicReference<ThreadSafeBitSetSegments> segments;
-
- public ThreadSafeBitSet() {
- this(DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS); /// 16384 bits, 2048 bytes, 256 longs per segment
+ public static final int DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS = 14;
+
+ private final int numLongsPerSegment;
+ private final int log2SegmentSize;
+ private final int segmentMask;
+ private final AtomicReference<ThreadSafeBitSetSegments> segments;
+
+ public ThreadSafeBitSet() {
+ this(DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS); // / 16384 bits, 2048 bytes, 256 longs per segment
+ }
+
+ public ThreadSafeBitSet(int log2SegmentSizeInBits) {
+ this(log2SegmentSizeInBits, 0);
+ }
+
+ public ThreadSafeBitSet(int log2SegmentSizeInBits, int numBitsToPreallocate) {
+ if (log2SegmentSizeInBits < 6)
+ throw new IllegalArgumentException("Cannot specify fewer than 64 bits in each segment!");
+
+ this.log2SegmentSize = log2SegmentSizeInBits;
+ this.numLongsPerSegment = (1 << (log2SegmentSizeInBits - 6));
+ this.segmentMask = numLongsPerSegment - 1;
+
+ long numBitsPerSegment = ((long) numLongsPerSegment) * 64;
+ int numSegmentsToPreallocate =
+ numBitsToPreallocate == 0
+ ? 1
+ : (int) (((numBitsToPreallocate - 1) / numBitsPerSegment) + 1);
+
+ segments = new AtomicReference<ThreadSafeBitSetSegments>();
+ segments.set(new ThreadSafeBitSetSegments(numSegmentsToPreallocate, numLongsPerSegment));
+ }
+
+ public void set(int position) {
+ int segmentPosition =
+ position >>> log2SegmentSize; // / which segment -- div by num bits per segment
+ int longPosition =
+ (position >>> 6)
+ & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition =
+ position & 0x3F; // / which bit in the long -- remainder of div by num bits in long (64)
+
+ AtomicLongArray segment = getSegment(segmentPosition);
+
+ long mask = 1L << bitPosition;
+
+ // Thread safety: we need to loop until we win the race to set the long value.
+ while (true) {
+ // determine what the new long value will be after we set the appropriate bit.
+ long currentLongValue = segment.get(longPosition);
+ long newLongValue = currentLongValue | mask;
+
+ // if no other thread has modified the value since we read it, we won the race and we are
+ // done.
+ if (segment.compareAndSet(longPosition, currentLongValue, newLongValue)) break;
}
-
- public ThreadSafeBitSet(int log2SegmentSizeInBits) {
- this(log2SegmentSizeInBits, 0);
+ }
+
+ public void clear(int position) {
+ int segmentPosition =
+ position >>> log2SegmentSize; // / which segment -- div by num bits per segment
+ int longPosition =
+ (position >>> 6)
+ & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition =
+ position & 0x3F; // / which bit in the long -- remainder of div by num bits in long (64)
+
+ AtomicLongArray segment = getSegment(segmentPosition);
+
+ long mask = ~(1L << bitPosition);
+
+ // Thread safety: we need to loop until we win the race to set the long value.
+ while (true) {
+ // determine what the new long value will be after we set the appropriate bit.
+ long currentLongValue = segment.get(longPosition);
+ long newLongValue = currentLongValue & mask;
+
+ // if no other thread has modified the value since we read it, we won the race and we are
+ // done.
+ if (segment.compareAndSet(longPosition, currentLongValue, newLongValue)) break;
}
-
- public ThreadSafeBitSet(int log2SegmentSizeInBits, int numBitsToPreallocate) {
- if(log2SegmentSizeInBits < 6)
- throw new IllegalArgumentException("Cannot specify fewer than 64 bits in each segment!");
-
- this.log2SegmentSize = log2SegmentSizeInBits;
- this.numLongsPerSegment = (1 << (log2SegmentSizeInBits - 6));
- this.segmentMask = numLongsPerSegment - 1;
-
- long numBitsPerSegment = numLongsPerSegment * 64;
- int numSegmentsToPreallocate = numBitsToPreallocate == 0 ? 1 : (int)(((numBitsToPreallocate - 1) / numBitsPerSegment) + 1);
-
- segments = new AtomicReference<ThreadSafeBitSetSegments>();
- segments.set(new ThreadSafeBitSetSegments(numSegmentsToPreallocate, numLongsPerSegment));
- }
-
- public void set(int position) {
- int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
- int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
- int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
-
- AtomicLongArray segment = getSegment(segmentPosition);
-
- long mask = 1L << bitPosition;
-
- // Thread safety: we need to loop until we win the race to set the long value.
- while(true) {
- // determine what the new long value will be after we set the appropriate bit.
- long currentLongValue = segment.get(longPosition);
- long newLongValue = currentLongValue | mask;
-
- // if no other thread has modified the value since we read it, we won the race and we are done.
- if(segment.compareAndSet(longPosition, currentLongValue, newLongValue))
- break;
- }
+ }
+
+ public boolean get(int position) {
+ int segmentPosition =
+ position >>> log2SegmentSize; // / which segment -- div by num bits per segment
+ int longPosition =
+ (position >>> 6)
+ & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition =
+ position & 0x3F; // / which bit in the long -- remainder of div by num bits in long (64)
+
+ AtomicLongArray segment = getSegment(segmentPosition);
+
+ long mask = 1L << bitPosition;
+
+ return ((segment.get(longPosition) & mask) != 0);
+ }
+
+ public long maxSetBit() {
+ ThreadSafeBitSetSegments segments = this.segments.get();
+
+ int segmentIdx = segments.numSegments() - 1;
+
+ for (; segmentIdx >= 0; segmentIdx--) {
+ AtomicLongArray segment = segments.getSegment(segmentIdx);
+ for (int longIdx = segment.length() - 1; longIdx >= 0; longIdx--) {
+ long l = segment.get(longIdx);
+ if (l != 0)
+ return (segmentIdx << log2SegmentSize)
+ + (((long) longIdx) * 64)
+ + (63 - Long.numberOfLeadingZeros(l));
+ }
}
- public void clear(int position) {
- int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
- int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
- int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
+ return -1;
+ }
- AtomicLongArray segment = getSegment(segmentPosition);
+ public int nextSetBit(int fromIndex) {
+ if (fromIndex < 0) throw new IndexOutOfBoundsException("fromIndex < 0: " + fromIndex);
- long mask = ~(1L << bitPosition);
+ int segmentPosition =
+ fromIndex >>> log2SegmentSize; // / which segment -- div by num bits per segment
- // Thread safety: we need to loop until we win the race to set the long value.
- while(true) {
- // determine what the new long value will be after we set the appropriate bit.
- long currentLongValue = segment.get(longPosition);
- long newLongValue = currentLongValue & mask;
+ ThreadSafeBitSetSegments segments = this.segments.get();
- // if no other thread has modified the value since we read it, we won the race and we are done.
- if(segment.compareAndSet(longPosition, currentLongValue, newLongValue))
- break;
- }
- }
+ if (segmentPosition >= segments.numSegments()) return -1;
- public boolean get(int position) {
- int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
- int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
- int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
+ int longPosition =
+ (fromIndex >>> 6)
+ & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition =
+ fromIndex & 0x3F; // / which bit in the long -- remainder of div by num bits in long (64)
+ AtomicLongArray segment = segments.getSegment(segmentPosition);
- AtomicLongArray segment = getSegment(segmentPosition);
+ long word = segment.get(longPosition) & (0xffffffffffffffffL << bitPosition);
- long mask = 1L << bitPosition;
+ while (true) {
+ if (word != 0)
+ return (segmentPosition << (log2SegmentSize))
+ + (longPosition << 6)
+ + Long.numberOfTrailingZeros(word);
+ if (++longPosition > segmentMask) {
+ segmentPosition++;
+ if (segmentPosition >= segments.numSegments()) return -1;
+ segment = segments.getSegment(segmentPosition);
+ longPosition = 0;
+ }
- return ((segment.get(longPosition) & mask) != 0);
+ word = segment.get(longPosition);
}
+ }
- public long maxSetBit() {
- ThreadSafeBitSetSegments segments = this.segments.get();
+ /**
+ * @return the number of bits which are set in this bit set.
+ */
+ public int cardinality() {
+ ThreadSafeBitSetSegments segments = this.segments.get();
- int segmentIdx = segments.numSegments() - 1;
-
- for(;segmentIdx >= 0; segmentIdx--) {
- AtomicLongArray segment = segments.getSegment(segmentIdx);
- for(int longIdx=segment.length() - 1; longIdx >= 0; longIdx--) {
- long l = segment.get(longIdx);
- if(l != 0)
- return (segmentIdx << log2SegmentSize) + (longIdx * 64) + (63 - Long.numberOfLeadingZeros(l));
- }
- }
+ int numSetBits = 0;
- return -1;
+ for (int i = 0; i < segments.numSegments(); i++) {
+ AtomicLongArray segment = segments.getSegment(i);
+ for (int j = 0; j < segment.length(); j++) {
+ numSetBits += Long.bitCount(segment.get(j));
+ }
}
- public int nextSetBit(int fromIndex) {
- if (fromIndex < 0)
- throw new IndexOutOfBoundsException("fromIndex < 0: " + fromIndex);
-
- int segmentPosition = fromIndex >>> log2SegmentSize; /// which segment -- div by num bits per segment
-
- ThreadSafeBitSetSegments segments = this.segments.get();
-
- if(segmentPosition >= segments.numSegments())
- return -1;
-
- int longPosition = (fromIndex >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
- int bitPosition = fromIndex & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
- AtomicLongArray segment = segments.getSegment(segmentPosition);
-
- long word = segment.get(longPosition) & (0xffffffffffffffffL << bitPosition);
-
- while (true) {
- if (word != 0)
- return (segmentPosition << (log2SegmentSize)) + (longPosition << 6) + Long.numberOfTrailingZeros(word);
- if (++longPosition > segmentMask) {
- segmentPosition++;
- if(segmentPosition >= segments.numSegments())
- return -1;
- segment = segments.getSegment(segmentPosition);
- longPosition = 0;
- }
-
- word = segment.get(longPosition);
- }
- }
-
-
- /**
- * @return the number of bits which are set in this bit set.
- */
- public int cardinality() {
- ThreadSafeBitSetSegments segments = this.segments.get();
+ return numSetBits;
+ }
- int numSetBits = 0;
+ /**
+ * @return the number of bits which are current specified by this bit set. This is the maximum
+ * value to which you might need to iterate, if you were to iterate over all bits in this set.
+ */
+ public int currentCapacity() {
+ return segments.get().numSegments() * (1 << log2SegmentSize);
+ }
- for(int i=0;i<segments.numSegments();i++) {
- AtomicLongArray segment = segments.getSegment(i);
- for(int j=0;j<segment.length();j++) {
- numSetBits += Long.bitCount(segment.get(j));
- }
- }
+ /** Clear all bits to 0. */
+ public void clearAll() {
+ ThreadSafeBitSetSegments segments = this.segments.get();
- return numSetBits;
- }
+ for (int i = 0; i < segments.numSegments(); i++) {
+ AtomicLongArray segment = segments.getSegment(i);
- /**
- * @return the number of bits which are current specified by this bit set. This is the maximum value
- * to which you might need to iterate, if you were to iterate over all bits in this set.
- */
- public int currentCapacity() {
- return segments.get().numSegments() * (1 << log2SegmentSize);
+ for (int j = 0; j < segment.length(); j++) {
+ segment.set(j, 0L);
+ }
}
-
- /**
- * Clear all bits to 0.
- */
- public void clearAll() {
- ThreadSafeBitSetSegments segments = this.segments.get();
-
- for(int i=0;i<segments.numSegments();i++) {
- AtomicLongArray segment = segments.getSegment(i);
-
- for(int j=0;j<segment.length();j++) {
- segment.set(j, 0L);
- }
- }
+ }
+
+ /**
+ * Return a new bit set which contains all bits which are contained in this bit set, and which are
+ * NOT contained in the <code>other</code> bit set.
+ *
+ * <p>In other words, return a new bit set, which is a bitwise and with the bitwise not of the
+ * other bit set.
+ *
+ * @param other the other bit set
+ * @return the resulting bit set
+ */
+ public ThreadSafeBitSet andNot(ThreadSafeBitSet other) {
+ if (other.log2SegmentSize != log2SegmentSize)
+ throw new IllegalArgumentException("Segment sizes must be the same");
+
+ ThreadSafeBitSetSegments thisSegments = this.segments.get();
+ ThreadSafeBitSetSegments otherSegments = other.segments.get();
+ ThreadSafeBitSetSegments newSegments =
+ new ThreadSafeBitSetSegments(thisSegments.numSegments(), numLongsPerSegment);
+
+ for (int i = 0; i < thisSegments.numSegments(); i++) {
+ AtomicLongArray thisArray = thisSegments.getSegment(i);
+ AtomicLongArray otherArray =
+ (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
+ AtomicLongArray newArray = newSegments.getSegment(i);
+
+ for (int j = 0; j < thisArray.length(); j++) {
+ long thisLong = thisArray.get(j);
+ long otherLong = (otherArray == null) ? 0 : otherArray.get(j);
+
+ newArray.set(j, thisLong & ~otherLong);
+ }
}
- /**
- * Return a new bit set which contains all bits which are contained in this bit set, and which are NOT contained in the <code>other</code> bit set.<p>
- *
- * In other words, return a new bit set, which is a bitwise and with the bitwise not of the other bit set.
- *
- * @param other the other bit set
- * @return the resulting bit set
- */
- public ThreadSafeBitSet andNot(ThreadSafeBitSet other) {
- if(other.log2SegmentSize != log2SegmentSize)
- throw new IllegalArgumentException("Segment sizes must be the same");
-
- ThreadSafeBitSetSegments thisSegments = this.segments.get();
- ThreadSafeBitSetSegments otherSegments = other.segments.get();
- ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(thisSegments.numSegments(), numLongsPerSegment);
-
- for(int i=0;i<thisSegments.numSegments();i++) {
- AtomicLongArray thisArray = thisSegments.getSegment(i);
- AtomicLongArray otherArray = (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
- AtomicLongArray newArray = newSegments.getSegment(i);
-
- for(int j=0;j<thisArray.length();j++) {
- long thisLong = thisArray.get(j);
- long otherLong = (otherArray == null) ? 0 : otherArray.get(j);
-
- newArray.set(j, thisLong & ~otherLong);
- }
- }
-
- ThreadSafeBitSet andNot = new ThreadSafeBitSet(log2SegmentSize);
- andNot.segments.set(newSegments);
- return andNot;
+ ThreadSafeBitSet andNot = new ThreadSafeBitSet(log2SegmentSize);
+ andNot.segments.set(newSegments);
+ return andNot;
+ }
+
+ /**
+ * Return a new bit set which contains all bits which are contained in *any* of the specified bit
+ * sets.
+ *
+ * @param bitSets the other bit sets
+ * @return the resulting bit set
+ */
+ public static ThreadSafeBitSet orAll(ThreadSafeBitSet... bitSets) {
+ if (bitSets.length == 0) return new ThreadSafeBitSet();
+
+ int log2SegmentSize = bitSets[0].log2SegmentSize;
+ int numLongsPerSegment = bitSets[0].numLongsPerSegment;
+
+ ThreadSafeBitSetSegments segments[] = new ThreadSafeBitSetSegments[bitSets.length];
+ int maxNumSegments = 0;
+
+ for (int i = 0; i < bitSets.length; i++) {
+ if (bitSets[i].log2SegmentSize != log2SegmentSize)
+ throw new IllegalArgumentException("Segment sizes must be the same");
+
+ segments[i] = bitSets[i].segments.get();
+ if (segments[i].numSegments() > maxNumSegments) maxNumSegments = segments[i].numSegments();
}
- /**
- * Return a new bit set which contains all bits which are contained in *any* of the specified bit sets.
- *
- * @param bitSets the other bit sets
- * @return the resulting bit set
- */
- public static ThreadSafeBitSet orAll(ThreadSafeBitSet... bitSets) {
- if(bitSets.length == 0)
- return new ThreadSafeBitSet();
-
- int log2SegmentSize = bitSets[0].log2SegmentSize;
- int numLongsPerSegment = bitSets[0].numLongsPerSegment;
-
- ThreadSafeBitSetSegments segments[] = new ThreadSafeBitSetSegments[bitSets.length];
- int maxNumSegments = 0;
-
- for(int i=0;i<bitSets.length;i++) {
- if(bitSets[i].log2SegmentSize != log2SegmentSize)
- throw new IllegalArgumentException("Segment sizes must be the same");
-
- segments[i] = bitSets[i].segments.get();
- if(segments[i].numSegments() > maxNumSegments)
- maxNumSegments = segments[i].numSegments();
- }
-
- ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(maxNumSegments, numLongsPerSegment);
+ ThreadSafeBitSetSegments newSegments =
+ new ThreadSafeBitSetSegments(maxNumSegments, numLongsPerSegment);
- AtomicLongArray segment[] = new AtomicLongArray[segments.length];
+ AtomicLongArray segment[] = new AtomicLongArray[segments.length];
- for(int i=0;i<maxNumSegments;i++) {
- for(int j=0;j<segments.length;j++) {
- segment[j] = i < segments[j].numSegments() ? segments[j].getSegment(i) : null;
- }
+ for (int i = 0; i < maxNumSegments; i++) {
+ for (int j = 0; j < segments.length; j++) {
+ segment[j] = i < segments[j].numSegments() ? segments[j].getSegment(i) : null;
+ }
- AtomicLongArray newSegment = newSegments.getSegment(i);
+ AtomicLongArray newSegment = newSegments.getSegment(i);
- for(int j=0;j<numLongsPerSegment;j++) {
- long value = 0;
- for(int k=0;k<segments.length;k++) {
- if(segment[k] != null)
- value |= segment[k].get(j);
- }
- newSegment.set(j, value);
- }
+ for (int j = 0; j < numLongsPerSegment; j++) {
+ long value = 0;
+ for (int k = 0; k < segments.length; k++) {
+ if (segment[k] != null) value |= segment[k].get(j);
}
-
- ThreadSafeBitSet or = new ThreadSafeBitSet(log2SegmentSize);
- or.segments.set(newSegments);
- return or;
+ newSegment.set(j, value);
+ }
}
- /**
- * Get the segment at <code>segmentIndex</code>. If this segment does not yet exist, create it.
- *
- * @param segmentIndex the segment index
- * @return the segment
- */
- private AtomicLongArray getSegment(int segmentIndex) {
- ThreadSafeBitSetSegments visibleSegments = segments.get();
-
- while(visibleSegments.numSegments() <= segmentIndex) {
- /// Thread safety: newVisibleSegments contains all of the segments from the currently visible segments, plus extra.
- /// all of the segments in the currently visible segments are canonical and will not change.
- ThreadSafeBitSetSegments newVisibleSegments = new ThreadSafeBitSetSegments(visibleSegments, segmentIndex + 1, numLongsPerSegment);
-
- /// because we are using a compareAndSet, if this thread "wins the race" and successfully sets this variable, then the segments
- /// which are newly defined in newVisibleSegments become canonical.
- if(segments.compareAndSet(visibleSegments, newVisibleSegments)) {
- visibleSegments = newVisibleSegments;
- } else {
- /// If we "lose the race" and are growing the ThreadSafeBitSet segments larger,
- /// then we will gather the new canonical sets from the update which we missed on the next iteration of this loop.
- /// Newly defined segments in newVisibleSegments will be discarded, they do not get to become canonical.
- visibleSegments = segments.get();
- }
- }
-
- return visibleSegments.getSegment(segmentIndex);
+ ThreadSafeBitSet or = new ThreadSafeBitSet(log2SegmentSize);
+ or.segments.set(newSegments);
+ return or;
+ }
+
+ /**
+ * Get the segment at <code>segmentIndex</code>. If this segment does not yet exist, create it.
+ *
+ * @param segmentIndex the segment index
+ * @return the segment
+ */
+ private AtomicLongArray getSegment(int segmentIndex) {
+ ThreadSafeBitSetSegments visibleSegments = segments.get();
+
+ while (visibleSegments.numSegments() <= segmentIndex) {
+ /// Thread safety: newVisibleSegments contains all of the segments from the currently visible
+ // segments, plus extra.
+ /// all of the segments in the currently visible segments are canonical and will not change.
+ ThreadSafeBitSetSegments newVisibleSegments =
+ new ThreadSafeBitSetSegments(visibleSegments, segmentIndex + 1, numLongsPerSegment);
+
+ /// because we are using a compareAndSet, if this thread "wins the race" and successfully sets
+ // this variable, then the segments
+ /// which are newly defined in newVisibleSegments become canonical.
+ if (segments.compareAndSet(visibleSegments, newVisibleSegments)) {
+ visibleSegments = newVisibleSegments;
+ } else {
+ /// If we "lose the race" and are growing the ThreadSafeBitSet segments larger,
+ /// then we will gather the new canonical sets from the update which we missed on the next
+ // iteration of this loop.
+ /// Newly defined segments in newVisibleSegments will be discarded, they do not get to
+ // become canonical.
+ visibleSegments = segments.get();
+ }
}
- private static class ThreadSafeBitSetSegments {
-
- private final AtomicLongArray segments[];
-
- private ThreadSafeBitSetSegments(int numSegments, int segmentLength) {
- AtomicLongArray segments[] = new AtomicLongArray[numSegments];
+ return visibleSegments.getSegment(segmentIndex);
+ }
- for(int i=0;i<numSegments;i++) {
- segments[i] = new AtomicLongArray(segmentLength);
- }
+ private static class ThreadSafeBitSetSegments {
- /// Thread safety: Because this.segments is final, the preceding operations in this constructor are guaranteed to be visible to any
- /// other thread which accesses this.segments.
- this.segments = segments;
- }
+ private final AtomicLongArray segments[];
- private ThreadSafeBitSetSegments(ThreadSafeBitSetSegments copyFrom, int numSegments, int segmentLength) {
- AtomicLongArray segments[] = new AtomicLongArray[numSegments];
+ private ThreadSafeBitSetSegments(int numSegments, int segmentLength) {
+ AtomicLongArray segments[] = new AtomicLongArray[numSegments];
- for(int i=0;i<numSegments;i++) {
- segments[i] = i < copyFrom.numSegments() ? copyFrom.getSegment(i) : new AtomicLongArray(segmentLength);
- }
+ for (int i = 0; i < numSegments; i++) {
+ segments[i] = new AtomicLongArray(segmentLength);
+ }
- /// see above re: thread-safety of this assignment
- this.segments = segments;
- }
+ /// Thread safety: Because this.segments is final, the preceding operations in this
+ // constructor are guaranteed to be visible to any
+ /// other thread which accesses this.segments.
+ this.segments = segments;
+ }
- public int numSegments() {
- return segments.length;
- }
+ private ThreadSafeBitSetSegments(
+ ThreadSafeBitSetSegments copyFrom, int numSegments, int segmentLength) {
+ AtomicLongArray segments[] = new AtomicLongArray[numSegments];
- public AtomicLongArray getSegment(int index) {
- return segments[index];
- }
+ for (int i = 0; i < numSegments; i++) {
+ segments[i] =
+ i < copyFrom.numSegments()
+ ? copyFrom.getSegment(i)
+ : new AtomicLongArray(segmentLength);
+ }
+ /// see above re: thread-safety of this assignment
+ this.segments = segments;
}
- public void serializeBitsTo(DataOutputStream os) throws IOException {
- ThreadSafeBitSetSegments segments = this.segments.get();
-
- os.writeInt(segments.numSegments() * numLongsPerSegment);
-
- for(int i=0;i<segments.numSegments();i++) {
- AtomicLongArray arr = segments.getSegment(i);
+ public int numSegments() {
+ return segments.length;
+ }
- for(int j=0;j<arr.length();j++) {
- os.writeLong(arr.get(j));
- }
- }
+ public AtomicLongArray getSegment(int index) {
+ return segments[index];
}
+ }
- @Override
- public boolean equals(Object obj) {
- if(!(obj instanceof ThreadSafeBitSet))
- return false;
+ public void serializeBitsTo(DataOutputStream os) throws IOException {
+ ThreadSafeBitSetSegments segments = this.segments.get();
- ThreadSafeBitSet other = (ThreadSafeBitSet)obj;
+ os.writeInt(segments.numSegments() * numLongsPerSegment);
- if(other.log2SegmentSize != log2SegmentSize)
- throw new IllegalArgumentException("Segment sizes must be the same");
+ for (int i = 0; i < segments.numSegments(); i++) {
+ AtomicLongArray arr = segments.getSegment(i);
- ThreadSafeBitSetSegments thisSegments = this.segments.get();
- ThreadSafeBitSetSegments otherSegments = other.segments.get();
+ for (int j = 0; j < arr.length(); j++) {
+ os.writeLong(arr.get(j));
+ }
+ }
+ }
- for(int i=0;i<thisSegments.numSegments();i++) {
- AtomicLongArray thisArray = thisSegments.getSegment(i);
- AtomicLongArray otherArray = (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ThreadSafeBitSet)) return false;
- for(int j=0;j<thisArray.length();j++) {
- long thisLong = thisArray.get(j);
- long otherLong = (otherArray == null) ? 0 : otherArray.get(j);
+ ThreadSafeBitSet other = (ThreadSafeBitSet) obj;
- if(thisLong != otherLong)
- return false;
- }
- }
+ if (other.log2SegmentSize != log2SegmentSize)
+ throw new IllegalArgumentException("Segment sizes must be the same");
- for(int i=thisSegments.numSegments();i<otherSegments.numSegments();i++) {
- AtomicLongArray otherArray = otherSegments.getSegment(i);
+ ThreadSafeBitSetSegments thisSegments = this.segments.get();
+ ThreadSafeBitSetSegments otherSegments = other.segments.get();
- for(int j=0;j<otherArray.length();j++) {
- long l = otherArray.get(j);
+ for (int i = 0; i < thisSegments.numSegments(); i++) {
+ AtomicLongArray thisArray = thisSegments.getSegment(i);
+ AtomicLongArray otherArray =
+ (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
- if(l != 0)
- return false;
- }
- }
+ for (int j = 0; j < thisArray.length(); j++) {
+ long thisLong = thisArray.get(j);
+ long otherLong = (otherArray == null) ? 0 : otherArray.get(j);
- return true;
+ if (thisLong != otherLong) return false;
+ }
}
- @Override
- public int hashCode() {
- int result = log2SegmentSize;
- result = 31 * result + Arrays.hashCode(segments.get().segments);
- return result;
- }
+ for (int i = thisSegments.numSegments(); i < otherSegments.numSegments(); i++) {
+ AtomicLongArray otherArray = otherSegments.getSegment(i);
- /**
- * @return a new BitSet with same bits set
- */
- public BitSet toBitSet() {
- BitSet resultSet = new BitSet();
- int ordinal = this.nextSetBit(0);
- while(ordinal!=-1) {
- resultSet.set(ordinal);
- ordinal = this.nextSetBit(ordinal + 1);
- }
- return resultSet;
+ for (int j = 0; j < otherArray.length(); j++) {
+ long l = otherArray.get(j);
+
+ if (l != 0) return false;
+ }
}
- @Override
- public String toString() {
- return toBitSet().toString();
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = log2SegmentSize;
+ result = 31 * result + Arrays.hashCode(segments.get().segments);
+ return result;
+ }
+
+ /**
+ * @return a new BitSet with same bits set
+ */
+ public BitSet toBitSet() {
+ BitSet resultSet = new BitSet();
+ int ordinal = this.nextSetBit(0);
+ while (ordinal != -1) {
+ resultSet.set(ordinal);
+ ordinal = this.nextSetBit(ordinal + 1);
}
-}
\ No newline at end of file
+ return resultSet;
+ }
+
+ @Override
+ public String toString() {
+ return toBitSet().toString();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java
index 626d01ce168..426d3b64788 100644
--- a/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java
+++ b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.search;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
@@ -24,13 +26,7 @@ import org.apache.lucene.util.FixedBitSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-
-/**
- *
- */
-
+/** */
public class ThreadSafeBitSetCollector extends SimpleCollector {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final ThreadSafeBitSet bits;
@@ -38,21 +34,19 @@ public class ThreadSafeBitSetCollector extends SimpleCollector {
int base;
-
-
public ThreadSafeBitSetCollector(ThreadSafeBitSet bits, int maxDoc) {
this.bits = bits;
this.maxDoc = maxDoc;
}
-
@Override
public void collect(int doc) throws IOException {
doc += base;
- log.error("collect doc {} {}", (doc + base), this);
- bits.set(doc);
-
+ if (log.isErrorEnabled()) {
+ log.error("collect doc: {}, base: {}", doc, base, this);
+ }
+ bits.set(doc);
}
/** The number of documents that have been collected */
@@ -67,7 +61,7 @@ public class ThreadSafeBitSetCollector extends SimpleCollector {
int cnt = 0;
int i = -1;
while (true) {
- i = bits.nextSetBit(i+1);
+ i = bits.nextSetBit(i + 1);
if (i == -1) {
break;
}
@@ -76,12 +70,10 @@ public class ThreadSafeBitSetCollector extends SimpleCollector {
}
return new BitDocSet(fixedBitSet, cnt);
-
}
@Override
- public void setScorer(Scorable scorer) throws IOException {
- }
+ public void setScorer(Scorable scorer) throws IOException {}
@Override
public ScoreMode scoreMode() {
@@ -91,7 +83,8 @@ public class ThreadSafeBitSetCollector extends SimpleCollector {
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
this.base = context.docBase;
- log.error("next reader base=" + base);
+ if (log.isErrorEnabled()) {
+ log.error("next reader base={}", base);
+ }
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
index fb3a1137734..7b99a95fe21 100644
--- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
+++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
@@ -1786,7 +1786,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
CommonParams.ROWS,
"200",
CommonParams.SORT,
- "score desc, id asc");
+ "id asc");
assertIsExactHitCount(
"q", "{!cache=false}id:1", CommonParams.MIN_EXACT_COUNT, "1", CommonParams.ROWS, "1");
assertApproximatedHitCount(
diff --git a/solr/core/src/test/org/apache/solr/search/TestFiltering.java b/solr/core/src/test/org/apache/solr/search/TestFiltering.java
index e59e007cafb..e9194e170f9 100644
--- a/solr/core/src/test/org/apache/solr/search/TestFiltering.java
+++ b/solr/core/src/test/org/apache/solr/search/TestFiltering.java
@@ -93,19 +93,14 @@ public class TestFiltering extends SolrTestCaseJ4 {
QueryResult res = new QueryResult();
searcher.search(res, cmd);
set = res.getDocSet();
- assertSame(set, live);
- System.out.println("Live: "+bitsString(live.getFixedBitSet()));
- System.out.println("Set: "+bitsString(set.getFixedBitSet()));
- // FixedBitSet xor = live.getFixedBitSet().clone();
- // xor.xor(set.getFixedBitSet());
- // System.out.println("xor: "+bitsString(xor));
+ assertEffectivelySame(set.getFixedBitSet(), live.getFixedBitSet());
cmd.setQuery(QParser.getParser(qstr + " OR id:0", null, req).getQuery());
cmd.setFilterList(QParser.getParser(qstr + " OR id:1", null, req).getQuery());
res = new QueryResult();
searcher.search(res, cmd);
set = res.getDocSet();
- assertSame(set, live);
+ assertEffectivelySame(set.getFixedBitSet(), live.getFixedBitSet());
}
} finally {
@@ -113,14 +108,20 @@ public class TestFiltering extends SolrTestCaseJ4 {
}
}
+ /** If the a XOR b == 0, then both a & b are effectively the same bitset */
+ private void assertEffectivelySame(FixedBitSet a, FixedBitSet b) {
+ FixedBitSet xor = a.clone();
+ xor.xor(b);
+ assertEquals(new FixedBitSet(xor.length()), xor);
+ }
+
private String bitsString(Bits bits) {
StringBuilder s = new StringBuilder();
- for (int i=0; i<bits.length(); i++)
- s.append(bits.get(i) ? 1 : 0);
+ for (int i = 0; i < bits.length(); i++) s.append(bits.get(i) ? 1 : 0);
return s.toString();
}
- public void testCaching() throws Exception {
+ public void testCaching() throws Exception {
clearIndex();
assertU(adoc("id", "4", "val_i", "1"));
assertU(adoc("id", "1", "val_i", "2"));
(solr) 01/02: SOLR-13350: Multithreaded search using CollectorManager
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-13350
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 3cee3e591995520cdfed41a5228a2e01c3b5cc0f
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Thu May 4 21:35:09 2023 +0530
SOLR-13350: Multithreaded search using CollectorManager
---
.../java/org/apache/solr/core/CoreContainer.java | 15 +-
.../java/org/apache/solr/search/QueryResult.java | 1 +
.../org/apache/solr/search/SolrIndexSearcher.java | 334 +++++++++++++++-
.../solr/search/SolrMultiCollectorManager.java | 143 +++++++
.../org/apache/solr/search/ThreadSafeBitSet.java | 441 +++++++++++++++++++++
.../solr/search/ThreadSafeBitSetCollector.java | 97 +++++
.../test/org/apache/solr/search/TestFiltering.java | 15 +-
.../modules/indexing-guide/examples/stemdict.txt | 23 +-
8 files changed, 1056 insertions(+), 13 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 1da0802cf08..012c31633b8 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -49,6 +49,8 @@ import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.function.Function;
@@ -175,7 +177,11 @@ public class CoreContainer {
final SolrCores solrCores;
- public static class CoreLoadFailure {
+ public Executor getCollectorExecutor() {
+ return collectorExecutor;
+ }
+
+ public static class CoreLoadFailure {
public final CoreDescriptor cd;
public final Exception exception;
@@ -278,6 +284,8 @@ public class CoreContainer {
public final NodeRoles nodeRoles = new NodeRoles(System.getProperty(NodeRoles.NODE_ROLES_PROP));
+ private final ExecutorService collectorExecutor;
+
private final ClusterSingletons clusterSingletons =
new ClusterSingletons(
() ->
@@ -431,6 +439,9 @@ public class CoreContainer {
this.allowPaths = allowPathBuilder.build();
this.allowListUrlChecker = AllowListUrlChecker.create(config);
+
+ this.collectorExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(6,
+ new SolrNamedThreadFactory("searcherCollector"));
}
@SuppressWarnings({"unchecked"})
@@ -656,6 +667,7 @@ public class CoreContainer {
distributedCollectionCommandRunner = Optional.empty();
allowPaths = null;
allowListUrlChecker = null;
+ collectorExecutor = null;
}
public static CoreContainer createAndLoad(Path solrHome) {
@@ -1247,6 +1259,7 @@ public class CoreContainer {
}
ExecutorUtil.shutdownAndAwaitTermination(coreContainerAsyncTaskExecutor);
+ ExecutorUtil.shutdownAndAwaitTermination(collectorExecutor);
ExecutorService customThreadPool =
ExecutorUtil.newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("closeThreadPool"));
diff --git a/solr/core/src/java/org/apache/solr/search/QueryResult.java b/solr/core/src/java/org/apache/solr/search/QueryResult.java
index 9c9a8e6f60d..5d4500a4878 100755
--- a/solr/core/src/java/org/apache/solr/search/QueryResult.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryResult.java
@@ -46,6 +46,7 @@ public class QueryResult {
docListAndSet = new DocListAndSet();
}
docListAndSet.docSet = set;
+ // log.error("set docset {}", docListAndSet.docSet.getBits().length());
}
public boolean isPartialResults() {
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 9c3a2fbc2f3..a8c11016848 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -26,11 +26,13 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -51,10 +53,12 @@ import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.CollectorManager;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
@@ -63,6 +67,7 @@ import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.ScoreMode;
@@ -87,6 +92,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.automaton.ByteRunAutomaton;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -108,6 +114,7 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.facet.UnInvertedField;
+import org.apache.solr.search.join.GraphQuery;
import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.search.stats.StatsSource;
import org.apache.solr.uninverting.UninvertingReader;
@@ -335,7 +342,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
boolean reserveDirectory,
DirectoryFactory directoryFactory)
throws IOException {
- super(wrapReader(core, r));
+ super(wrapReader(core, r), core.getCoreContainer().getCollectorExecutor());
this.path = path;
this.directoryFactory = directoryFactory;
@@ -1799,7 +1806,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
int last = len;
if (last < 0 || last > maxDoc()) last = maxDoc();
final int lastDocRequested = last;
- int nDocsReturned;
+ int nDocsReturned = 0;
int totalHits;
float maxScore;
int[] ids;
@@ -1858,7 +1865,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter);
- nDocsReturned = 0;
ids = new int[nDocsReturned];
scores = new float[nDocsReturned];
totalHits = numHits[0];
@@ -1867,10 +1873,17 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
qr.setNextCursorMark(cmd.getCursorMark());
hitsRelation = Relation.EQUAL_TO;
} else {
+ log.info("calling from 2, query: "+query.getClass()); // nocommit
+ MTCollectorQueryCheck allowMT = new MTCollectorQueryCheck();
+ query.visit(allowMT);
+ TopDocs topDocs;
+ if (pf.postFilter != null || cmd.getSegmentTerminateEarly() || cmd.getTimeAllowed() > 0
+ || !allowMT.allowed()) {
+ log.debug("skipping collector manager");
final TopDocsCollector<?> topCollector = buildTopDocsCollector(len, cmd);
MaxScoreCollector maxScoreCollector = null;
Collector collector = topCollector;
- if ((cmd.getFlags() & GET_SCORES) != 0) {
+ if (needScores) {
maxScoreCollector = new MaxScoreCollector();
collector = MultiCollector.wrap(topCollector, maxScoreCollector);
}
@@ -1878,7 +1891,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter).scoreMode();
totalHits = topCollector.getTotalHits();
- TopDocs topDocs = topCollector.topDocs(0, len);
+ topDocs = topCollector.topDocs(0, len);
if (scoreModeUsed == ScoreMode.COMPLETE || scoreModeUsed == ScoreMode.COMPLETE_NO_SCORES) {
hitsRelation = TotalHits.Relation.EQUAL_TO;
} else {
@@ -1886,7 +1899,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}
if (cmd.getSort() != null
&& cmd.getQuery() instanceof RankQuery == false
- && (cmd.getFlags() & GET_SCORES) != 0) {
+ && needScores) {
TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
}
populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
@@ -1896,6 +1909,37 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore())
: 0.0f;
nDocsReturned = topDocs.scoreDocs.length;
+
+ } else {
+ log.info("using CollectorManager");
+ SearchResult searchResult = searchCollectorManagers(len, cmd, query, true, needScores, false);
+ Object[] res = searchResult.result;
+ TopDocsResult result = (TopDocsResult) res[0];
+
+ totalHits = result.totalHits;
+ topDocs = result.topDocs;
+
+ if (res.length > 1) {
+ MaxScoreResult result2 = (MaxScoreResult) res[1];
+ maxScore = totalHits > 0 ? result2.maxScore : 0.0f;
+ } else {
+ maxScore = Float.NaN;
+ }
+
+ populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
+ if (cmd.getSort() != null && !(cmd.getQuery() instanceof RankQuery) && needScores) {
+ TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
+ }
+ nDocsReturned = topDocs.scoreDocs.length;
+ ScoreMode scoreModeUsed = searchResult.scoreMode;
+ if (scoreModeUsed == ScoreMode.COMPLETE || scoreModeUsed == ScoreMode.COMPLETE_NO_SCORES) {
+ hitsRelation = TotalHits.Relation.EQUAL_TO;
+ } else {
+ hitsRelation = topDocs.totalHits.relation;
+ }
+
+ }
+
ids = new int[nDocsReturned];
scores = (cmd.getFlags() & GET_SCORES) != 0 ? new float[nDocsReturned] : null;
for (int i = 0; i < nDocsReturned; i++) {
@@ -1910,6 +1954,190 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
qr.setDocList(new DocSlice(0, sliceLen, ids, scores, totalHits, maxScore, hitsRelation));
}
+ SearchResult searchCollectorManagers(int len, QueryCommand cmd, Query query,
+ boolean needTopDocs, boolean needMaxScore, boolean needDocSet) throws IOException {
+ Collection<CollectorManager<Collector, Object>> collectors = new ArrayList<>();
+ ScoreMode scoreMode = null;
+
+ Collector[] firstCollectors = new Collector[3];
+
+ if (needTopDocs) {
+
+ collectors.add(new CollectorManager<>() {
+ @Override
+ public Collector newCollector() throws IOException {
+ @SuppressWarnings("rawtypes")
+ TopDocsCollector collector = buildTopDocsCollector(len, cmd);
+ if (firstCollectors[0] == null) {
+ firstCollectors[0] = collector;
+ }
+ return collector;
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Object reduce(Collection collectors) throws IOException {
+
+ TopDocs[] topDocs = new TopDocs[collectors.size()];
+
+ int totalHits = -1;
+ int i = 0;
+
+ Collector collector;
+ for (Object o : collectors) {
+ collector = (Collector) o;
+ if (collector instanceof TopDocsCollector) {
+ TopDocs td = ((TopDocsCollector) collector).topDocs(0, len);
+ assert td != null : Arrays.asList(topDocs);
+ topDocs[i++] = td;
+ }
+ }
+
+ TopDocs mergedTopDocs = null;
+
+ if (topDocs.length > 0 && topDocs[0] != null) {
+ if (topDocs[0] instanceof TopFieldDocs) {
+ TopFieldDocs[] topFieldDocs = Arrays.copyOf(topDocs, topDocs.length, TopFieldDocs[].class);
+ mergedTopDocs = TopFieldDocs.merge(weightSort(cmd.getSort()), len, topFieldDocs);
+ } else {
+ mergedTopDocs = TopDocs.merge(0, len, topDocs);
+ }
+ totalHits = (int) mergedTopDocs.totalHits.value;
+ }
+ return new TopDocsResult(mergedTopDocs, totalHits);
+ }
+ });
+ }
+ if (needMaxScore) {
+ collectors.add(new CollectorManager<>() {
+ @Override
+ public Collector newCollector() throws IOException {
+ MaxScoreCollector collector = new MaxScoreCollector();
+ if (firstCollectors[1] == null) {
+ firstCollectors[1] = collector;
+ }
+ return collector;
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Object reduce(Collection collectors) throws IOException {
+
+ MaxScoreCollector collector;
+ float maxScore = 0.0f;
+ for (Iterator var4 = collectors.iterator(); var4.hasNext(); maxScore = Math.max(maxScore, collector.getMaxScore())) {
+ collector = (MaxScoreCollector) var4.next();
+ }
+
+ return new MaxScoreResult(maxScore);
+ }
+ });
+ }
+ if (needDocSet) {
+ int maxDoc = rawReader.maxDoc();
+ log.error("raw read max={}", rawReader.maxDoc());
+
+ LeafSlice[] leaves = getSlices();
+ int[] docBase = new int[1];
+
+ // DocSetCollector collector = new DocSetCollector(maxDoc);
+
+ ThreadSafeBitSet bits = new ThreadSafeBitSet(14, 2);
+
+
+ collectors.add(new CollectorManager<>() {
+ @Override
+ public Collector newCollector() throws IOException {
+ int numDocs = 0;
+
+ if (leaves != null) {
+ LeafSlice leaf = leaves[docBase[0]++];
+
+ for (LeafReaderContext reader : leaf.leaves) {
+ numDocs += reader.reader().maxDoc();
+ }
+ } else {
+ numDocs = maxDoc();
+ }
+ log.error("new docset collector for {} max={}", numDocs, maxDoc());
+
+ return new ThreadSafeBitSetCollector(bits, maxDoc);
+ }
+
+ @Override
+ @SuppressWarnings({"rawtypes"})
+ public Object reduce(Collection collectors) throws IOException {
+
+ return new DocSetResult(((ThreadSafeBitSetCollector)collectors.iterator().next()).getDocSet());
+ }
+ });
+ }
+ for (Collector collector : firstCollectors) {
+ if (collector != null) {
+ if (scoreMode == null) {
+ scoreMode = collector.scoreMode();
+ } else if (scoreMode != collector.scoreMode()) {
+ scoreMode = ScoreMode.COMPLETE;
+ }
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ CollectorManager<Collector, Object>[] colls = collectors.toArray(new CollectorManager[0]);
+ SolrMultiCollectorManager manager = new SolrMultiCollectorManager(colls);
+ Object[] ret;
+ try {
+ ret = super.search(query, manager);
+ } catch (Exception ex) {
+ if (ex instanceof RuntimeException &&
+ ex.getCause() != null & ex.getCause() instanceof ExecutionException
+ && ex.getCause().getCause() != null && ex.getCause().getCause() instanceof RuntimeException) {
+ throw (RuntimeException) ex.getCause().getCause();
+ } else {
+ throw ex;
+ }
+ }
+
+
+ return new SearchResult(scoreMode, ret);
+ }
+
+ static class TopDocsResult {
+ final TopDocs topDocs;
+ final int totalHits;
+
+ public TopDocsResult(TopDocs topDocs, int totalHits) {
+ this.topDocs = topDocs;
+ this.totalHits = totalHits;
+ }
+ }
+
+ static class MaxScoreResult {
+ final float maxScore;
+
+ public MaxScoreResult(float maxScore) {
+ this.maxScore = maxScore;
+ }
+ }
+
+ static class DocSetResult {
+ final DocSet docSet;
+
+ public DocSetResult(DocSet docSet) {
+ this.docSet = docSet;
+ }
+ }
+
+ static class SearchResult {
+ final Object[] result;
+ final ScoreMode scoreMode;
+
+ public SearchResult(ScoreMode scoreMode, Object[] result) {
+ this.scoreMode = scoreMode;
+ this.result = result;
+ }
+ }
+
// any DocSet returned is for the query only, without any filtering... that way it may
// be cached if desired.
private DocSet getDocListAndSetNC(QueryResult qr, QueryCommand cmd) throws IOException {
@@ -1919,10 +2147,10 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
final int lastDocRequested = last;
int nDocsReturned;
int totalHits;
- float maxScore;
+ float maxScore = Float.NaN;
int[] ids;
float[] scores;
- DocSet set;
+ DocSet set = null;
boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;
int maxDoc = maxDoc();
@@ -1979,6 +2207,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
// no docs on this page, so cursor doesn't change
qr.setNextCursorMark(cmd.getCursorMark());
} else {
+ MTCollectorQueryCheck allowMT = new MTCollectorQueryCheck();
+ query.visit(allowMT);
+ TopDocs topDocs;
+ if (pf.postFilter != null || cmd.getSegmentTerminateEarly() || cmd.getTimeAllowed() > 0
+ || !allowMT.allowed()) {
+ @SuppressWarnings({"rawtypes"})
final TopDocsCollector<? extends ScoreDoc> topCollector = buildTopDocsCollector(len, cmd);
DocSetCollector setCollector = new DocSetCollector(maxDoc);
MaxScoreCollector maxScoreCollector = null;
@@ -1998,7 +2232,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
totalHits = topCollector.getTotalHits();
assert (totalHits == set.size()) || qr.isPartialResults();
- TopDocs topDocs = topCollector.topDocs(0, len);
+ topDocs = topCollector.topDocs(0, len);
if (cmd.getSort() != null
&& !(cmd.getQuery() instanceof RankQuery)
&& (cmd.getFlags() & GET_SCORES) != 0) {
@@ -2009,6 +2243,44 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
totalHits > 0
? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore())
: 0.0f;
+ } else {
+ log.debug("using CollectorManager");
+
+ boolean needMaxScore = (cmd.getFlags() & GET_SCORES) != 0;
+ SearchResult searchResult = searchCollectorManagers(len, cmd, query, true, needMaxScore, true);
+ Object[] res = searchResult.result;
+ TopDocsResult result = (TopDocsResult) res[0];
+ totalHits = result.totalHits;
+ topDocs = result.topDocs;
+ if (needMaxScore) {
+ if (res.length > 1) {
+ MaxScoreResult result2 = (MaxScoreResult) res[1];
+ maxScore = totalHits > 0 ? result2.maxScore : 0.0f;
+ }
+ if (res.length > 2) {
+ DocSetResult result3 = (DocSetResult) res[2];
+ set = result3.docSet;
+ }
+ } else {
+ if (res.length > 1) {
+ DocSetResult result2 = (DocSetResult) res[1];
+ set = result2.docSet;
+ }
+ }
+
+ populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
+// if (cmd.getSort() != null && !(cmd.getQuery() instanceof RankQuery) && (cmd.getFlags() & GET_SCORES) != 0) {
+// TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
+// }
+ // nDocsReturned = topDocs.scoreDocs.length;
+ //TODO: Is this correct?
+ //hitsRelation = topDocs.totalHits.relation;
+ // } else {
+ // hitsRelation = Relation.EQUAL_TO;
+ // }
+
+ }
+
nDocsReturned = topDocs.scoreDocs.length;
ids = new int[nDocsReturned];
@@ -2251,7 +2523,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}
// bit of a hack to tell if a set is sorted - do it better in the future.
- boolean inOrder = set instanceof BitDocSet || set instanceof SortedIntDocSet;
+ // boolean inOrder = set instanceof BitDocSet || set instanceof SortedIntDocSet;
TopDocsCollector<? extends ScoreDoc> topCollector = buildTopDocsCollector(nDocs, cmd);
@@ -2621,4 +2893,46 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
public long getWarmupTime() {
return warmupTime;
}
+
+ private static class MTCollectorQueryCheck extends QueryVisitor {
+
+ private QueryVisitor subVisitor = this;
+
+ private boolean allowMt(Query query) {
+ if (query instanceof RankQuery || query instanceof GraphQuery || query instanceof JoinQuery) {
+ return false;
+ }
+ return true;
+ }
+
+ public void consumeTerms(Query query, Term... terms) {
+ if (!allowMt(query)) {
+ subVisitor = EMPTY_VISITOR;
+ }
+ }
+
+ public void consumeTermsMatching(
+ Query query, String field, Supplier<ByteRunAutomaton> automaton) {
+ if (!allowMt(query)) {
+ subVisitor = EMPTY_VISITOR;
+ } else {
+ super.consumeTermsMatching(query, field, automaton);
+ }
+ }
+
+ public void visitLeaf(Query query) {
+ if (!allowMt(query)) {
+ subVisitor = EMPTY_VISITOR;
+ }
+ }
+
+ public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) {
+ return subVisitor;
+ }
+
+ public boolean allowed() {
+ return subVisitor != EMPTY_VISITOR;
+ }
+ }
+
}
diff --git a/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java b/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java
new file mode 100644
index 00000000000..0af33da4a63
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/SolrMultiCollectorManager.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.*;
+
+/**
+ * A {@link CollectorManager} implements which wrap a set of {@link CollectorManager} as {@link
+ * MultiCollector} acts for {@link Collector}.
+ */
+public class SolrMultiCollectorManager
+ implements CollectorManager<SolrMultiCollectorManager.Collectors, Object[]> {
+
+ private final CollectorManager<Collector, ?>[] collectorManagers;
+
+ @SafeVarargs
+ @SuppressWarnings({"varargs", "unchecked"})
+ public SolrMultiCollectorManager(
+ final CollectorManager<? extends Collector, ?>... collectorManagers) {
+ if (collectorManagers.length < 1) {
+ throw new IllegalArgumentException("There must be at least one collector");
+ }
+ this.collectorManagers = (CollectorManager[]) collectorManagers;
+ }
+
+ @Override
+ public Collectors newCollector() throws IOException {
+ return new Collectors();
+ }
+
+ @Override
+ public Object[] reduce(Collection<Collectors> reducableCollectors) throws IOException {
+ final int size = reducableCollectors.size();
+ final Object[] results = new Object[collectorManagers.length];
+ for (int i = 0; i < collectorManagers.length; i++) {
+ final List<Collector> reducableCollector = new ArrayList<>(size);
+ for (Collectors collectors : reducableCollectors)
+ reducableCollector.add(collectors.collectors[i]);
+ results[i] = collectorManagers[i].reduce(reducableCollector);
+ }
+ return results;
+ }
+
+ /** Wraps multiple collectors for processing */
+ public class Collectors implements Collector {
+
+ private final Collector[] collectors;
+
+ private Collectors() throws IOException {
+ collectors = new Collector[collectorManagers.length];
+ for (int i = 0; i < collectors.length; i++)
+ collectors[i] = collectorManagers[i].newCollector();
+ }
+
+ @Override
+ public final LeafCollector getLeafCollector(final LeafReaderContext context)
+ throws IOException {
+ return new LeafCollectors(context, scoreMode() == ScoreMode.TOP_SCORES);
+ }
+
+ @Override
+ public final ScoreMode scoreMode() {
+ ScoreMode scoreMode = null;
+ for (Collector collector : collectors) {
+ if (scoreMode == null) {
+ scoreMode = collector.scoreMode();
+ } else if (scoreMode != collector.scoreMode()) {
+ return ScoreMode.COMPLETE;
+ }
+ }
+ return scoreMode;
+ }
+
+ /**
+ * Wraps multiple leaf collectors and delegates collection across each one
+ *
+ * @lucene.internal
+ */
+ public class LeafCollectors implements LeafCollector {
+
+ private final LeafCollector[] leafCollectors;
+ private final boolean skipNonCompetitiveScores;
+
+ private LeafCollectors(final LeafReaderContext context, boolean skipNonCompetitiveScores) throws IOException {
+ this.skipNonCompetitiveScores = skipNonCompetitiveScores;
+ leafCollectors = new LeafCollector[collectors.length];
+ for (int i = 0; i < collectors.length; i++)
+ leafCollectors[i] = collectors[i].getLeafCollector(context);
+ }
+
+ @Override
+ public final void setScorer(final Scorable scorer) throws IOException {
+ if (skipNonCompetitiveScores) {
+ for (LeafCollector leafCollector : leafCollectors)
+ if (leafCollector != null) leafCollector.setScorer(scorer);
+ } else {
+ FilterScorable fScorer =
+ new FilterScorable(scorer) {
+ @Override
+ public void setMinCompetitiveScore(float minScore) throws IOException {
+ // Ignore calls to setMinCompetitiveScore so that if we wrap two
+ // collectors and one of them wants to skip low-scoring hits, then
+ // the other collector still sees all hits.
+ }
+ };
+ for (LeafCollector leafCollector : leafCollectors) {
+ if (leafCollector != null) {
+ leafCollector.setScorer(fScorer);
+ }
+ }
+ }
+ }
+
+ @Override
+ public final void collect(final int doc) throws IOException {
+ for (LeafCollector leafCollector : leafCollectors) {
+ if (leafCollector != null) {
+ leafCollector.collect(doc);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java
new file mode 100644
index 00000000000..8384b25b9a2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSet.java
@@ -0,0 +1,441 @@
+/*
+ * Copyright 2016-2019 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.solr.search;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.concurrent.atomic.AtomicLongArray;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * This is a lock-free, thread-safe version of a {@link java.util.BitSet}.<p>
+ *
+ * Instead of a long array to hold the bits, this implementation uses an AtomicLongArray, then
+ * does the appropriate compare-and-swap operations when setting the bits.
+ *
+ * @author dkoszewnik
+ *
+ */
+public class ThreadSafeBitSet {
+
+ public static final int DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS = 14;
+
+ private final int numLongsPerSegment;
+ private final int log2SegmentSize;
+ private final int segmentMask;
+ private final AtomicReference<ThreadSafeBitSetSegments> segments;
+
+ public ThreadSafeBitSet() {
+ this(DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS); /// 16384 bits, 2048 bytes, 256 longs per segment
+ }
+
+ public ThreadSafeBitSet(int log2SegmentSizeInBits) {
+ this(log2SegmentSizeInBits, 0);
+ }
+
+ public ThreadSafeBitSet(int log2SegmentSizeInBits, int numBitsToPreallocate) {
+ if(log2SegmentSizeInBits < 6)
+ throw new IllegalArgumentException("Cannot specify fewer than 64 bits in each segment!");
+
+ this.log2SegmentSize = log2SegmentSizeInBits;
+ this.numLongsPerSegment = (1 << (log2SegmentSizeInBits - 6));
+ this.segmentMask = numLongsPerSegment - 1;
+
+ long numBitsPerSegment = numLongsPerSegment * 64;
+ int numSegmentsToPreallocate = numBitsToPreallocate == 0 ? 1 : (int)(((numBitsToPreallocate - 1) / numBitsPerSegment) + 1);
+
+ segments = new AtomicReference<ThreadSafeBitSetSegments>();
+ segments.set(new ThreadSafeBitSetSegments(numSegmentsToPreallocate, numLongsPerSegment));
+ }
+
+ public void set(int position) {
+ int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
+ int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
+
+ AtomicLongArray segment = getSegment(segmentPosition);
+
+ long mask = 1L << bitPosition;
+
+ // Thread safety: we need to loop until we win the race to set the long value.
+ while(true) {
+ // determine what the new long value will be after we set the appropriate bit.
+ long currentLongValue = segment.get(longPosition);
+ long newLongValue = currentLongValue | mask;
+
+ // if no other thread has modified the value since we read it, we won the race and we are done.
+ if(segment.compareAndSet(longPosition, currentLongValue, newLongValue))
+ break;
+ }
+ }
+
+ public void clear(int position) {
+ int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
+ int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
+
+ AtomicLongArray segment = getSegment(segmentPosition);
+
+ long mask = ~(1L << bitPosition);
+
+ // Thread safety: we need to loop until we win the race to set the long value.
+ while(true) {
+ // determine what the new long value will be after we set the appropriate bit.
+ long currentLongValue = segment.get(longPosition);
+ long newLongValue = currentLongValue & mask;
+
+ // if no other thread has modified the value since we read it, we won the race and we are done.
+ if(segment.compareAndSet(longPosition, currentLongValue, newLongValue))
+ break;
+ }
+ }
+
+ public boolean get(int position) {
+ int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment
+ int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
+
+ AtomicLongArray segment = getSegment(segmentPosition);
+
+ long mask = 1L << bitPosition;
+
+ return ((segment.get(longPosition) & mask) != 0);
+ }
+
+ public long maxSetBit() {
+ ThreadSafeBitSetSegments segments = this.segments.get();
+
+ int segmentIdx = segments.numSegments() - 1;
+
+ for(;segmentIdx >= 0; segmentIdx--) {
+ AtomicLongArray segment = segments.getSegment(segmentIdx);
+ for(int longIdx=segment.length() - 1; longIdx >= 0; longIdx--) {
+ long l = segment.get(longIdx);
+ if(l != 0)
+ return (segmentIdx << log2SegmentSize) + (longIdx * 64) + (63 - Long.numberOfLeadingZeros(l));
+ }
+ }
+
+ return -1;
+ }
+
+ public int nextSetBit(int fromIndex) {
+ if (fromIndex < 0)
+ throw new IndexOutOfBoundsException("fromIndex < 0: " + fromIndex);
+
+ int segmentPosition = fromIndex >>> log2SegmentSize; /// which segment -- div by num bits per segment
+
+ ThreadSafeBitSetSegments segments = this.segments.get();
+
+ if(segmentPosition >= segments.numSegments())
+ return -1;
+
+ int longPosition = (fromIndex >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment
+ int bitPosition = fromIndex & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64)
+ AtomicLongArray segment = segments.getSegment(segmentPosition);
+
+ long word = segment.get(longPosition) & (0xffffffffffffffffL << bitPosition);
+
+ while (true) {
+ if (word != 0)
+ return (segmentPosition << (log2SegmentSize)) + (longPosition << 6) + Long.numberOfTrailingZeros(word);
+ if (++longPosition > segmentMask) {
+ segmentPosition++;
+ if(segmentPosition >= segments.numSegments())
+ return -1;
+ segment = segments.getSegment(segmentPosition);
+ longPosition = 0;
+ }
+
+ word = segment.get(longPosition);
+ }
+ }
+
+
+ /**
+ * @return the number of bits which are set in this bit set.
+ */
+ public int cardinality() {
+ ThreadSafeBitSetSegments segments = this.segments.get();
+
+ int numSetBits = 0;
+
+ for(int i=0;i<segments.numSegments();i++) {
+ AtomicLongArray segment = segments.getSegment(i);
+ for(int j=0;j<segment.length();j++) {
+ numSetBits += Long.bitCount(segment.get(j));
+ }
+ }
+
+ return numSetBits;
+ }
+
+ /**
+ * @return the number of bits which are current specified by this bit set. This is the maximum value
+ * to which you might need to iterate, if you were to iterate over all bits in this set.
+ */
+ public int currentCapacity() {
+ return segments.get().numSegments() * (1 << log2SegmentSize);
+ }
+
+ /**
+ * Clear all bits to 0.
+ */
+ public void clearAll() {
+ ThreadSafeBitSetSegments segments = this.segments.get();
+
+ for(int i=0;i<segments.numSegments();i++) {
+ AtomicLongArray segment = segments.getSegment(i);
+
+ for(int j=0;j<segment.length();j++) {
+ segment.set(j, 0L);
+ }
+ }
+ }
+
+ /**
+ * Return a new bit set which contains all bits which are contained in this bit set, and which are NOT contained in the <code>other</code> bit set.<p>
+ *
+ * In other words, return a new bit set, which is a bitwise and with the bitwise not of the other bit set.
+ *
+ * @param other the other bit set
+ * @return the resulting bit set
+ */
+ public ThreadSafeBitSet andNot(ThreadSafeBitSet other) {
+ if(other.log2SegmentSize != log2SegmentSize)
+ throw new IllegalArgumentException("Segment sizes must be the same");
+
+ ThreadSafeBitSetSegments thisSegments = this.segments.get();
+ ThreadSafeBitSetSegments otherSegments = other.segments.get();
+ ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(thisSegments.numSegments(), numLongsPerSegment);
+
+ for(int i=0;i<thisSegments.numSegments();i++) {
+ AtomicLongArray thisArray = thisSegments.getSegment(i);
+ AtomicLongArray otherArray = (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
+ AtomicLongArray newArray = newSegments.getSegment(i);
+
+ for(int j=0;j<thisArray.length();j++) {
+ long thisLong = thisArray.get(j);
+ long otherLong = (otherArray == null) ? 0 : otherArray.get(j);
+
+ newArray.set(j, thisLong & ~otherLong);
+ }
+ }
+
+ ThreadSafeBitSet andNot = new ThreadSafeBitSet(log2SegmentSize);
+ andNot.segments.set(newSegments);
+ return andNot;
+ }
+
+ /**
+ * Return a new bit set which contains all bits which are contained in *any* of the specified bit sets.
+ *
+ * @param bitSets the other bit sets
+ * @return the resulting bit set
+ */
+ public static ThreadSafeBitSet orAll(ThreadSafeBitSet... bitSets) {
+ if(bitSets.length == 0)
+ return new ThreadSafeBitSet();
+
+ int log2SegmentSize = bitSets[0].log2SegmentSize;
+ int numLongsPerSegment = bitSets[0].numLongsPerSegment;
+
+ ThreadSafeBitSetSegments segments[] = new ThreadSafeBitSetSegments[bitSets.length];
+ int maxNumSegments = 0;
+
+ for(int i=0;i<bitSets.length;i++) {
+ if(bitSets[i].log2SegmentSize != log2SegmentSize)
+ throw new IllegalArgumentException("Segment sizes must be the same");
+
+ segments[i] = bitSets[i].segments.get();
+ if(segments[i].numSegments() > maxNumSegments)
+ maxNumSegments = segments[i].numSegments();
+ }
+
+ ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(maxNumSegments, numLongsPerSegment);
+
+ AtomicLongArray segment[] = new AtomicLongArray[segments.length];
+
+ for(int i=0;i<maxNumSegments;i++) {
+ for(int j=0;j<segments.length;j++) {
+ segment[j] = i < segments[j].numSegments() ? segments[j].getSegment(i) : null;
+ }
+
+ AtomicLongArray newSegment = newSegments.getSegment(i);
+
+ for(int j=0;j<numLongsPerSegment;j++) {
+ long value = 0;
+ for(int k=0;k<segments.length;k++) {
+ if(segment[k] != null)
+ value |= segment[k].get(j);
+ }
+ newSegment.set(j, value);
+ }
+ }
+
+ ThreadSafeBitSet or = new ThreadSafeBitSet(log2SegmentSize);
+ or.segments.set(newSegments);
+ return or;
+ }
+
+ /**
+ * Get the segment at <code>segmentIndex</code>. If this segment does not yet exist, create it.
+ *
+ * @param segmentIndex the segment index
+ * @return the segment
+ */
+ private AtomicLongArray getSegment(int segmentIndex) {
+ ThreadSafeBitSetSegments visibleSegments = segments.get();
+
+ while(visibleSegments.numSegments() <= segmentIndex) {
+ /// Thread safety: newVisibleSegments contains all of the segments from the currently visible segments, plus extra.
+ /// all of the segments in the currently visible segments are canonical and will not change.
+ ThreadSafeBitSetSegments newVisibleSegments = new ThreadSafeBitSetSegments(visibleSegments, segmentIndex + 1, numLongsPerSegment);
+
+ /// because we are using a compareAndSet, if this thread "wins the race" and successfully sets this variable, then the segments
+ /// which are newly defined in newVisibleSegments become canonical.
+ if(segments.compareAndSet(visibleSegments, newVisibleSegments)) {
+ visibleSegments = newVisibleSegments;
+ } else {
+ /// If we "lose the race" and are growing the ThreadSafeBitSet segments larger,
+ /// then we will gather the new canonical sets from the update which we missed on the next iteration of this loop.
+ /// Newly defined segments in newVisibleSegments will be discarded, they do not get to become canonical.
+ visibleSegments = segments.get();
+ }
+ }
+
+ return visibleSegments.getSegment(segmentIndex);
+ }
+
+ private static class ThreadSafeBitSetSegments {
+
+ private final AtomicLongArray segments[];
+
+ private ThreadSafeBitSetSegments(int numSegments, int segmentLength) {
+ AtomicLongArray segments[] = new AtomicLongArray[numSegments];
+
+ for(int i=0;i<numSegments;i++) {
+ segments[i] = new AtomicLongArray(segmentLength);
+ }
+
+ /// Thread safety: Because this.segments is final, the preceding operations in this constructor are guaranteed to be visible to any
+ /// other thread which accesses this.segments.
+ this.segments = segments;
+ }
+
+ private ThreadSafeBitSetSegments(ThreadSafeBitSetSegments copyFrom, int numSegments, int segmentLength) {
+ AtomicLongArray segments[] = new AtomicLongArray[numSegments];
+
+ for(int i=0;i<numSegments;i++) {
+ segments[i] = i < copyFrom.numSegments() ? copyFrom.getSegment(i) : new AtomicLongArray(segmentLength);
+ }
+
+ /// see above re: thread-safety of this assignment
+ this.segments = segments;
+ }
+
+ public int numSegments() {
+ return segments.length;
+ }
+
+ public AtomicLongArray getSegment(int index) {
+ return segments[index];
+ }
+
+ }
+
+ public void serializeBitsTo(DataOutputStream os) throws IOException {
+ ThreadSafeBitSetSegments segments = this.segments.get();
+
+ os.writeInt(segments.numSegments() * numLongsPerSegment);
+
+ for(int i=0;i<segments.numSegments();i++) {
+ AtomicLongArray arr = segments.getSegment(i);
+
+ for(int j=0;j<arr.length();j++) {
+ os.writeLong(arr.get(j));
+ }
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(!(obj instanceof ThreadSafeBitSet))
+ return false;
+
+ ThreadSafeBitSet other = (ThreadSafeBitSet)obj;
+
+ if(other.log2SegmentSize != log2SegmentSize)
+ throw new IllegalArgumentException("Segment sizes must be the same");
+
+ ThreadSafeBitSetSegments thisSegments = this.segments.get();
+ ThreadSafeBitSetSegments otherSegments = other.segments.get();
+
+ for(int i=0;i<thisSegments.numSegments();i++) {
+ AtomicLongArray thisArray = thisSegments.getSegment(i);
+ AtomicLongArray otherArray = (i < otherSegments.numSegments()) ? otherSegments.getSegment(i) : null;
+
+ for(int j=0;j<thisArray.length();j++) {
+ long thisLong = thisArray.get(j);
+ long otherLong = (otherArray == null) ? 0 : otherArray.get(j);
+
+ if(thisLong != otherLong)
+ return false;
+ }
+ }
+
+ for(int i=thisSegments.numSegments();i<otherSegments.numSegments();i++) {
+ AtomicLongArray otherArray = otherSegments.getSegment(i);
+
+ for(int j=0;j<otherArray.length();j++) {
+ long l = otherArray.get(j);
+
+ if(l != 0)
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = log2SegmentSize;
+ result = 31 * result + Arrays.hashCode(segments.get().segments);
+ return result;
+ }
+
+ /**
+ * @return a new BitSet with same bits set
+ */
+ public BitSet toBitSet() {
+ BitSet resultSet = new BitSet();
+ int ordinal = this.nextSetBit(0);
+ while(ordinal!=-1) {
+ resultSet.set(ordinal);
+ ordinal = this.nextSetBit(ordinal + 1);
+ }
+ return resultSet;
+ }
+
+ @Override
+ public String toString() {
+ return toBitSet().toString();
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java
new file mode 100644
index 00000000000..626d01ce168
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/ThreadSafeBitSetCollector.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.SimpleCollector;
+import org.apache.lucene.util.FixedBitSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+
+/**
+ *
+ */
+
+public class ThreadSafeBitSetCollector extends SimpleCollector {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ final ThreadSafeBitSet bits;
+ final int maxDoc;
+
+ int base;
+
+
+
+ public ThreadSafeBitSetCollector(ThreadSafeBitSet bits, int maxDoc) {
+ this.bits = bits;
+ this.maxDoc = maxDoc;
+ }
+
+
+ @Override
+ public void collect(int doc) throws IOException {
+
+ doc += base;
+ log.error("collect doc {} {}", (doc + base), this);
+ bits.set(doc);
+
+ }
+
+ /** The number of documents that have been collected */
+ public int size() {
+ return maxDoc;
+ }
+
+ public DocSet getDocSet() {
+ log.error("Max Set Bit {}", bits.maxSetBit());
+
+ FixedBitSet fixedBitSet = new FixedBitSet(maxDoc + 1);
+ int cnt = 0;
+ int i = -1;
+ while (true) {
+ i = bits.nextSetBit(i+1);
+ if (i == -1) {
+ break;
+ }
+ cnt++;
+ fixedBitSet.set(i);
+ }
+
+ return new BitDocSet(fixedBitSet, cnt);
+
+ }
+
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return ScoreMode.COMPLETE_NO_SCORES;
+ }
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext context) throws IOException {
+ this.base = context.docBase;
+ log.error("next reader base=" + base);
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/search/TestFiltering.java b/solr/core/src/test/org/apache/solr/search/TestFiltering.java
index 5d9ad6bb9da..e59e007cafb 100644
--- a/solr/core/src/test/org/apache/solr/search/TestFiltering.java
+++ b/solr/core/src/test/org/apache/solr/search/TestFiltering.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Locale;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrInputDocument;
@@ -93,6 +94,11 @@ public class TestFiltering extends SolrTestCaseJ4 {
searcher.search(res, cmd);
set = res.getDocSet();
assertSame(set, live);
+ System.out.println("Live: "+bitsString(live.getFixedBitSet()));
+ System.out.println("Set: "+bitsString(set.getFixedBitSet()));
+ // FixedBitSet xor = live.getFixedBitSet().clone();
+ // xor.xor(set.getFixedBitSet());
+ // System.out.println("xor: "+bitsString(xor));
cmd.setQuery(QParser.getParser(qstr + " OR id:0", null, req).getQuery());
cmd.setFilterList(QParser.getParser(qstr + " OR id:1", null, req).getQuery());
@@ -107,7 +113,14 @@ public class TestFiltering extends SolrTestCaseJ4 {
}
}
- public void testCaching() throws Exception {
+ private String bitsString(Bits bits) {
+ StringBuilder s = new StringBuilder();
+ for (int i=0; i<bits.length(); i++)
+ s.append(bits.get(i) ? 1 : 0);
+ return s.toString();
+ }
+
+ public void testCaching() throws Exception {
clearIndex();
assertU(adoc("id", "4", "val_i", "1"));
assertU(adoc("id", "1", "val_i", "2"));
diff --git a/solr/solr-ref-guide/modules/indexing-guide/examples/stemdict.txt b/solr/solr-ref-guide/modules/indexing-guide/examples/stemdict.txt
deleted file mode 120000
index c242c73fe09..00000000000
--- a/solr/solr-ref-guide/modules/indexing-guide/examples/stemdict.txt
+++ /dev/null
@@ -1 +0,0 @@
-../../../../core/src/test-files/solr/collection1/conf/stemdict.txt
\ No newline at end of file
diff --git a/solr/solr-ref-guide/modules/indexing-guide/examples/stemdict.txt b/solr/solr-ref-guide/modules/indexing-guide/examples/stemdict.txt
new file mode 100644
index 00000000000..f57a4ad490f
--- /dev/null
+++ b/solr/solr-ref-guide/modules/indexing-guide/examples/stemdict.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# test that we can override the stemming algorithm with our own mappings
+# these must be tab-separated
+monkeys monkey
+otters otter
+# some crazy ones that a stemmer would never do
+dogs cat