You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/06/25 02:06:42 UTC
[20/47] lucene-solr:feature/autoscaling: SOLR-10921: raise lucene
BooleanQuery.maxClauseCount, add higher level checking via QueryUtils.build
SOLR-10921: raise lucene BooleanQuery.maxClauseCount, add higher level checking via QueryUtils.build
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/98276481
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/98276481
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/98276481
Branch: refs/heads/feature/autoscaling
Commit: 98276481e4104d4916a6b0e901d79c6b81f68d30
Parents: 5de15ff
Author: yonik <yo...@apache.org>
Authored: Thu Jun 22 16:05:19 2017 -0400
Committer: yonik <yo...@apache.org>
Committed: Thu Jun 22 16:05:19 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 5 +
.../java/org/apache/solr/core/SolrConfig.java | 3 +
.../src/java/org/apache/solr/core/SolrCore.java | 16 +-
.../apache/solr/parser/SolrQueryParserBase.java | 5 +-
.../java/org/apache/solr/schema/FieldType.java | 4 +-
.../org/apache/solr/search/DisMaxQParser.java | 4 +-
.../solr/search/ExtendedDismaxQParser.java | 10 +-
.../java/org/apache/solr/search/QueryUtils.java | 11 ++
.../apache/solr/search/mlt/CloudMLTQParser.java | 3 +-
.../solr/search/mlt/SimpleMLTQParser.java | 3 +-
.../apache/solr/search/TestSolrQueryParser.java | 10 ++
.../search/facet/TestJsonFacetRefinement.java | 176 +++++++++++--------
.../basic_configs/conf/solrconfig.xml | 15 +-
.../conf/solrconfig.xml | 16 +-
.../conf/solrconfig.xml | 15 +-
15 files changed, 157 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 71dfa6a..a8cf4a3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -194,6 +194,11 @@ Bug Fixes
* SOLR-10886: Using V2Request.process(solrClient) method throws NPE if the API returns an error (Cao Manh Dat)
+* SOLR-10921: Work around the static Lucene BooleanQuery.maxClauseCount that causes Solr's maxBooleanClauses
+ setting behavior to be "last core wins". This patch sets BooleanQuery.maxClauseCount to its maximum value,
+ thus disabling the global check, and replaces it with specific checks where desired via
+ QueryUtils.build(). (yonik)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/core/SolrConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index 12fd6ef..bc946e8 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -367,6 +367,9 @@ public class SolrConfig extends Config implements MapSerializable {
public static final Map<String, SolrPluginInfo> classVsSolrPluginInfo;
static {
+ // Raise the Lucene static limit so we can control this with higher granularity. See SOLR-10921
+ BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE-1);
+
Map<String, SolrPluginInfo> map = new HashMap<>();
for (SolrPluginInfo plugin : plugins) map.put(plugin.clazz.getName(), plugin);
classVsSolrPluginInfo = Collections.unmodifiableMap(map);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index f0bb88b..76e2cad 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -68,7 +68,6 @@ import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -249,18 +248,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
}
static int boolean_query_max_clause_count = Integer.MIN_VALUE;
-
- // only change the BooleanQuery maxClauseCount once for ALL cores...
- void booleanQueryMaxClauseCount() {
- synchronized(SolrCore.class) {
- if (boolean_query_max_clause_count == Integer.MIN_VALUE) {
- boolean_query_max_clause_count = solrConfig.booleanQueryMaxClauseCount;
- BooleanQuery.setMaxClauseCount(boolean_query_max_clause_count);
- } else if (boolean_query_max_clause_count != solrConfig.booleanQueryMaxClauseCount ) {
- log.debug("BooleanQuery.maxClauseCount={}, ignoring {}", boolean_query_max_clause_count, solrConfig.booleanQueryMaxClauseCount);
- }
- }
- }
+
/**
* The SolrResourceLoader used to load all resources for this core.
@@ -931,8 +919,6 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
this.maxWarmingSearchers = config.maxWarmingSearchers;
this.slowQueryThresholdMillis = config.slowQueryThresholdMillis;
- booleanQueryMaxClauseCount();
-
final CountDownLatch latch = new CountDownLatch(1);
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
index 21e0aa0..4c62e85 100644
--- a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
+++ b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
@@ -56,6 +56,7 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TextField;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.SolrConstantScoreQuery;
import org.apache.solr.search.SyntaxError;
@@ -657,7 +658,7 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
}
}
- BooleanQuery bq = booleanBuilder.build();
+ BooleanQuery bq = QueryUtils.build(booleanBuilder,parser);
if (bq.clauses().size() == 1) { // Unwrap single SHOULD query
BooleanClause clause = bq.clauses().iterator().next();
if (clause.getOccur() == BooleanClause.Occur.SHOULD) {
@@ -910,7 +911,7 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
Query subq = ft.getFieldQuery(this.parser, rawq.sfield, externalVal);
booleanBuilder.add(subq, BooleanClause.Occur.SHOULD);
}
- normal = booleanBuilder.build();
+ normal = QueryUtils.build(booleanBuilder, parser);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index a4058fb..388b659 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -67,6 +67,7 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader;
import org.slf4j.Logger;
@@ -754,12 +755,13 @@ public abstract class FieldType extends FieldProperties {
/** @lucene.experimental */
public Query getSetQuery(QParser parser, SchemaField field, Collection<String> externalVals) {
if (!field.indexed()) {
+ // TODO: if the field isn't indexed, this feels like the wrong query type to use?
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (String externalVal : externalVals) {
Query subq = getFieldQuery(parser, field, externalVal);
builder.add(subq, BooleanClause.Occur.SHOULD);
}
- return builder.build();
+ return QueryUtils.build(builder, parser);
}
List<BytesRef> lst = new ArrayList<>(externalVals.size());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java b/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java
index 2af5def..fa6100d 100644
--- a/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java
@@ -113,7 +113,7 @@ public class DisMaxQParser extends QParser {
addBoostQuery(query, solrParams);
addBoostFunctions(query, solrParams);
- return query.build();
+ return QueryUtils.build(query, this);
}
protected void addBoostFunctions(BooleanQuery.Builder query, SolrParams solrParams) throws SyntaxError {
@@ -252,7 +252,7 @@ public class DisMaxQParser extends QParser {
SolrPluginUtils.flattenBooleanQuery(t, (BooleanQuery) dis);
boolean mmAutoRelax = params.getBool(DisMaxParams.MM_AUTORELAX, false);
SolrPluginUtils.setMinShouldMatch(t, minShouldMatch, mmAutoRelax);
- query = t.build();
+ query = QueryUtils.build(t, this);
}
return query;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
index 9ec07e8..173f039 100644
--- a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
@@ -192,7 +192,7 @@ public class ExtendedDismaxQParser extends QParser {
//
// create a boosted query (scores multiplied by boosts)
//
- Query topQuery = query.build();
+ Query topQuery = QueryUtils.build(query, this);
List<ValueSource> boosts = getMultiplicativeBoosts();
if (boosts.size()>1) {
ValueSource prod = new ProductFloatFunction(boosts.toArray(new ValueSource[boosts.size()]));
@@ -282,7 +282,7 @@ public class ExtendedDismaxQParser extends QParser {
BooleanQuery.Builder t = new BooleanQuery.Builder();
SolrPluginUtils.flattenBooleanQuery(t, (BooleanQuery)query);
SolrPluginUtils.setMinShouldMatch(t, config.minShouldMatch, config.mmAutoRelax);
- query = t.build();
+ query = QueryUtils.build(t, this);
}
return query;
}
@@ -1163,7 +1163,7 @@ public class ExtendedDismaxQParser extends QParser {
for (Query sub : lst) {
q.add(sub, BooleanClause.Occur.SHOULD);
}
- return q.build();
+ return QueryUtils.build(q, parser);
}
} else {
@@ -1225,7 +1225,7 @@ public class ExtendedDismaxQParser extends QParser {
}
q.add(newBooleanClause(new DisjunctionMaxQuery(subs, a.tie), BooleanClause.Occur.SHOULD));
}
- return q.build();
+ return QueryUtils.build(q, parser);
} else {
return new DisjunctionMaxQuery(lst, a.tie);
}
@@ -1234,7 +1234,7 @@ public class ExtendedDismaxQParser extends QParser {
for (Query sub : lst) {
q.add(sub, BooleanClause.Occur.SHOULD);
}
- return q.build();
+ return QueryUtils.build(q, parser);
}
} else {
// verify that a fielded query is actually on a field that exists... if not,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/search/QueryUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QueryUtils.java b/solr/core/src/java/org/apache/solr/search/QueryUtils.java
index 6c07ff6..3e2b6a0 100644
--- a/solr/core/src/java/org/apache/solr/search/QueryUtils.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryUtils.java
@@ -22,6 +22,7 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrException;
import java.util.Collection;
@@ -129,4 +130,14 @@ public class QueryUtils {
return new BoostQuery(newBq, boost);
}
+ /** @lucene.experimental throw exception if max boolean clauses are exceeded */
+ public static BooleanQuery build(BooleanQuery.Builder builder, QParser parser) {
+ int configuredMax = parser != null ? parser.getReq().getCore().getSolrConfig().booleanQueryMaxClauseCount : BooleanQuery.getMaxClauseCount();
+ BooleanQuery bq = builder.build();
+ if (bq.clauses().size() > configuredMax) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Too many clauses in boolean query: encountered=" + bq.clauses().size() + " configured in solrconfig.xml via maxBooleanClauses=" + configuredMax);
+ }
+ return bq;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
index 17b7d3b..7669db8 100644
--- a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java
@@ -46,6 +46,7 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.QueryUtils;
import org.apache.solr.util.SolrPluginUtils;
import static org.apache.solr.common.params.CommonParams.ID;
@@ -161,7 +162,7 @@ public class CloudMLTQParser extends QParser {
newQ.add(q, clause.getOccur());
}
- boostedMLTQuery = newQ.build();
+ boostedMLTQuery = QueryUtils.build(newQ, this);
}
// exclude current document from results
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java
index cc87e09..4a3400b 100644
--- a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java
@@ -33,6 +33,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.SolrPluginUtils;
@@ -134,7 +135,7 @@ public class SimpleMLTQParser extends QParser {
newQ.add(q, clause.getOccur());
}
- boostedMLTQuery = newQ.build();
+ boostedMLTQuery = QueryUtils.build(newQ, this);
}
// exclude current document from results
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
index 3877c8c..65c4d8f 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
@@ -358,6 +358,16 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
String q = sb.toString();
// This will still fail when used as the main query, but will pass in a filter query since TermsQuery can be used.
+ try {
+ ignoreException("Too many clauses");
+ assertJQ(req("q",q)
+ ,"/response/numFound==6");
+ fail();
+ } catch (Exception e) {
+ // expect "too many clauses" exception... see SOLR-10921
+ assertTrue(e.getMessage().contains("many clauses"));
+ }
+
assertJQ(req("q","*:*", "fq", q)
,"/response/numFound==6");
assertJQ(req("q","*:*", "fq", q, "sow", "false")
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
index 94b753f..d91867b 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
@@ -41,7 +41,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
@BeforeClass
public static void beforeTests() throws Exception {
JSONTestUtil.failRepeatedKeys = true;
- initCore("solrconfig-tlog.xml","schema_latest.xml");
+ initCore("solrconfig-tlog.xml", "schema_latest.xml");
}
public static void initServers() throws Exception {
@@ -67,7 +67,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
assertNull(json);
continue;
}
- if (test.length()==0) continue;
+ if (test.length() == 0) continue;
String err = JSONTestUtil.match(json, test, delta);
@@ -104,7 +104,9 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
}
- /** Use SimpleOrderedMap rather than Map to match responses from shards */
+ /**
+ * Use SimpleOrderedMap rather than Map to match responses from shards
+ */
public static Object fromJSON(String json) throws IOException {
JSONParser parser = new JSONParser(json);
ObjectBuilder ob = new ObjectBuilder(parser) {
@@ -115,7 +117,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
@Override
public void addKeyVal(Object map, Object key, Object val) throws IOException {
- ((SimpleOrderedMap)map).add(key.toString(), val);
+ ((SimpleOrderedMap) map).add(key.toString(), val);
}
};
@@ -132,19 +134,19 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
FacetMerger merger = null;
FacetMerger.Context ctx = new FacetMerger.Context(nShards);
- for (int i=0; i<nShards; i++) {
+ for (int i = 0; i < nShards; i++) {
Object response = fromJSON(responsesAndTests[i]);
- if (i==0) {
+ if (i == 0) {
merger = facetRequest.createFacetMerger(response);
}
- ctx.newShard("s"+i);
+ ctx.newShard("s" + i);
merger.merge(response, ctx);
}
- for (int i=0; i<nShards; i++) {
- ctx.setShard("s"+i);
+ for (int i = 0; i < nShards; i++) {
+ ctx.setShard("s" + i);
Object refinement = merger.getRefinement(ctx);
- String tests = responsesAndTests[nShards+i];
+ String tests = responsesAndTests[nShards + i];
match(refinement, 1e-5, tests);
}
@@ -161,7 +163,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
"{x: {buckets:[{val:x2, count:4}, {val:x3, count:2}] } }", // shard1 response
null, // shard0 expected refinement info
"=={x:{_l:[x1]}}" // shard1 expected refinement info
- );
+ );
// same test w/o refinement turned on
doTestRefine("{x : {type:terms, field:X, limit:2} }", // the facet request
@@ -236,25 +238,25 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
@Test
public void testBasicRefinement() throws Exception {
- ModifiableSolrParams p = params("cat_s", "cat_s", "cat_i", "cat_i", "xy_s", "xy_s", "num_d", "num_d", "qw_s", "qw_s", "er_s","er_s");
- doBasicRefinement( p );
+ ModifiableSolrParams p = params("cat_s", "cat_s", "cat_i", "cat_i", "xy_s", "xy_s", "num_d", "num_d", "qw_s", "qw_s", "er_s", "er_s");
+ doBasicRefinement(p);
- p.set("terms","method:dvhash,");
- doBasicRefinement( p );
+ p.set("terms", "method:dvhash,");
+ doBasicRefinement(p);
// multi-valued
- p = params("cat_s", "cat_ss", "cat_i", "cat_is", "xy_s", "xy_ss", "num_d", "num_d", "qw_s", "qw_ss", "er_s","er_ss");
- doBasicRefinement( p );
+ p = params("cat_s", "cat_ss", "cat_i", "cat_is", "xy_s", "xy_ss", "num_d", "num_d", "qw_s", "qw_ss", "er_s", "er_ss");
+ doBasicRefinement(p);
// single valued docvalues
- p = params("cat_s", "cat_sd", "cat_i", "cat_id", "xy_s", "xy_sd", "num_d", "num_dd", "qw_s", "qw_sd", "er_s","er_sd");
- doBasicRefinement( p );
+ p = params("cat_s", "cat_sd", "cat_i", "cat_id", "xy_s", "xy_sd", "num_d", "num_dd", "qw_s", "qw_sd", "er_s", "er_sd");
+ doBasicRefinement(p);
}
public void doBasicRefinement(ModifiableSolrParams p) throws Exception {
initServers();
Client client = servers.getClient(random().nextInt());
- client.queryDefaults().set( "shards", servers.getShards(), "debugQuery", Boolean.toString(random().nextBoolean()) );
+ client.queryDefaults().set("shards", servers.getShards(), "debugQuery", Boolean.toString(random().nextBoolean()));
List<SolrClient> clients = client.getClientProvider().all();
assertTrue(clients.size() >= 3);
@@ -268,16 +270,16 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
String er_s = p.get("er_s"); // this field is designed to test numBuckets refinement... the first phase will only have a single bucket returned for the top count bucket of cat_s
String num_d = p.get("num_d");
- clients.get(0).add( sdoc("id", "01", "all_s","all", cat_s, "A", cat_i,1, xy_s, "X" ,num_d, -1, qw_s, "Q", er_s,"E") ); // A wins count tie
- clients.get(0).add( sdoc("id", "02", "all_s","all", cat_s, "B", cat_i,2, xy_s, "Y", num_d, 3 ) );
+ clients.get(0).add(sdoc("id", "01", "all_s", "all", cat_s, "A", cat_i, 1, xy_s, "X", num_d, -1, qw_s, "Q", er_s, "E")); // A wins count tie
+ clients.get(0).add(sdoc("id", "02", "all_s", "all", cat_s, "B", cat_i, 2, xy_s, "Y", num_d, 3));
- clients.get(1).add( sdoc("id", "11", "all_s","all", cat_s, "B", cat_i,2, xy_s, "X", num_d, -5 , er_s,"E") ); // B highest count
- clients.get(1).add( sdoc("id", "12", "all_s","all", cat_s, "B", cat_i,2, xy_s, "Y", num_d, -11, qw_s, "W" ) );
- clients.get(1).add( sdoc("id", "13", "all_s","all", cat_s, "A", cat_i,1, xy_s, "X", num_d, 7 , er_s,"R") ); // "R" will only be picked up via refinement when parent facet is cat_s
+ clients.get(1).add(sdoc("id", "11", "all_s", "all", cat_s, "B", cat_i, 2, xy_s, "X", num_d, -5, er_s, "E")); // B highest count
+ clients.get(1).add(sdoc("id", "12", "all_s", "all", cat_s, "B", cat_i, 2, xy_s, "Y", num_d, -11, qw_s, "W"));
+ clients.get(1).add(sdoc("id", "13", "all_s", "all", cat_s, "A", cat_i, 1, xy_s, "X", num_d, 7, er_s, "R")); // "R" will only be picked up via refinement when parent facet is cat_s
- clients.get(2).add( sdoc("id", "21", "all_s","all", cat_s, "A", cat_i,1, xy_s, "X", num_d, 17, qw_s, "W", er_s,"E") ); // A highest count
- clients.get(2).add( sdoc("id", "22", "all_s","all", cat_s, "A", cat_i,1, xy_s, "Y", num_d, -19 ) );
- clients.get(2).add( sdoc("id", "23", "all_s","all", cat_s, "B", cat_i,2, xy_s, "X", num_d, 11 ) );
+ clients.get(2).add(sdoc("id", "21", "all_s", "all", cat_s, "A", cat_i, 1, xy_s, "X", num_d, 17, qw_s, "W", er_s, "E")); // A highest count
+ clients.get(2).add(sdoc("id", "22", "all_s", "all", cat_s, "A", cat_i, 1, xy_s, "Y", num_d, -19));
+ clients.get(2).add(sdoc("id", "23", "all_s", "all", cat_s, "B", cat_i, 2, xy_s, "X", num_d, 11));
client.commit();
@@ -285,16 +287,16 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
// One shard will have _facet_={"refine":{"cat0":{"_l":["A"]}}} on the second phase
/****
- // fake a refinement request... good for development/debugging
- assertJQ(clients.get(1),
- params(p, "q", "*:*", "_facet_","{refine:{cat0:{_l:[A]}}}", "isShard","true", "distrib","false", "shards.purpose","2097216", "ids","11,12,13",
- "json.facet", "{" +
- "cat0:{type:terms, field:cat_s, sort:'count desc', limit:1, overrequest:0, refine:true}" +
- "}"
- )
- , "facets=={foo:555}"
- );
- ****/
+ // fake a refinement request... good for development/debugging
+ assertJQ(clients.get(1),
+ params(p, "q", "*:*", "_facet_","{refine:{cat0:{_l:[A]}}}", "isShard","true", "distrib","false", "shards.purpose","2097216", "ids","11,12,13",
+ "json.facet", "{" +
+ "cat0:{type:terms, field:cat_s, sort:'count desc', limit:1, overrequest:0, refine:true}" +
+ "}"
+ )
+ , "facets=={foo:555}"
+ );
+ ****/
client.testJQ(params(p, "q", "*:*",
"json.facet", "{" +
@@ -473,51 +475,69 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
final String sort_limit_over = "sort:'count desc', limit:1, overrequest:0, ";
// simplistic join domain testing: no refinement == low count
client.testJQ(params(p, "q", "${xy_s}:Y", // query only matches one doc per shard
- "json.facet", "{"+
- " cat0:{${terms} type:terms, field:${cat_s}, "+sort_limit_over+" refine:false,"+
- // self join on all_s ensures every doc on every shard included in facets
- " domain: { join: { from:all_s, to:all_s } } }" +
- "}"
- )
- ,
- "/response/numFound==3",
- "facets=={ count:3, " +
- // w/o overrequest and refinement, count for 'A' is lower than it should be
- // (we don't see the A from the middle shard)
- " cat0:{ buckets:[ {val:A,count:3} ] } }");
+ "json.facet", "{" +
+ " cat0:{${terms} type:terms, field:${cat_s}, " + sort_limit_over + " refine:false," +
+ // self join on all_s ensures every doc on every shard included in facets
+ " domain: { join: { from:all_s, to:all_s } } }" +
+ "}"
+ )
+ ,
+ "/response/numFound==3",
+ "facets=={ count:3, " +
+ // w/o overrequest and refinement, count for 'A' is lower than it should be
+ // (we don't see the A from the middle shard)
+ " cat0:{ buckets:[ {val:A,count:3} ] } }");
// simplistic join domain testing: refinement == correct count
client.testJQ(params(p, "q", "${xy_s}:Y", // query only matches one doc per shard
- "json.facet", "{" +
- " cat0:{${terms} type:terms, field:${cat_s}, "+sort_limit_over+" refine:true,"+
- // self join on all_s ensures every doc on every shard included in facets
- " domain: { join: { from:all_s, to:all_s } } }" +
- "}"
- )
- ,
- "/response/numFound==3",
- "facets=={ count:3," +
- // w/o overrequest, we need refining to get the correct count for 'A'.
- " cat0:{ buckets:[ {val:A,count:4} ] } }");
+ "json.facet", "{" +
+ " cat0:{${terms} type:terms, field:${cat_s}, " + sort_limit_over + " refine:true," +
+ // self join on all_s ensures every doc on every shard included in facets
+ " domain: { join: { from:all_s, to:all_s } } }" +
+ "}"
+ )
+ ,
+ "/response/numFound==3",
+ "facets=={ count:3," +
+ // w/o overrequest, we need refining to get the correct count for 'A'.
+ " cat0:{ buckets:[ {val:A,count:4} ] } }");
// contrived join domain + refinement (at second level) + testing
client.testJQ(params(p, "q", "${xy_s}:Y", // query only matches one doc per shard
- "json.facet", "{" +
- // top level facet has a single term
- " all:{${terms} type:terms, field:all_s, "+sort_limit_over+" refine:true, " +
- " facet:{ "+
- // subfacet will facet on cat after joining on all (so all docs should be included in subfacet)
- " cat0:{${terms} type:terms, field:${cat_s}, "+sort_limit_over+" refine:true,"+
- " domain: { join: { from:all_s, to:all_s } } } } }" +
- "}"
- )
- ,
- "/response/numFound==3",
- "facets=={ count:3," +
- // all 3 docs matching base query have same 'all' value in top facet
- " all:{ buckets:[ { val:all, count:3, " +
- // sub facet has refinement, so count for 'A' should be correct
- " cat0:{ buckets: [{val:A,count:4}] } } ] } }");
+ "json.facet", "{" +
+ // top level facet has a single term
+ " all:{${terms} type:terms, field:all_s, " + sort_limit_over + " refine:true, " +
+ " facet:{ " +
+ // subfacet will facet on cat after joining on all (so all docs should be included in subfacet)
+ " cat0:{${terms} type:terms, field:${cat_s}, " + sort_limit_over + " refine:true," +
+ " domain: { join: { from:all_s, to:all_s } } } } }" +
+ "}"
+ )
+ ,
+ "/response/numFound==3",
+ "facets=={ count:3," +
+ // all 3 docs matching base query have same 'all' value in top facet
+ " all:{ buckets:[ { val:all, count:3, " +
+ // sub facet has refinement, so count for 'A' should be correct
+ " cat0:{ buckets: [{val:A,count:4}] } } ] } }");
+
+ }
+ // Unlike solrconfig.xml this test using solrconfig-tlog.xml should not fail with too-many-exceptions (see TestSolrQueryParser.testManyClauses)
+ @Test
+ public void testManyClauses() throws Exception {
+ String a = "1 a 2 b 3 c 10 d 11 12 "; // 10 terms
+ StringBuilder sb = new StringBuilder("id:(");
+ for (int i = 0; i < 1024; i++) { // historically, the max number of boolean clauses defaulted to 1024
+ sb.append('z').append(i).append(' ');
+ }
+ sb.append(a);
+ sb.append(")");
+
+ String q = sb.toString();
+
+ ignoreException("Too many clauses");
+ assertJQ(req("q", q)
+ , "/response/numFound==");
}
-
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml b/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
index 64d59fc..15d7ab4 100644
--- a/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
@@ -389,18 +389,11 @@
Query section - these settings control query time things like caches
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
<query>
- <!-- Max Boolean Clauses
- Maximum number of clauses in each BooleanQuery, an exception
- is thrown if exceeded.
-
- ** WARNING **
-
- This option actually modifies a global Lucene property that
- will affect all SolrCores. If multiple solrconfig.xml files
- disagree on this property, the value at any given moment will
- be based on the last SolrCore to be initialized.
-
+ <!-- Maximum number of clauses in each BooleanQuery, an exception
+ is thrown if exceeded. It is safe to increase or remove this setting,
+ since it is purely an arbitrary limit to try and catch user errors where
+ large boolean queries may not be the best implementation choice.
-->
<maxBooleanClauses>1024</maxBooleanClauses>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
index 1cb477b..8fbf839 100644
--- a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
@@ -389,22 +389,14 @@
Query section - these settings control query time things like caches
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
<query>
- <!-- Max Boolean Clauses
- Maximum number of clauses in each BooleanQuery, an exception
- is thrown if exceeded.
-
- ** WARNING **
-
- This option actually modifies a global Lucene property that
- will affect all SolrCores. If multiple solrconfig.xml files
- disagree on this property, the value at any given moment will
- be based on the last SolrCore to be initialized.
-
+ <!-- Maximum number of clauses in each BooleanQuery, an exception
+ is thrown if exceeded. It is safe to increase or remove this setting,
+ since it is purely an arbitrary limit to try and catch user errors where
+ large boolean queries may not be the best implementation choice.
-->
<maxBooleanClauses>1024</maxBooleanClauses>
-
<!-- Solr Internal Query Caches
There are two implementations of cache available for Solr,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98276481/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
index b1ad3a5..73899a6 100644
--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
@@ -395,18 +395,11 @@
Query section - these settings control query time things like caches
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
<query>
- <!-- Max Boolean Clauses
- Maximum number of clauses in each BooleanQuery, an exception
- is thrown if exceeded.
-
- ** WARNING **
-
- This option actually modifies a global Lucene property that
- will affect all SolrCores. If multiple solrconfig.xml files
- disagree on this property, the value at any given moment will
- be based on the last SolrCore to be initialized.
-
+ <!-- Maximum number of clauses in each BooleanQuery, an exception
+ is thrown if exceeded. It is safe to increase or remove this setting,
+ since it is purely an arbitrary limit to try and catch user errors where
+ large boolean queries may not be the best implementation choice.
-->
<maxBooleanClauses>1024</maxBooleanClauses>