You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2019/12/28 20:12:24 UTC
[lucene-solr] branch master updated: SOLR-13808: caching {!bool
filter=..} by default.
This is an automated email from the ASF dual-hosted git repository.
mkhl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 3f29fe0 SOLR-13808: caching {!bool filter=..} by default.
3f29fe0 is described below
commit 3f29fe0b804baf0a40af378441c82ee7c6b8ec19
Author: Mikhail Khludnev <mk...@apache.org>
AuthorDate: Sun Oct 27 19:45:23 2019 +0100
SOLR-13808: caching {!bool filter=..} by default.
---
solr/CHANGES.txt | 4 ++
.../org/apache/solr/search/BoolQParserPlugin.java | 14 +++-
.../apache/solr/search/json/TestJsonRequest.java | 84 ++++++++++++++++++----
solr/solr-ref-guide/src/other-parsers.adoc | 2 +-
4 files changed, 90 insertions(+), 14 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 02933b7..5c6649d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -150,6 +150,8 @@ Upgrade Notes
If you prefer to keep the old (but insecure) serialization strategy, you can start your nodes using the
property: `-Dsolr.useUnsafeOverseerResponse=true`. Keep in mind that this will be removed in future version of Solr.
+ * SOLR-13808: add cache=false into uderneath BoolQParser's filter clause or {"bool":{"filter":..}} to avoid caching in
+ filterCache. (Mikhail Khludnev)
New Features
---------------------
@@ -178,6 +180,8 @@ Improvements
* SOLR-13984: Java's SecurityManager sandbox can be enabled via environment variable,
SOLR_SECURITY_MANAGER_ENABLED=true. (rmuir)
+* SOLR-13808: filter in BoolQParser and {"bool":{"filter":..}} in Query DSL are cached by default (Mikhail Khludnev)
+
Optimizations
---------------------
(No changes)
diff --git a/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
index c0bebe5..04a9d5a 100644
--- a/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
@@ -21,6 +21,7 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.query.FilterQuery;
import org.apache.solr.request.SolrQueryRequest;
/**
@@ -49,7 +50,18 @@ public class BoolQParserPlugin extends QParserPlugin {
private void addQueries(BooleanQuery.Builder builder, String[] subQueries, BooleanClause.Occur occur) throws SyntaxError {
if (subQueries != null) {
for (String subQuery : subQueries) {
- builder.add(subQuery(subQuery, null).parse(), occur);
+ final QParser subParser = subQuery(subQuery, null);
+ Query extQuery;
+ if (BooleanClause.Occur.FILTER.equals(occur)) {
+ extQuery = subParser.getQuery();
+ if (!(extQuery instanceof ExtendedQuery) || (
+ ((ExtendedQuery) extQuery).getCache())) {
+ extQuery = new FilterQuery(extQuery);
+ }
+ } else {
+ extQuery = subParser.parse();
+ }
+ builder.add(extQuery, occur);
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
index 9f34db0..935bb0f 100644
--- a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
+++ b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
@@ -16,21 +16,30 @@
*/
package org.apache.solr.search.json;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.SolrTestCaseHS;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.CaffeineCache;
+import org.apache.solr.search.DocSet;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+
@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending"})
public class TestJsonRequest extends SolrTestCaseHS {
private static SolrInstances servers; // for distributed testing
+ @SuppressWarnings("deprecation")
@BeforeClass
public static void beforeTests() throws Exception {
systemSetPropertySolrDisableShardsWhitelist("true");
@@ -44,6 +53,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
}
}
+ @SuppressWarnings("deprecation")
@AfterClass
public static void afterTests() throws Exception {
JSONTestUtil.failRepeatedKeys = false;
@@ -54,7 +64,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
systemClearPropertySolrDisableShardsWhitelist();
}
- @Test
+ @Test @Repeat(iterations = 100)
public void testLocalJsonRequest() throws Exception {
doJsonRequest(Client.localClient, false);
}
@@ -303,17 +313,9 @@ public class TestJsonRequest extends SolrTestCaseHS {
, "response/numFound==1"
);
- client.testJQ( params("json","{ " +
- " query : {" +
- " bool : {" +
- " must : '{!lucene q.op=AND df=cat_s}A'" +
- " must_not : '{!lucene v=\\'id:1\\'}'" +
- " }" +
- " }" +
- "}")
- , "response/numFound==1"
- );
-
+ assertCatANot1(client, "must");
+
+ testFilterCachingLocally(client);
client.testJQ( params("json","{" +
" query : '*:*'," +
@@ -407,6 +409,64 @@ public class TestJsonRequest extends SolrTestCaseHS {
}
+ private static void testFilterCachingLocally(Client client) throws Exception {
+ if(client.getClientProvider()==null) {
+ final SolrQueryRequest request = req();
+ try {
+ final CaffeineCache<Query,DocSet> filterCache = (CaffeineCache<Query,DocSet>) request.getSearcher().getFilterCache();
+ filterCache.clear();
+ final TermQuery catA = new TermQuery(new Term("cat_s", "A"));
+ assertNull("cache is empty",filterCache.get(catA));
+
+ if(random().nextBoolean()) {
+ if(random().nextBoolean()) {
+ if(random().nextBoolean()) {
+ assertCatANot1(client, "must");
+ }else {
+ assertCatANot1(client, "must", "cat_s:A");
+ }
+ } else {
+ assertCatANot1(client, "must","{!lucene q.op=AND df=cat_s "+"cache="+random().nextBoolean()+"}A" );
+ }
+ } else {
+ assertCatANot1(client, "filter", "{!lucene q.op=AND df=cat_s cache=false}A");
+ }
+ assertNull("no cache still",filterCache.get(catA));
+
+ if (random().nextBoolean()) {
+ if (random().nextBoolean()) {
+ assertCatANot1(client, "filter", "cat_s:A");
+ } else {
+ assertCatANot1(client, "filter");
+ }
+ } else {
+ assertCatANot1(client, "filter","{!lucene q.op=AND df=cat_s cache=true}A");
+ }
+ assertNotNull("got cached ",filterCache.get(catA));
+
+ } finally {
+ request.close();
+ }
+ }
+ }
+
+ private static void assertCatANot1(Client client, final String occur) throws Exception {
+ assertCatANot1(client, occur, "{!lucene q.op=AND df=cat_s}A");
+ }
+
+ private static void assertCatANot1(Client client, final String occur, String catAclause) throws Exception {
+ client.testJQ( params("json","{ " +
+ " query : {" +
+ " bool : {" +
+ " " + occur + " : '"+ catAclause+ "'" +
+ " must_not : '{!lucene v=\\'id:1\\'}'" +
+ " }" +
+ " }" +
+ "}")
+ , "response/numFound==1"
+ );
+ }
+
public static void doJsonRequestWithTag(Client client) throws Exception {
addDocs(client);
diff --git a/solr/solr-ref-guide/src/other-parsers.adoc b/solr/solr-ref-guide/src/other-parsers.adoc
index 3996a9f..389138c 100644
--- a/solr/solr-ref-guide/src/other-parsers.adoc
+++ b/solr/solr-ref-guide/src/other-parsers.adoc
@@ -188,7 +188,7 @@ A list of queries that *must not* appear in matching documents.
A list of queries *should* appear in matching documents. For a BooleanQuery with no `must` queries, one or more `should` queries must match a document for the BooleanQuery to match.
`filter`::
-A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored.
+A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored. Also, these queries are cached in filter cache. To avoid caching add either `cache=false` as local parameter, or `"cache":"false"` property to underneath Query DLS Object.
*Examples*