You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/09/02 15:43:11 UTC
[40/50] lucene-solr:feature/autoscaling: SOLR-11244: Query DSL for
Solr
SOLR-11244: Query DSL for Solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d3013ab6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d3013ab6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d3013ab6
Branch: refs/heads/feature/autoscaling
Commit: d3013ab600636c4cd958af3a395ef84e6570af6a
Parents: 63a0c8d
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Sep 1 20:21:43 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Sep 1 20:21:43 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solr/request/json/JsonQueryConverter.java | 110 ++++++++++++
.../apache/solr/request/json/RequestUtil.java | 10 +-
.../apache/solr/search/BoolQParserPlugin.java | 58 ++++++
.../org/apache/solr/search/QParserPlugin.java | 1 +
.../apache/solr/search/TestSmileRequest.java | 2 +-
.../solr/search/json/TestJsonRequest.java | 179 ++++++++++++++++++-
7 files changed, 354 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4884cf3..493d52f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -81,6 +81,8 @@ New Features
* SOLR-11215: Make a metric accessible through a single param. (ab)
+* SOLR-11244: Query DSL for Solr (Cao Manh Dat)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java b/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java
new file mode 100644
index 0000000..e732470
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.request.json;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+
+/**
+ * Convert json query object to local params.
+ *
+ * @lucene.internal
+ */
+class JsonQueryConverter {
+ private int numParams = 0;
+
+ String toLocalParams(Object jsonQueryObject, Map<String, String[]> additionalParams) {
+ if (jsonQueryObject instanceof String) return jsonQueryObject.toString();
+ StringBuilder builder = new StringBuilder();
+ buildLocalParams(builder, jsonQueryObject, true, additionalParams);
+ return builder.toString();
+ }
+
+ private String putParam(String val, Map<String, String[]> additionalParams) {
+ String name = "_tt"+(numParams++);
+ additionalParams.put(name, new String[]{val});
+ return name;
+ }
+
+ private void buildLocalParams(StringBuilder builder, Object val, boolean isQParser, Map<String, String[]> additionalParams) {
+ if (!isQParser && !(val instanceof Map)) {
+ // val is value of a query parser, and it is not a map
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Error when parsing json query, expect a json object here, but found : "+val);
+ }
+ if (val instanceof String) {
+ builder.append('$').append(putParam(val.toString(), additionalParams));
+ return;
+ }
+ if (val instanceof Number) {
+ builder.append(val);
+ return;
+ }
+ if (!(val instanceof Map)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Error when parsing json query, expect a json object here, but found : "+val);
+ }
+
+ Map<String,Object> map = (Map<String, Object>) val;
+ if (isQParser) {
+ if (map.size() != 1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Error when parsing json query, expect only one query parser here, but found : "+map.keySet());
+ }
+ String qtype = map.keySet().iterator().next();
+ Object subVal = map.get(qtype);
+
+ // We don't want to introduce unnecessary variable at root level
+ boolean useSubBuilder = builder.length() > 0;
+ StringBuilder subBuilder = builder;
+
+ if (useSubBuilder) subBuilder = new StringBuilder();
+
+ subBuilder = subBuilder.append("{!").append(qtype).append(' ');;
+ buildLocalParams(subBuilder, subVal, false, additionalParams);
+ subBuilder.append("}");
+
+ if (useSubBuilder) builder.append('$').append(putParam(subBuilder.toString(), additionalParams));
+ } else {
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ String key = entry.getKey();
+ if (entry.getValue() instanceof List) {
+ if (key.equals("query")) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Error when parsing json query, value of query field should not be a list, found : " + entry.getValue());
+ }
+ List l = (List) entry.getValue();
+ for (Object subVal : l) {
+ builder.append(key).append("=");
+ buildLocalParams(builder, subVal, true, additionalParams);
+ builder.append(" ");
+ }
+ } else {
+ if (key.equals("query")) {
+ key = "v";
+ }
+ builder.append(key).append("=");
+ buildLocalParams(builder, entry.getValue(), true, additionalParams);
+ builder.append(" ");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java b/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java
index ac0dc19..6e7e02a 100644
--- a/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java
+++ b/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java
@@ -190,16 +190,20 @@ public class RequestUtil {
}
// implement compat for existing components...
+ JsonQueryConverter jsonQueryConverter = new JsonQueryConverter();
if (json != null && !isShard) {
for (Map.Entry<String,Object> entry : json.entrySet()) {
String key = entry.getKey();
String out = null;
+ boolean isQuery = false;
boolean arr = false;
if ("query".equals(key)) {
out = "q";
+ isQuery = true;
} else if ("filter".equals(key)) {
out = "fq";
arr = true;
+ isQuery = true;
} else if ("fields".equals(key)) {
out = "fl";
arr = true;
@@ -230,14 +234,14 @@ public class RequestUtil {
if (lst != null) {
for (int i = 0; i < jsonSize; i++) {
Object v = lst.get(i);
- newval[existingSize + i] = v.toString();
+ newval[existingSize + i] = isQuery ? jsonQueryConverter.toLocalParams(v, newMap) : v.toString();
}
} else {
- newval[newval.length-1] = val.toString();
+ newval[newval.length-1] = isQuery ? jsonQueryConverter.toLocalParams(val, newMap) : val.toString();
}
newMap.put(out, newval);
} else {
- newMap.put(out, new String[]{val.toString()});
+ newMap.put(out, new String[]{isQuery ? jsonQueryConverter.toLocalParams(val, newMap) : val.toString()});
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
new file mode 100644
index 0000000..c0bebe5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+
+/**
+ * Create a boolean query from sub queries.
+ * Sub queries can be marked as must, must_not, filter or should
+ *
+ * <p>Example: <code>{!bool should=title:lucene should=title:solr must_not=id:1}</code>
+ */
+public class BoolQParserPlugin extends QParserPlugin {
+ public static final String NAME = "bool";
+
+ @Override
+ public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ return new QParser(qstr, localParams, params, req) {
+ @Override
+ public Query parse() throws SyntaxError {
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ SolrParams solrParams = SolrParams.wrapDefaults(localParams, params);
+ addQueries(builder, solrParams.getParams("must"), BooleanClause.Occur.MUST);
+ addQueries(builder, solrParams.getParams("must_not"), BooleanClause.Occur.MUST_NOT);
+ addQueries(builder, solrParams.getParams("filter"), BooleanClause.Occur.FILTER);
+ addQueries(builder, solrParams.getParams("should"), BooleanClause.Occur.SHOULD);
+ return builder.build();
+ }
+
+ private void addQueries(BooleanQuery.Builder builder, String[] subQueries, BooleanClause.Occur occur) throws SyntaxError {
+ if (subQueries != null) {
+ for (String subQuery : subQueries) {
+ builder.add(subQuery(subQuery, null).parse(), occur);
+ }
+ }
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index 2ee63cf..893783d 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -81,6 +81,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
map.put(SignificantTermsQParserPlugin.NAME, SignificantTermsQParserPlugin.class);
map.put(PayloadScoreQParserPlugin.NAME, PayloadScoreQParserPlugin.class);
map.put(PayloadCheckQParserPlugin.NAME, PayloadCheckQParserPlugin.class);
+ map.put(BoolQParserPlugin.NAME, BoolQParserPlugin.class);
standardPlugins = Collections.unmodifiableMap(map);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
index c6d72ee..2e157c7 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
@@ -81,7 +81,7 @@ public class TestSmileRequest extends SolrTestCaseJ4 {
}
};
client.queryDefaults().set("shards", servers.getShards());
- TestJsonRequest.doJsonRequest(client);
+ TestJsonRequest.doJsonRequest(client, false);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d3013ab6/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
index 9c151c1..4f47f8a 100644
--- a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
+++ b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
@@ -53,7 +53,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
@Test
public void testLocalJsonRequest() throws Exception {
- doJsonRequest(Client.localClient);
+ doJsonRequest(Client.localClient, false);
}
@Test
@@ -62,11 +62,10 @@ public class TestJsonRequest extends SolrTestCaseHS {
initServers();
Client client = servers.getClient( random().nextInt() );
client.queryDefaults().set( "shards", servers.getShards() );
- doJsonRequest(client);
+ doJsonRequest(client, true);
}
-
- public static void doJsonRequest(Client client) throws Exception {
+ public static void doJsonRequest(Client client, boolean isDistrib) throws Exception {
client.deleteByQuery("*:*", null);
client.add(sdoc("id", "1", "cat_s", "A", "where_s", "NY"), null);
client.add(sdoc("id", "2", "cat_s", "B", "where_s", "NJ"), null);
@@ -217,6 +216,178 @@ public class TestJsonRequest extends SolrTestCaseHS {
, "debug/json=={query:'cat_s:A', filter:'where_s:NY'}"
);
+ // test query dsl
+ client.testJQ( params("json", "{'query':'{!lucene}id:1'}")
+ , "response/numFound==1"
+ );
+
+ client.testJQ( params("json", "{" +
+ " 'query': {" +
+ " 'bool' : {" +
+ " 'should' : [" +
+ " {'lucene' : {'query' : 'id:1'}}," +
+ " 'id:2'" +
+ " ]" +
+ " }" +
+ " }" +
+ "}")
+ , "response/numFound==2"
+ );
+
+ client.testJQ( params("json", "{" +
+ " 'query': {" +
+ " 'bool' : {" +
+ " 'should' : [" +
+ " 'id:1'," +
+ " 'id:2'" +
+ " ]" +
+ " }" +
+ " }" +
+ "}")
+ , "response/numFound==2"
+ );
+
+ client.testJQ( params("json", "{ " +
+ " query : {" +
+ " boost : {" +
+ " query : {" +
+ " lucene : { " +
+ " df : cat_s, " +
+ " query : A " +
+ " }" +
+ " }, " +
+ " b : 1.5 " +
+ " } " +
+ " } " +
+ "}")
+ , "response/numFound==2"
+ );
+
+ client.testJQ( params("json","{ " +
+ " query : {" +
+ " bool : {" +
+ " must : {" +
+ " lucene : {" +
+ " q.op : AND," +
+ " df : cat_s," +
+ " query : A" +
+ " }" +
+ " }" +
+ " must_not : {lucene : {query:'id: 1'}}" +
+ " }" +
+ " }" +
+ "}")
+ , "response/numFound==1"
+ );
+
+ client.testJQ( params("json","{ " +
+ " query : {" +
+ " bool : {" +
+ " must : {" +
+ " lucene : {" +
+ " q.op : AND," +
+ " df : cat_s," +
+ " query : A" +
+ " }" +
+ " }" +
+ " must_not : [{lucene : {query:'id: 1'}}]" +
+ " }" +
+ " }" +
+ "}")
+ , "response/numFound==1"
+ );
+
+ client.testJQ( params("json","{ " +
+ " query : {" +
+ " bool : {" +
+ " must : '{!lucene q.op=AND df=cat_s}A'" +
+ " must_not : '{!lucene v=\\'id:1\\'}'" +
+ " }" +
+ " }" +
+ "}")
+ , "response/numFound==1"
+ );
+
+
+ client.testJQ( params("json","{" +
+ " query : '*:*'," +
+ " filter : {" +
+ " collapse : {" +
+ " field : cat_s" +
+ " } " +
+ " } " +
+ "}")
+ , isDistrib ? "" : "response/numFound==2"
+ );
+
+ client.testJQ( params("json","{" +
+ " query : {" +
+ " edismax : {" +
+ " query : 'A'," +
+ " qf : 'cat_s'," +
+ " bq : {" +
+ " edismax : {" +
+ " query : 'NJ'" +
+ " qf : 'where_s'" +
+ " }" +
+ " }" +
+ " }" +
+ " }, " +
+ " fields : id" +
+ "}")
+ , "response/numFound==2", isDistrib? "" : "response/docs==[{id:'4'},{id:'1'}]"
+ );
+
+ client.testJQ( params("json","{" +
+ " query : {" +
+ " edismax : {" +
+ " query : 'A'," +
+ " qf : 'cat_s'," +
+ " bq : {" +
+ " edismax : {" +
+ " query : 'NY'" +
+ " qf : 'where_s'" +
+ " }" +
+ " }" +
+ " }" +
+ " }, " +
+ " fields : id" +
+ "}")
+ , "response/numFound==2", isDistrib? "" : "response/docs==[{id:'1'},{id:'4'}]"
+ );
+
+ client.testJQ( params("json","{" +
+ " query : {" +
+ " dismax : {" +
+ " query : 'A NJ'" +
+ " qf : 'cat_s^0.1 where_s^100'" +
+ " } " +
+ " }, " +
+ " filter : '-id:2'," +
+ " fields : id" +
+ "}")
+ , "response/numFound==3", isDistrib? "" : "response/docs==[{id:'4'},{id:'5'},{id:'1'}]"
+ );
+
+ client.testJQ( params("json","{" +
+ " query : {" +
+ " dismax : {" +
+ " query : 'A NJ'" +
+ " qf : ['cat_s^100', 'where_s^0.1']" +
+ " } " +
+ " }, " +
+ " filter : '-id:2'," +
+ " fields : id" +
+ "}")
+ , "response/numFound==3", isDistrib? "" : "response/docs==[{id:'4'},{id:'1'},{id:'5'}]"
+ );
+
+ try {
+ client.testJQ(params("json", "{query:{'lucene':'id:1'}}"));
+ fail();
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("id:1"));
+ }
try {
// test failure on unknown parameter