You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2016/09/24 15:24:04 UTC
lucene-solr:branch_6x: SOLR-8395: join single value numerics.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 63ac156c7 -> eb10b2c26
SOLR-8395: join single value numerics.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eb10b2c2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eb10b2c2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eb10b2c2
Branch: refs/heads/branch_6x
Commit: eb10b2c2668819d1f803ee358595487a6989a640
Parents: 63ac156
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Fri Sep 23 22:46:28 2016 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Sat Sep 24 18:21:50 2016 +0300
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../search/join/ScoreJoinQParserPlugin.java | 39 ++++++++++---
.../configsets/minimal/conf/schema-join.xml | 29 ++++++++++
.../test/org/apache/solr/TestCrossCoreJoin.java | 59 ++++++++++++++++----
.../search/join/TestScoreJoinQPNoScore.java | 41 ++++++++++++++
5 files changed, 149 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb10b2c2/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8b7c546..93112e7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -60,6 +60,8 @@ New Features
* SOLR-9534: You can now set Solr's log level through environment variable SOLR_LOG_LEVEL
Also adds conveience arguments -q (quiet: WARN) and -v (verbose: DEBUG) to bin/solr (janhoy)
+
+* SOLR-8395: query time {!join} for single value numeric fields. (Cao Manh Dat via mkhl)
* SOLR-9537: Support facet scoring with the scoreNodes expression (Joel Bernstein)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb10b2c2/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
index 999cd64..45728e8 100644
--- a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
@@ -20,8 +20,10 @@ import java.io.IOException;
import java.util.Map;
import java.util.Objects;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.join.JoinUtil;
import org.apache.lucene.search.join.ScoreMode;
@@ -78,9 +80,10 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
private final long fromCoreOpenTime;
public OtherCoreJoinQuery(Query fromQuery, String fromField,
- String fromIndex, long fromCoreOpenTime, ScoreMode scoreMode,
+ String fromIndex, FieldType.LegacyNumericType numericType,
+ long fromCoreOpenTime, ScoreMode scoreMode,
String toField) {
- super(fromQuery, fromField, toField, scoreMode);
+ super(fromQuery, fromField, toField, numericType, scoreMode);
this.fromIndex = fromIndex;
this.fromCoreOpenTime = fromCoreOpenTime;
}
@@ -100,8 +103,7 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
fromHolder = fromCore.getRegisteredSearcher();
final Query joinQuery;
try {
- joinQuery = JoinUtil.createJoinQuery(fromField, true,
- toField, fromQuery, fromHolder.get(), scoreMode);
+ joinQuery = createJoinQuery(fromHolder.get());
} finally {
fromCore.close();
fromHolder.decref();
@@ -146,23 +148,35 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
protected final ScoreMode scoreMode;
protected final String fromField;
protected final String toField;
+ protected final FieldType.LegacyNumericType numericType;
SameCoreJoinQuery(Query fromQuery, String fromField, String toField,
+ FieldType.LegacyNumericType numericType,
ScoreMode scoreMode) {
this.fromQuery = fromQuery;
this.scoreMode = scoreMode;
this.fromField = fromField;
this.toField = toField;
+ this.numericType = numericType;
}
@Override
public Query rewrite(IndexReader reader) throws IOException {
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
- final Query jq = JoinUtil.createJoinQuery(fromField, true,
- toField, fromQuery, info.getReq().getSearcher(), scoreMode);
+ final Query jq = createJoinQuery(info.getReq().getSearcher());
return jq.rewrite(reader);
}
+ protected Query createJoinQuery(IndexSearcher searcher) throws IOException {
+ if (numericType != null) {
+ return JoinUtil.createJoinQuery(fromField, true,
+ toField,numericType, fromQuery, searcher, scoreMode);
+ } else {
+ return JoinUtil.createJoinQuery(fromField, true,
+ toField, fromQuery, searcher, scoreMode);
+ }
+ }
+
@Override
public String toString(String field) {
@@ -218,6 +232,13 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
private Query createQuery(final String fromField, final String fromQueryStr,
String fromIndex, final String toField, final ScoreMode scoreMode,
boolean byPassShortCircutCheck) throws SyntaxError {
+ FieldType.LegacyNumericType fromNumericType = req.getSchema().getField(fromField).getType().getNumericType();
+ FieldType.LegacyNumericType toNumericType = req.getSchema().getField(toField).getType().getNumericType();
+ // Both will equals to null if they are not numeric type
+ if (fromNumericType != toNumericType) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "From and to field must have same numeric type. Found from=" +fromNumericType+" to=" + toNumericType);
+ }
final String myCore = req.getCore().getCoreDescriptor().getName();
@@ -243,8 +264,8 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
if (fromHolder != null) {
fromCoreOpenTime = fromHolder.get().getOpenNanoTime();
}
- return new OtherCoreJoinQuery(fromQuery, fromField, coreName, fromCoreOpenTime,
- scoreMode, toField);
+ return new OtherCoreJoinQuery(fromQuery, fromField, coreName, fromNumericType,
+ fromCoreOpenTime, scoreMode, toField);
} finally {
otherReq.close();
fromCore.close();
@@ -253,7 +274,7 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
} else {
QParser fromQueryParser = subQuery(fromQueryStr, null);
final Query fromQuery = fromQueryParser.getQuery();
- return new SameCoreJoinQuery(fromQuery, fromField, toField, scoreMode);
+ return new SameCoreJoinQuery(fromQuery, fromField, toField, fromNumericType, scoreMode);
}
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb10b2c2/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml b/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml
new file mode 100644
index 0000000..a2c5d15
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+ <types>
+ <fieldType name="string" class="solr.StrField"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ </types>
+ <fields>
+ <dynamicField name="*_i_dv" type="int" indexed="true" stored="false" docValues="true"/>
+ <dynamicField name="*_l_dv" type="long" indexed="true" stored="false" docValues="true"/>
+ <dynamicField name="*" type="string" indexed="true" stored="true" />
+ </fields>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb10b2c2/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
index d15b3bc..f3c222e 100644
--- a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
+++ b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
@@ -29,12 +29,13 @@ import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.search.join.TestScoreJoinQPNoScore;
import org.apache.solr.servlet.DirectSolrConnection;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.apache.solr.search.join.TestScoreJoinQPNoScore.whateverScore;
+
public class TestCrossCoreJoin extends SolrTestCaseJ4 {
private static SolrCore fromCore;
@@ -46,18 +47,31 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
// File testHome = createTempDir().toFile();
// FileUtils.copyDirectory(getFile("solrj/solr"), testHome);
- initCore("solrconfig.xml", "schema12.xml", TEST_HOME(), "collection1");
+ initCore("solrconfig-basic.xml", "schema-docValuesJoin.xml", TEST_HOME(), "collection1");
final CoreContainer coreContainer = h.getCoreContainer();
- fromCore = coreContainer.create("fromCore", ImmutableMap.of("configSet", "minimal"));
+ fromCore = coreContainer.create("fromCore", ImmutableMap.of("configSet", "minimal", "schema", "schema-join.xml"));
+
+ assertU(add(doc("id", "1", "uid_l_dv", "1", "name_s", "john",
+ "title_s", "Director", "dept_ss", "Engineering", "u_role_i_dv", "1")));
+ assertU(add(doc("id", "2", "uid_l_dv", "2", "name_s", "mark",
+ "title_s", "VP", "dept_ss", "Marketing", "u_role_i_dv", "1")));
+ assertU(add(doc("id", "3", "uid_l_dv", "3", "name_s", "nancy",
+ "title_s", "MTS", "dept_ss", "Sales", "u_role_i_dv", "2")));
+ assertU(add(doc("id", "4", "uid_l_dv", "4", "name_s", "dave",
+ "title_s", "MTS", "dept_ss", "Support", "dept_ss", "Engineering", "u_role_i_dv", "3")));
+ assertU(add(doc("id", "5", "uid_l_dv", "5", "name_s", "tina",
+ "title_s", "VP", "dept_ss", "Engineering", "u_role_i_dv", "3")));
- assertU(add(doc("id", "1", "name", "john", "title", "Director", "dept_s", "Engineering")));
- assertU(add(doc("id", "2", "name", "mark", "title", "VP", "dept_s", "Marketing")));
- assertU(add(doc("id", "3", "name", "nancy", "title", "MTS", "dept_s", "Sales")));
- assertU(add(doc("id", "4", "name", "dave", "title", "MTS", "dept_s", "Support", "dept_s", "Engineering")));
- assertU(add(doc("id", "5", "name", "tina", "title", "VP", "dept_s", "Engineering")));
assertU(commit());
+ update(fromCore, add(doc("id", "15", "rel_from_l_dv", "1","rel_to_l_dv", "2")));
+ update(fromCore, add(doc("id", "16", "rel_from_l_dv", "2","rel_to_l_dv", "1")));
+
+ update(fromCore, add(doc("id", "20", "role_i_dv", "1", "name_s", "admin")));
+ update(fromCore, add(doc("id", "21", "role_i_dv", "2", "name_s", "mod")));
+ update(fromCore, add(doc("id", "22", "role_i_dv", "3", "name_s", "user")));
+
update(fromCore, add(doc("id", "10", "dept_id_s", "Engineering", "text", "These guys develop stuff", "cat", "dev")));
update(fromCore, add(doc("id", "11", "dept_id_s", "Marketing", "text", "These guys make you look good")));
update(fromCore, add(doc("id", "12", "dept_id_s", "Sales", "text", "These guys sell stuff")));
@@ -66,6 +80,27 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
}
+ @Test
+ public void testJoinNumeric() throws Exception {
+ // Test join long field
+ assertJQ(req("q","{!join fromIndex=fromCore " +
+ "from=rel_to_l_dv to=uid_l_dv"+ whateverScore()+"}rel_from_l_dv:1", "fl","id")
+ ,"/response=={'numFound':1,'start':0,'docs':[{'id':'2'}]}"
+ );
+
+ // Test join int field
+ assertJQ(req("q","{!join fromIndex=fromCore " +
+ "from=role_i_dv to=u_role_i_dv"+whateverScore()+"}name_s:admin", "fl","id")
+ ,"/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'2'}]}"
+ );
+
+ // Test join int field
+ assertQEx("From and to field must have same numeric type",req("q","{!join fromIndex=fromCore " +
+ "from=role_i_dv to=uid_l_dv"+whateverScore()+"}name_s:admin", "fl","id")
+ ,ErrorCode.BAD_REQUEST
+ );
+ }
+
public static String update(SolrCore core, String xml) throws Exception {
DirectSolrConnection connection = new DirectSolrConnection(core);
@@ -80,18 +115,18 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
@Test
public void testScoreJoin() throws Exception {
- doTestJoin("{!join " + TestScoreJoinQPNoScore.whateverScore());
+ doTestJoin("{!join " + whateverScore());
}
void doTestJoin(String joinPrefix) throws Exception {
- assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id",
+ assertJQ(req("q", joinPrefix + " to=dept_ss from=dept_id_s fromIndex=fromCore}cat:dev", "fl", "id",
"debugQuery", random().nextBoolean() ? "true":"false")
, "/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
);
// find people that develop stuff - but limit via filter query to a name of "john"
// this tests filters being pushed down to queries (SOLR-3062)
- assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id", "fq", "name:john",
+ assertJQ(req("q", joinPrefix + " to=dept_ss from=dept_id_s fromIndex=fromCore}cat:dev", "fl", "id", "fq", "name_s:john",
"debugQuery", random().nextBoolean() ? "true":"false")
, "/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}"
);
@@ -99,7 +134,7 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
@Test
public void testCoresAreDifferent() throws Exception {
- assertQEx("schema12.xml" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST);
+ assertQEx("'to' schema" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST);
final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "/select", 0, 100, Collections.emptyMap());
final String resp = query(fromCore, req);
assertTrue(resp, resp.contains("numFound=\"1\""));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb10b2c2/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
index 3b23be8..d60643f 100644
--- a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
+++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
@@ -32,6 +32,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
@@ -56,6 +57,46 @@ public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 {
initCore("solrconfig-basic.xml","schema-docValuesJoin.xml");
}
+ public void testJoinNumeric() throws Exception {
+ assertU(add(doc("id", "1", "uid_l_dv", "1","name_s", "john", "u_role_i_dv", "1")));
+ assertU(add(doc("id", "2", "uid_l_dv", "2","name_s", "mark", "u_role_i_dv", "1")));
+ assertU(add(doc("id", "3", "uid_l_dv", "3","name_s", "nacy", "u_role_i_dv", "2")));
+ assertU(add(doc("id", "4", "uid_l_dv", "4","name_s", "dave", "u_role_i_dv", "3")));
+
+ assertU(add(doc("id", "10", "rel_from_l_dv", "1","rel_to_l_dv", "2")));
+ assertU(add(doc("id", "11", "rel_from_l_dv", "2","rel_to_l_dv", "1")));
+
+ assertU(add(doc("id", "15", "role_i_dv", "1", "name_s", "admin")));
+ assertU(add(doc("id", "16", "role_i_dv", "2", "name_s", "mod")));
+ assertU(add(doc("id", "16", "role_i_dv", "3", "name_s", "user")));
+
+ assertU(commit());
+
+ // Test join long field
+ assertJQ(req("q","{!join from=rel_to_l_dv to=uid_l_dv"+whateverScore()+"}rel_from_l_dv:1", "fl","id")
+ ,"/response=={'numFound':1,'start':0,'docs':[{'id':'2'}]}"
+ );
+
+ assertJQ(req("q","{!join from=uid_l_dv to=rel_from_l_dv"+whateverScore()+"}name_s:john", "fl","id")
+ ,"/response=={'numFound':1,'start':0,'docs':[{'id':'10'}]}"
+ );
+
+ // Test join int field
+ assertJQ(req("q","{!join from=role_i_dv to=u_role_i_dv"+whateverScore()+"}name_s:admin", "fl","id")
+ ,"/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'2'}]}"
+ );
+
+ assertQEx("From and to field must have same numeric type",
+ req("q","{!join from=role_i_dv to=uid_l_dv"+whateverScore()+"}name_s:admin", "fl","id")
+ , SolrException.ErrorCode.BAD_REQUEST
+ );
+
+ assertQEx("From and to field must have same numeric type",
+ req("q","{!join from=role_i_dv to=id"+whateverScore()+"}name_s:admin", "fl","id")
+ , SolrException.ErrorCode.BAD_REQUEST
+ );
+ }
+
@Test
public void testJoin() throws Exception {
assertU(add(doc("id", "1","name_s", "john", "title_s", "Director", "dept_ss","Engineering")));