You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2016/12/30 09:38:42 UTC
lucene-solr:branch_6x: SOLR-9620: fix cross core query time join by
numeric fields
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x f6a3557ee -> 6168a5d63
SOLR-9620: fix cross core query time join by numeric fields
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6168a5d6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6168a5d6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6168a5d6
Branch: refs/heads/branch_6x
Commit: 6168a5d63e43efd4d193b9ad6fae1f372794a1c8
Parents: f6a3557
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Wed Dec 28 23:45:07 2016 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Fri Dec 30 12:17:54 2016 +0300
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../search/join/ScoreJoinQParserPlugin.java | 46 ++++++++++-------
.../conf/schema-minimal-numeric-join.xml | 45 +++++++++++++++++
.../configsets/minimal/conf/schema-join.xml | 29 -----------
.../test/org/apache/solr/TestCrossCoreJoin.java | 52 ++++++++++++--------
5 files changed, 105 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6168a5d6/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d42a987..8aec7f8 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -234,6 +234,8 @@ Bug Fixes
* SOLR-9900: fix false positives on range queries with ReversedWildcardFilterFactory (Yonik Seeley via Mikhail Khludnev)
+* SOLR-9620: fix cross core query-time join by numeric fields (Mikhail Khludnev)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6168a5d6/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
index 45728e8..6f3f8c9 100644
--- a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
@@ -21,6 +21,7 @@ import java.util.Map;
import java.util.Objects;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldType.LegacyNumericType;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
@@ -40,6 +41,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.JoinQParserPlugin;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
@@ -80,10 +82,9 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
private final long fromCoreOpenTime;
public OtherCoreJoinQuery(Query fromQuery, String fromField,
- String fromIndex, FieldType.LegacyNumericType numericType,
- long fromCoreOpenTime, ScoreMode scoreMode,
+ String fromIndex, long fromCoreOpenTime, ScoreMode scoreMode,
String toField) {
- super(fromQuery, fromField, toField, numericType, scoreMode);
+ super(fromQuery, fromField, toField, scoreMode);
this.fromIndex = fromIndex;
this.fromCoreOpenTime = fromCoreOpenTime;
}
@@ -103,7 +104,12 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
fromHolder = fromCore.getRegisteredSearcher();
final Query joinQuery;
try {
- joinQuery = createJoinQuery(fromHolder.get());
+ SolrIndexSearcher searcher = fromHolder.get();
+
+ FieldType.LegacyNumericType fromNumericType = getNumericType(searcher.getSchema(),
+ info.getReq().getSchema());
+
+ joinQuery = createJoinQuery(searcher, fromNumericType);
} finally {
fromCore.close();
fromHolder.decref();
@@ -148,26 +154,24 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
protected final ScoreMode scoreMode;
protected final String fromField;
protected final String toField;
- protected final FieldType.LegacyNumericType numericType;
SameCoreJoinQuery(Query fromQuery, String fromField, String toField,
- FieldType.LegacyNumericType numericType,
ScoreMode scoreMode) {
this.fromQuery = fromQuery;
this.scoreMode = scoreMode;
this.fromField = fromField;
this.toField = toField;
- this.numericType = numericType;
}
@Override
public Query rewrite(IndexReader reader) throws IOException {
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
- final Query jq = createJoinQuery(info.getReq().getSearcher());
+ IndexSchema schema = info.getReq().getSchema();
+ final Query jq = createJoinQuery(info.getReq().getSearcher(), getNumericType(schema, schema));
return jq.rewrite(reader);
}
- protected Query createJoinQuery(IndexSearcher searcher) throws IOException {
+ protected Query createJoinQuery(IndexSearcher searcher, LegacyNumericType numericType) throws IOException {
if (numericType != null) {
return JoinUtil.createJoinQuery(fromField, true,
toField,numericType, fromQuery, searcher, scoreMode);
@@ -177,6 +181,16 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
}
}
+ protected FieldType.LegacyNumericType getNumericType(IndexSchema fromSchema, IndexSchema toSchema) {
+ FieldType.LegacyNumericType fromNumericType = fromSchema.getField(fromField).getType().getNumericType();
+ FieldType.LegacyNumericType toNumericType = toSchema.getField(toField).getType().getNumericType();
+ // Both will equals to null if they are not numeric type
+ if (fromNumericType != toNumericType) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "From and to field must have same numeric type. Found from=" +fromNumericType+" to=" + toNumericType);
+ }
+ return fromNumericType;
+ }
@Override
public String toString(String field) {
@@ -232,13 +246,6 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
private Query createQuery(final String fromField, final String fromQueryStr,
String fromIndex, final String toField, final ScoreMode scoreMode,
boolean byPassShortCircutCheck) throws SyntaxError {
- FieldType.LegacyNumericType fromNumericType = req.getSchema().getField(fromField).getType().getNumericType();
- FieldType.LegacyNumericType toNumericType = req.getSchema().getField(toField).getType().getNumericType();
- // Both will equals to null if they are not numeric type
- if (fromNumericType != toNumericType) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "From and to field must have same numeric type. Found from=" +fromNumericType+" to=" + toNumericType);
- }
final String myCore = req.getCore().getCoreDescriptor().getName();
@@ -262,9 +269,10 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
fromHolder = fromCore.getRegisteredSearcher();
if (fromHolder != null) {
- fromCoreOpenTime = fromHolder.get().getOpenNanoTime();
+ SolrIndexSearcher solrIndexSearcher = fromHolder.get();
+ fromCoreOpenTime = solrIndexSearcher.getOpenNanoTime();
}
- return new OtherCoreJoinQuery(fromQuery, fromField, coreName, fromNumericType,
+ return new OtherCoreJoinQuery(fromQuery, fromField, coreName,
fromCoreOpenTime, scoreMode, toField);
} finally {
otherReq.close();
@@ -274,7 +282,7 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
} else {
QParser fromQueryParser = subQuery(fromQueryStr, null);
final Query fromQuery = fromQueryParser.getQuery();
- return new SameCoreJoinQuery(fromQuery, fromField, toField, fromNumericType, scoreMode);
+ return new SameCoreJoinQuery(fromQuery, fromField, toField, scoreMode);
}
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6168a5d6/solr/core/src/test-files/solr/collection1/conf/schema-minimal-numeric-join.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-minimal-numeric-join.xml b/solr/core/src/test-files/solr/collection1/conf/schema-minimal-numeric-join.xml
new file mode 100644
index 0000000..8e62871
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-minimal-numeric-join.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- The Solr schema file, version 1.6 -->
+
+<schema name="test-numeric-join" version="1.6">
+
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+
+ <fieldType name="nametext" class="solr.TextField">
+ <analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
+ </fieldType>
+
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
+ <field name="name" type="nametext" indexed="true" stored="true"/>
+ <field name="text" type="nametext" indexed="true" stored="true"/>
+ <field name="cat" type="string" indexed="true" stored="true"/>
+ <field name="dept_id" type="string" indexed="true" stored="true"/>
+ <field name="rel_int" type="int" indexed="true" stored="true"/>
+
+ <field name="rel_long" type="long" indexed="true" stored="true"/>
+
+ <field name="rel_from_long" type="long" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6168a5d6/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml b/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml
deleted file mode 100644
index a2c5d15..0000000
--- a/solr/core/src/test-files/solr/configsets/minimal/conf/schema-join.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<schema name="minimal" version="1.1">
- <types>
- <fieldType name="string" class="solr.StrField"/>
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
- </types>
- <fields>
- <dynamicField name="*_i_dv" type="int" indexed="true" stored="false" docValues="true"/>
- <dynamicField name="*_l_dv" type="long" indexed="true" stored="false" docValues="true"/>
- <dynamicField name="*" type="string" indexed="true" stored="true" />
- </fields>
-</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6168a5d6/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
index f3c222e..25b45d7 100644
--- a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
+++ b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
@@ -17,6 +17,7 @@
package org.apache.solr;
import java.io.StringWriter;
+import java.nio.file.FileSystems;
import java.util.Collections;
import com.google.common.collect.ImmutableMap;
@@ -38,19 +39,25 @@ import static org.apache.solr.search.join.TestScoreJoinQPNoScore.whateverScore;
public class TestCrossCoreJoin extends SolrTestCaseJ4 {
+ private static final String rel_from_long = "rel_from_long";
+ private static final String rel_int = "rel_int";
+ private static final String rel_long = "rel_long";
+ private static final String name = "name";
+ private static final String cat = "cat";
private static SolrCore fromCore;
@BeforeClass
public static void beforeTests() throws Exception {
System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
-// initCore("solrconfig.xml","schema12.xml");
- // File testHome = createTempDir().toFile();
- // FileUtils.copyDirectory(getFile("solrj/solr"), testHome);
initCore("solrconfig-basic.xml", "schema-docValuesJoin.xml", TEST_HOME(), "collection1");
final CoreContainer coreContainer = h.getCoreContainer();
- fromCore = coreContainer.create("fromCore", ImmutableMap.of("configSet", "minimal", "schema", "schema-join.xml"));
+ fromCore = coreContainer.create("fromCore",
+ FileSystems.getDefault().getPath(TEST_HOME(),DEFAULT_TEST_CORENAME),
+ ImmutableMap.of("config", "solrconfig-basic.xml",
+ "schema", "schema-minimal-numeric-join.xml"),
+ true);
assertU(add(doc("id", "1", "uid_l_dv", "1", "name_s", "john",
"title_s", "Director", "dept_ss", "Engineering", "u_role_i_dv", "1")));
@@ -65,17 +72,17 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
assertU(commit());
- update(fromCore, add(doc("id", "15", "rel_from_l_dv", "1","rel_to_l_dv", "2")));
- update(fromCore, add(doc("id", "16", "rel_from_l_dv", "2","rel_to_l_dv", "1")));
+ update(fromCore, add(doc("id", "15", rel_from_long, "1",rel_long, "2")));
+ update(fromCore, add(doc("id", "16", rel_from_long, "2",rel_long, "1")));
- update(fromCore, add(doc("id", "20", "role_i_dv", "1", "name_s", "admin")));
- update(fromCore, add(doc("id", "21", "role_i_dv", "2", "name_s", "mod")));
- update(fromCore, add(doc("id", "22", "role_i_dv", "3", "name_s", "user")));
+ update(fromCore, add(doc("id", "20", rel_int, "1", "name", "admin")));
+ update(fromCore, add(doc("id", "21", rel_int, "2", "name", "mod")));
+ update(fromCore, add(doc("id", "22", rel_int, "3", "name", "user")));
- update(fromCore, add(doc("id", "10", "dept_id_s", "Engineering", "text", "These guys develop stuff", "cat", "dev")));
- update(fromCore, add(doc("id", "11", "dept_id_s", "Marketing", "text", "These guys make you look good")));
- update(fromCore, add(doc("id", "12", "dept_id_s", "Sales", "text", "These guys sell stuff")));
- update(fromCore, add(doc("id", "13", "dept_id_s", "Support", "text", "These guys help customers")));
+ update(fromCore, add(doc("id", "10", "dept_id", "Engineering", "text", "These guys develop stuff", cat, "dev")));
+ update(fromCore, add(doc("id", "11", "dept_id", "Marketing", "text", "These guys make you look good")));
+ update(fromCore, add(doc("id", "12", "dept_id", "Sales", "text", "These guys sell stuff")));
+ update(fromCore, add(doc("id", "13", "dept_id", "Support", "text", "These guys help customers")));
update(fromCore, commit());
}
@@ -84,19 +91,19 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
public void testJoinNumeric() throws Exception {
// Test join long field
assertJQ(req("q","{!join fromIndex=fromCore " +
- "from=rel_to_l_dv to=uid_l_dv"+ whateverScore()+"}rel_from_l_dv:1", "fl","id")
+ "from="+rel_long+" to=uid_l_dv"+ whateverScore()+"}"+rel_from_long+":1", "fl","id")
,"/response=={'numFound':1,'start':0,'docs':[{'id':'2'}]}"
);
// Test join int field
assertJQ(req("q","{!join fromIndex=fromCore " +
- "from=role_i_dv to=u_role_i_dv"+whateverScore()+"}name_s:admin", "fl","id")
+ "from="+rel_int+" to=u_role_i_dv"+whateverScore()+"}name:admin", "fl","id")
,"/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'2'}]}"
);
// Test join int field
assertQEx("From and to field must have same numeric type",req("q","{!join fromIndex=fromCore " +
- "from=role_i_dv to=uid_l_dv"+whateverScore()+"}name_s:admin", "fl","id")
+ "from="+rel_int+" to=uid_l_dv"+whateverScore()+"}name:admin", "fl","id")
,ErrorCode.BAD_REQUEST
);
}
@@ -119,23 +126,26 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 {
}
void doTestJoin(String joinPrefix) throws Exception {
- assertJQ(req("q", joinPrefix + " to=dept_ss from=dept_id_s fromIndex=fromCore}cat:dev", "fl", "id",
+ assertJQ(req("q", joinPrefix + " to=dept_ss from=dept_id fromIndex=fromCore}"+cat+":dev", "fl", "id",
"debugQuery", random().nextBoolean() ? "true":"false")
, "/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
);
// find people that develop stuff - but limit via filter query to a name of "john"
// this tests filters being pushed down to queries (SOLR-3062)
- assertJQ(req("q", joinPrefix + " to=dept_ss from=dept_id_s fromIndex=fromCore}cat:dev", "fl", "id", "fq", "name_s:john",
+ assertJQ(req("q", joinPrefix + " to=dept_ss from=dept_id fromIndex=fromCore}"+cat+":dev", "fl", "id", "fq", "name_s:john",
"debugQuery", random().nextBoolean() ? "true":"false")
, "/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}"
);
}
@Test
- public void testCoresAreDifferent() throws Exception {
- assertQEx("'to' schema" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST);
- final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "/select", 0, 100, Collections.emptyMap());
+ public void testSchemasAreDifferent() throws Exception {
+ for (String field: new String[]{cat, name, rel_long, rel_int}) {
+ assertQEx("'to' schema" + " has no \""+field+"\" field", req(field+":*"), ErrorCode.BAD_REQUEST);
+ }
+
+ final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, cat+":*", "standard", 0, 100, Collections.emptyMap());
final String resp = query(fromCore, req);
assertTrue(resp, resp.contains("numFound=\"1\""));
assertTrue(resp, resp.contains("<str name=\"id\">10</str>"));