You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2015/07/28 15:24:05 UTC
svn commit: r1693092 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/search/
core/src/java/org/apache/solr/search/join/
core/src/test-files/solr/collection1/conf/ core/src/test/org/apache/solr/
core/src/test/org/apache/solr/search/ core/s...
Author: mkhl
Date: Tue Jul 28 13:24:04 2015
New Revision: 1693092
URL: http://svn.apache.org/r1693092
Log:
SOLR-6234: Scoring for query time join
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java (with props)
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/common-build.xml
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1693092&r1=1693091&r2=1693092&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Jul 28 13:24:04 2015
@@ -65,6 +65,8 @@ Other Changes
* SOLR-7624: Remove deprecated zkCredientialsProvider element in solrcloud section of solr.xml.
(Xu Zhang, Per Steffensen, Ramkumar Aiyengar, Mark Miller)
+
+* SOLR-6234: Scoring for query time join (Mikhail Khludnev)
================== 5.3.0 ==================
Modified: lucene/dev/trunk/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/common-build.xml?rev=1693092&r1=1693091&r2=1693092&view=diff
==============================================================================
--- lucene/dev/trunk/solr/common-build.xml (original)
+++ lucene/dev/trunk/solr/common-build.xml Tue Jul 28 13:24:04 2015
@@ -320,6 +320,7 @@
<link offline="true" href="${lucene.javadoc.url}expressions" packagelistloc="${lucenedocs}/expressions"/>
<link offline="true" href="${lucene.javadoc.url}suggest" packagelistloc="${lucenedocs}/suggest"/>
<link offline="true" href="${lucene.javadoc.url}grouping" packagelistloc="${lucenedocs}/grouping"/>
+ <link offline="true" href="${lucene.javadoc.url}join" packagelistloc="${lucenedocs}/join"/>
<link offline="true" href="${lucene.javadoc.url}queries" packagelistloc="${lucenedocs}/queries"/>
<link offline="true" href="${lucene.javadoc.url}queryparser" packagelistloc="${lucenedocs}/queryparser"/>
<link offline="true" href="${lucene.javadoc.url}highlighter" packagelistloc="${lucenedocs}/highlighter"/>
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java?rev=1693092&r1=1693091&r2=1693092&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java Tue Jul 28 13:24:04 2015
@@ -60,6 +60,7 @@ import org.apache.solr.request.LocalSolr
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.TrieField;
+import org.apache.solr.search.join.ScoreJoinQParserPlugin;
import org.apache.solr.util.RefCounted;
public class JoinQParserPlugin extends QParserPlugin {
@@ -72,8 +73,17 @@ public class JoinQParserPlugin extends Q
@Override
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
+
@Override
public Query parse() throws SyntaxError {
+ if(localParams!=null && localParams.get(ScoreJoinQParserPlugin.SCORE)!=null){
+ return new ScoreJoinQParserPlugin().createParser(qstr, localParams, params, req).parse();
+ }else{
+ return parseJoin();
+ }
+ }
+
+ Query parseJoin() throws SyntaxError {
String fromField = getParam("from");
String fromIndex = getParam("fromIndex");
String toField = getParam("to");
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java?rev=1693092&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java Tue Jul 28 13:24:04 2015
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.join;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.JoinUtil;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.uninverting.UninvertingReader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.search.JoinQParserPlugin;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SyntaxError;
+import org.apache.solr.util.RefCounted;
+
+/**
+ * Create a query-time join query with scoring.
+ * It just calls {@link JoinUtil#createJoinQuery(String, boolean, String, Query, org.apache.lucene.search.IndexSearcher, ScoreMode)}.
+ * It runs subordinate query and collects values of "from" field and scores, then it lookups these collected values in "to" field, and
+ * yields aggregated scores.
+ * Local parameters are similar to {@link JoinQParserPlugin} <a href="http://wiki.apache.org/solr/Join">{!join}</a>
+ * This plugin doesn't have own name, and is called by specifying local parameter <code>{!join score=...}...</code>.
+ * Note: this parser is invoked even if you specify <code>score=none</code>.
+ * <br>Example:<code>q={!join from=manu_id_s to=id score=total}foo</code>
+ * <ul>
+ * <li>from - "foreign key" field name to collect values while enumerating subordinate query (denoted as <code>foo</code> in example above).
+ * it's better to have this field declared as <code>type="string" docValues="true"</code>.
+ * note: if <a href="http://wiki.apache.org/solr/DocValues">docValues</a> are not enabled for this field, it will work anyway,
+ * but it costs some memory for {@link UninvertingReader}.
+ * Also, numeric doc values are not supported until <a href="https://issues.apache.org/jira/browse/LUCENE-5868">LUCENE-5868</a>.
+ * Thus, it only supports {@link DocValuesType#SORTED}, {@link DocValuesType#SORTED_SET}, {@link DocValuesType#BINARY}. </li>
+ * <li>fromIndex - optional parameter, a core name where subordinate query should run (and <code>from</code> values are collected) rather than current core.
+ * <br>Example:<code>q={!join from=manu_id_s to=id score=total fromIndex=products}foo</code>
+ * <br>Follow up <a href="https://issues.apache.org/jira/browse/SOLR-7775">SOLR-7775</a> for SolrCloud collections support.</li>
+ * <li>to - "primary key" field name which is searched for values collected from subordinate query.
+ * it should be declared as <code>indexed="true"</code>. Now it's treated as a single value field.</li>
+ * <li>score - one of {@link ScoreMode}: None,Avg,Total,Max. Lowercase is also accepted.</li>
+ * </ul>
+ */
+public class ScoreJoinQParserPlugin extends QParserPlugin {
+
+ public static final String SCORE = "score";
+
+ static class OtherCoreJoinQuery extends SameCoreJoinQuery {
+ private final String fromIndex;
+ private final long fromCoreOpenTime;
+
+ public OtherCoreJoinQuery(Query fromQuery, String fromField,
+ String fromIndex, long fromCoreOpenTime, ScoreMode scoreMode,
+ String toField) {
+ super(fromQuery, fromField, toField, scoreMode);
+ this.fromIndex = fromIndex;
+ this.fromCoreOpenTime = fromCoreOpenTime;
+ }
+
+ @Override
+ public Query rewrite(IndexReader reader) throws IOException {
+ SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+
+ CoreContainer container = info.getReq().getCore().getCoreDescriptor().getCoreContainer();
+
+ final SolrCore fromCore = container.getCore(fromIndex);
+
+ if (fromCore == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex);
+ }
+ RefCounted<SolrIndexSearcher> fromHolder = null;
+ fromHolder = fromCore.getRegisteredSearcher();
+ final Query joinQuery;
+ try {
+ joinQuery = JoinUtil.createJoinQuery(fromField, true,
+ toField, fromQuery, fromHolder.get(), scoreMode);
+ } finally {
+ fromCore.close();
+ fromHolder.decref();
+ }
+ joinQuery.setBoost(getBoost());
+ return joinQuery.rewrite(reader);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result
+ + (int) (fromCoreOpenTime ^ (fromCoreOpenTime >>> 32));
+ result = prime * result
+ + ((fromIndex == null) ? 0 : fromIndex.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (!super.equals(obj)) return false;
+ if (getClass() != obj.getClass()) return false;
+ OtherCoreJoinQuery other = (OtherCoreJoinQuery) obj;
+ if (fromCoreOpenTime != other.fromCoreOpenTime) return false;
+ if (fromIndex == null) {
+ if (other.fromIndex != null) return false;
+ } else if (!fromIndex.equals(other.fromIndex)) return false;
+ return true;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "OtherCoreJoinQuery [fromIndex=" + fromIndex
+ + ", fromCoreOpenTime=" + fromCoreOpenTime + " extends "
+ + super.toString(field) + "]";
+ }
+ }
+
+ static class SameCoreJoinQuery extends Query {
+ protected final Query fromQuery;
+ protected final ScoreMode scoreMode;
+ protected final String fromField;
+ protected final String toField;
+
+ SameCoreJoinQuery(Query fromQuery, String fromField, String toField,
+ ScoreMode scoreMode) {
+ this.fromQuery = fromQuery;
+ this.scoreMode = scoreMode;
+ this.fromField = fromField;
+ this.toField = toField;
+ }
+
+ @Override
+ public Query rewrite(IndexReader reader) throws IOException {
+ SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+ final Query jq = JoinUtil.createJoinQuery(fromField, true,
+ toField, fromQuery, info.getReq().getSearcher(), scoreMode);
+ jq.setBoost(getBoost());
+ return jq.rewrite(reader);
+ }
+
+
+ @Override
+ public String toString(String field) {
+ return "SameCoreJoinQuery [fromQuery=" + fromQuery + ", fromField="
+ + fromField + ", toField=" + toField + ", scoreMode=" + scoreMode
+ + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result
+ + ((fromField == null) ? 0 : fromField.hashCode());
+ result = prime * result
+ + ((fromQuery == null) ? 0 : fromQuery.hashCode());
+ result = prime * result
+ + ((scoreMode == null) ? 0 : scoreMode.hashCode());
+ result = prime * result + ((toField == null) ? 0 : toField.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (!super.equals(obj)) return false;
+ if (getClass() != obj.getClass()) return false;
+ SameCoreJoinQuery other = (SameCoreJoinQuery) obj;
+ if (fromField == null) {
+ if (other.fromField != null) return false;
+ } else if (!fromField.equals(other.fromField)) return false;
+ if (fromQuery == null) {
+ if (other.fromQuery != null) return false;
+ } else if (!fromQuery.equals(other.fromQuery)) return false;
+ if (scoreMode != other.scoreMode) return false;
+ if (toField == null) {
+ if (other.toField != null) return false;
+ } else if (!toField.equals(other.toField)) return false;
+ return true;
+ }
+ }
+
+ final static Map<String, ScoreMode> lowercase = Collections.unmodifiableMap( new HashMap<String, ScoreMode>() {
+ {
+ for (ScoreMode s : ScoreMode.values()) {
+ put(s.name().toLowerCase(Locale.ROOT), s);
+ put(s.name(), s);
+ }
+ }
+ });
+
+ @Override
+ public void init(NamedList args) {
+ }
+
+
+ @Override
+ public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ return new QParser(qstr, localParams, params, req) {
+ @Override
+ public Query parse() throws SyntaxError {
+ final String fromField = localParams.get("from");
+ final String fromIndex = localParams.get("fromIndex");
+ final String toField = localParams.get("to");
+ final ScoreMode scoreMode = parseScore();
+
+ final String v = localParams.get(CommonParams.VALUE);
+
+ final Query q = createQuery(fromField, v, fromIndex, toField, scoreMode,
+ CommonParams.TRUE.equals(localParams.get("TESTenforceSameCoreAsAnotherOne")));
+
+ return q;
+ }
+
+ private Query createQuery(final String fromField, final String fromQueryStr,
+ String fromIndex, final String toField, final ScoreMode scoreMode,
+ boolean byPassShortCircutCheck) throws SyntaxError {
+
+ final String myCore = req.getCore().getCoreDescriptor().getName();
+
+ if (fromIndex != null && (!fromIndex.equals(myCore) || byPassShortCircutCheck)) {
+ CoreContainer container = req.getCore().getCoreDescriptor().getCoreContainer();
+
+ final SolrCore fromCore = container.getCore(fromIndex);
+ RefCounted<SolrIndexSearcher> fromHolder = null;
+
+ if (fromCore == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex);
+ }
+
+ long fromCoreOpenTime = 0;
+ LocalSolrQueryRequest otherReq = new LocalSolrQueryRequest(fromCore, params);
+
+ try {
+ QParser fromQueryParser = QParser.getParser(fromQueryStr, "lucene", otherReq);
+ Query fromQuery = fromQueryParser.getQuery();
+
+ fromHolder = fromCore.getRegisteredSearcher();
+ if (fromHolder != null) {
+ fromCoreOpenTime = fromHolder.get().getOpenTime();
+ }
+ return new OtherCoreJoinQuery(fromQuery, fromField, fromIndex, fromCoreOpenTime,
+ scoreMode, toField);
+ } finally {
+ otherReq.close();
+ fromCore.close();
+ if (fromHolder != null) fromHolder.decref();
+ }
+ } else {
+ QParser fromQueryParser = subQuery(fromQueryStr, null);
+ final Query fromQuery = fromQueryParser.getQuery();
+ return new SameCoreJoinQuery(fromQuery, fromField, toField, scoreMode);
+ }
+ }
+
+ private ScoreMode parseScore() {
+
+ String score = getParam(SCORE);
+ final ScoreMode scoreMode = lowercase.get(score);
+ if (scoreMode == null) {
+ throw new IllegalArgumentException("Unable to parse ScoreMode from: " + score);
+ }
+ return scoreMode;
+ }
+ };
+ }
+}
Added: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml?rev=1693092&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml (added)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml Tue Jul 28 13:24:04 2015
@@ -0,0 +1,89 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="doc-values-for-Join" version="1.5">
+ <types>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldtype name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true" >
+ <analyzer type="index">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ </types>
+
+ <fields>
+ <field name="id" type="string" indexed="true" stored="true" docValues="false" multiValued="false" required="true"/>
+ <field name="id_dv" type="string" indexed="false" stored="false" docValues="true" multiValued="false" required="true"/>
+ <dynamicField name="*_i" type="int" indexed="true" stored="false" docValues="false"/>
+ <dynamicField name="*_i_dv" type="int" indexed="true" stored="true" docValues="true"/>
+ <dynamicField name="*_is" type="int" indexed="true" stored="false" docValues="false" multiValued="true"/>
+ <dynamicField name="*_is_dv" type="int" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="false" docValues="false"/>
+ <dynamicField name="*_s_dv" type="string" indexed="true" stored="true" docValues="true"/>
+ <dynamicField name="*_ss" type="string" indexed="true" stored="false" docValues="false" multiValued="true"/>
+ <dynamicField name="*_ss_dv" type="string" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f" type="float" indexed="true" stored="false" docValues="false"/>
+ <dynamicField name="*_f_dv" type="float" indexed="true" stored="true" docValues="true"/>
+ <dynamicField name="*_fs_dv" type="float" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l" type="long" indexed="true" stored="false" docValues="false"/>
+ <dynamicField name="*_l_dv" type="long" indexed="true" stored="false" docValues="true"/>
+ <dynamicField name="*_ls_dv" type="long" indexed="true" stored="false" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d" type="double" indexed="true" stored="false" docValues="false"/>
+ <dynamicField name="*_d_dv" type="double" indexed="true" stored="false" docValues="true"/>
+ <dynamicField name="*_ds_dv" type="double" indexed="true" stored="false" docValues="true" multiValued="true"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="false" docValues="false"/>
+ <dynamicField name="*_dt_dv" type="date" indexed="true" stored="false" docValues="true"/>
+ <dynamicField name="*_dts_dv" type="date" indexed="true" stored="false" docValues="true" multiValued="true"/>
+ <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
+ </fields>
+
+ <defaultSearchField>id</defaultSearchField>
+ <uniqueKey>id</uniqueKey>
+
+ <copyField source="*_i" dest="*_i_dv" />
+ <copyField source="*_f" dest="*_f_dv" />
+ <copyField source="*_is" dest="*_is_dv" />
+ <copyField source="*_s" dest="*_s_dv" />
+ <copyField source="*_ss" dest="*_ss_dv" />
+ <copyField source="id" dest="id_dv" />
+</schema>
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java?rev=1693092&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java Tue Jul 28 13:24:04 2015
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr;
+
+import java.io.StringWriter;
+import java.util.Collections;
+
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.join.TestScoreJoinQPNoScore;
+import org.apache.solr.servlet.DirectSolrConnection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestCrossCoreJoin extends SolrTestCaseJ4 {
+
+ private static SolrCore fromCore;
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
+// initCore("solrconfig.xml","schema12.xml");
+
+ // File testHome = createTempDir().toFile();
+ // FileUtils.copyDirectory(getFile("solrj/solr"), testHome);
+ initCore("solrconfig.xml", "schema12.xml", TEST_HOME(), "collection1");
+ final CoreContainer coreContainer = h.getCoreContainer();
+ final CoreDescriptor toCoreDescriptor = coreContainer.getCoreDescriptor("collection1");
+ final CoreDescriptor fromCoreDescriptor = new CoreDescriptor("fromCore", toCoreDescriptor) {
+ @Override
+ public String getSchemaName() {
+ return "schema.xml";
+ }
+ };
+
+ fromCore = coreContainer.create(fromCoreDescriptor);
+
+ assertU(add(doc("id", "1", "name", "john", "title", "Director", "dept_s", "Engineering")));
+ assertU(add(doc("id", "2", "name", "mark", "title", "VP", "dept_s", "Marketing")));
+ assertU(add(doc("id", "3", "name", "nancy", "title", "MTS", "dept_s", "Sales")));
+ assertU(add(doc("id", "4", "name", "dave", "title", "MTS", "dept_s", "Support", "dept_s", "Engineering")));
+ assertU(add(doc("id", "5", "name", "tina", "title", "VP", "dept_s", "Engineering")));
+ assertU(commit());
+
+ update(fromCore, add(doc("id", "10", "dept_id_s", "Engineering", "text", "These guys develop stuff", "cat", "dev")));
+ update(fromCore, add(doc("id", "11", "dept_id_s", "Marketing", "text", "These guys make you look good")));
+ update(fromCore, add(doc("id", "12", "dept_id_s", "Sales", "text", "These guys sell stuff")));
+ update(fromCore, add(doc("id", "13", "dept_id_s", "Support", "text", "These guys help customers")));
+ update(fromCore, commit());
+
+ }
+
+
+ public static String update(SolrCore core, String xml) throws Exception {
+ DirectSolrConnection connection = new DirectSolrConnection(core);
+ SolrRequestHandler handler = core.getRequestHandler("/update");
+ return connection.request(handler, null, xml);
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+ doTestJoin("{!join");
+ }
+
+ @Test
+ public void testScoreJoin() throws Exception {
+ doTestJoin("{!join " + TestScoreJoinQPNoScore.whateverScore());
+ }
+
+ void doTestJoin(String joinPrefix) throws Exception {
+ assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id",
+ "debugQuery", random().nextBoolean() ? "true":"false")
+ , "/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
+ );
+
+ // find people that develop stuff - but limit via filter query to a name of "john"
+ // this tests filters being pushed down to queries (SOLR-3062)
+ assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id", "fq", "name:john",
+ "debugQuery", random().nextBoolean() ? "true":"false")
+ , "/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}"
+ );
+ }
+
+ @Test
+ public void testCoresAreDifferent() throws Exception {
+ assertQEx("schema12.xml" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST);
+ final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "lucene", 0, 100, Collections.emptyMap());
+ final String resp = query(fromCore, req);
+ assertTrue(resp, resp.contains("numFound=\"1\""));
+ assertTrue(resp, resp.contains("<int name=\"id\">10</int>"));
+
+ }
+
+ public String query(SolrCore core, SolrQueryRequest req) throws Exception {
+ String handler = "standard";
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
+ core.execute(core.getRequestHandler(handler), req, rsp);
+ if (rsp.getException() != null) {
+ throw rsp.getException();
+ }
+ StringWriter sw = new StringWriter(32000);
+ QueryResponseWriter responseWriter = core.getQueryResponseWriter(req);
+ responseWriter.write(sw, req, rsp);
+ req.close();
+ SolrRequestInfo.clearRequestInfo();
+ return sw.toString();
+ }
+
+ @AfterClass
+ public static void nukeAll() {
+ fromCore = null;
+ }
+}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java?rev=1693092&r1=1693091&r2=1693092&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java Tue Jul 28 13:24:04 2015
@@ -384,6 +384,23 @@ public class QueryEqualityTest extends S
}
}
+ public void testQueryScoreJoin() throws Exception {
+ SolrQueryRequest req = req("myVar", "5",
+ "df", "text",
+ "ff", "foo_s",
+ "tt", "bar_s",
+ "scoreavg","avg");
+
+ try {
+ assertQueryEquals("join", req,
+ "{!join from=foo_s to=bar_s score=avg}asdf",
+ "{!join from=$ff to=$tt score=Avg}asdf",
+ "{!join from=$ff to='bar_s' score=$scoreavg}text:asdf");
+ } finally {
+ req.close();
+ }
+ }
+
public void testTerms() throws Exception {
assertQueryEquals("terms", "{!terms f=foo_i}10,20,30,-10,-20,-30", "{!terms f=foo_i}10,20,30,-10,-20,-30");
}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java?rev=1693092&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java Tue Jul 28 13:24:04 2015
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.join;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.solr.JSONTestUtil;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.JoinQParserPlugin;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SyntaxError;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.noggit.JSONUtil;
+import org.noggit.ObjectBuilder;
+
+public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
+ initCore("solrconfig-basic.xml","schema-docValuesJoin.xml");
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+ assertU(add(doc("id", "1","name_s", "john", "title_s", "Director", "dept_ss","Engineering")));
+ assertU(add(doc("id", "2","name_s", "mark", "title_s", "VP", "dept_ss","Marketing")));
+ assertU(add(doc("id", "3","name_s", "nancy", "title_s", "MTS", "dept_ss","Sales")));
+ assertU(add(doc("id", "4","name_s", "dave", "title_s", "MTS", "dept_ss","Support", "dept_ss","Engineering")));
+ assertU(add(doc("id", "5","name_s", "tina", "title_s", "VP", "dept_ss","Engineering")));
+
+ assertU(add(doc("id","10", "dept_id_s", "Engineering", "text_t","These guys develop stuff")));
+ assertU(add(doc("id","11", "dept_id_s", "Marketing", "text_t","These guys make you look good")));
+ assertU(add(doc("id","12", "dept_id_s", "Sales", "text_t","These guys sell stuff")));
+ assertU(add(doc("id","13", "dept_id_s", "Support", "text_t","These guys help customers")));
+
+ assertU(commit());
+
+ // test debugging TODO no debug in JoinUtil
+ // assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id", "debugQuery","true")
+ // ,"/debug/join/{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS=={'_MATCH_':'fromSetSize,toSetSize', 'fromSetSize':2, 'toSetSize':3}"
+ // );
+
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id")
+ ,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}"
+ );
+
+ // empty from
+ assertJQ(req("q","{!join from=noexist_s to=dept_id_s"+whateverScore()+"}*:*", "fl","id")
+ ,"/response=={'numFound':0,'start':0,'docs':[]}"
+ );
+
+ // empty to
+ assertJQ(req("q","{!join from=dept_ss to=noexist_s"+whateverScore()+"}*:*", "fl","id")
+ ,"/response=={'numFound':0,'start':0,'docs':[]}"
+ );
+
+ // self join... return everyone with she same title as Dave
+ assertJQ(req("q","{!join from=title_s to=title_s"+whateverScore()+"}name_s:dave", "fl","id")
+ ,"/response=={'numFound':2,'start':0,'docs':[{'id':'3'},{'id':'4'}]}"
+ );
+
+ // find people that develop stuff
+ assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id")
+ ,"/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
+ );
+
+ // self join on multivalued text_t field
+ assertJQ(req("q","{!join from=title_s to=title_s"+whateverScore()+"}name_s:dave", "fl","id")
+ ,"/response=={'numFound':2,'start':0,'docs':[{'id':'3'},{'id':'4'}]}"
+ );
+
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id", "debugQuery","true")
+ ,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}"
+ );
+
+ // expected outcome for a sub query matching dave joined against departments
+ final String davesDepartments =
+ "/response=={'numFound':2,'start':0,'docs':[{'id':'10'},{'id':'13'}]}";
+
+ // straight forward query
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}name_s:dave",
+ "fl","id"),
+ davesDepartments);
+
+ // variable deref for sub-query parsing
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s v=$qq"+whateverScore()+"}",
+ "qq","{!dismax}dave",
+ "qf","name_s",
+ "fl","id",
+ "debugQuery","true"),
+ davesDepartments);
+
+ // variable deref for sub-query parsing w/localparams
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s v=$qq"+whateverScore()+"}",
+ "qq","{!dismax qf=name_s}dave",
+ "fl","id",
+ "debugQuery","true"),
+ davesDepartments);
+
+ // defType local param to control sub-query parsing
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s defType=dismax"+whateverScore()+"}dave",
+ "qf","name_s",
+ "fl","id",
+ "debugQuery","true"),
+ davesDepartments);
+
+ // find people that develop stuff - but limit via filter query to a name of "john"
+ // this tests filters being pushed down to queries (SOLR-3062)
+ assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id", "fq", "name_s:john")
+ ,"/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}"
+ );
+
+
+ assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id"
+ )
+ ,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}");
+
+ // find people that develop stuff, even if it's requested as single value
+ assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id")
+ ,"/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
+ );
+
+ }
+
+ public void testJoinQueryType() throws SyntaxError, IOException{
+ SolrQueryRequest req = null;
+ try{
+ final String score = whateverScore();
+
+ req = req("{!join from=dept_id_s to=dept_ss"+score+"}text_t:develop");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
+
+ {
+ final Query query = QParser.getParser(req.getParams().get("q"), null, req).getQuery();
+ final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader());
+ assertTrue(
+ rewrittenQuery+" should be Lucene's",
+ rewrittenQuery.getClass().getPackage().getName()
+ .startsWith("org.apache.lucene"));
+ }
+ {
+ final Query query = QParser.getParser(
+ "{!join from=dept_id_s to=dept_ss}text_t:develop"
+ , null, req).getQuery();
+ final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader());
+ assertEquals(rewrittenQuery+" is expected to be from Solr",
+ JoinQParserPlugin.class.getPackage().getName(),
+ rewrittenQuery.getClass().getPackage().getName());
+ }
+ }finally{
+ if(req!=null){
+ req.close();
+ }
+ SolrRequestInfo.clearRequestInfo();
+ }
+ }
+
+ public static String whateverScore() {
+ final ScoreMode[] vals = ScoreMode.values();
+ return " score="+vals[random().nextInt(vals.length)]+" ";
+ }
+
+ @Test
+ public void testRandomJoin() throws Exception {
+ int indexIter=50 * RANDOM_MULTIPLIER;
+ int queryIter=50 * RANDOM_MULTIPLIER;
+
+ // groups of fields that have any chance of matching... used to
+ // increase test effectiveness by avoiding 0 resultsets much of the time.
+ String[][] compat = new String[][] {
+ {"small_s_dv","small2_s_dv","small2_ss_dv","small3_ss_dv"},
+ {"small_i_dv","small2_i_dv","small2_is_dv","small3_is_dv"}
+ };
+
+
+ while (--indexIter >= 0) {
+ int indexSize = random().nextInt(20 * RANDOM_MULTIPLIER);
+
+ List<FldType> types = new ArrayList<FldType>();
+ types.add(new FldType("id",ONE_ONE, new SVal('A','Z',4,4)));
+ /** no numeric fields so far LUCENE-5868
+ types.add(new FldType("score_f_dv",ONE_ONE, new FVal(1,100))); // field used to score
+ **/
+ types.add(new FldType("small_s_dv",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
+ types.add(new FldType("small2_s_dv",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
+ types.add(new FldType("small2_ss_dv",ZERO_TWO, new SVal('a',(char)('c'+indexSize/3),1,1)));
+ types.add(new FldType("small3_ss_dv",new IRange(0,25), new SVal('A','z',1,1)));
+ /** no numeric fields so far LUCENE-5868
+ types.add(new FldType("small_i_dv",ZERO_ONE, new IRange(0,5+indexSize/3)));
+ types.add(new FldType("small2_i_dv",ZERO_ONE, new IRange(0,5+indexSize/3)));
+ types.add(new FldType("small2_is_dv",ZERO_TWO, new IRange(0,5+indexSize/3)));
+ types.add(new FldType("small3_is_dv",new IRange(0,25), new IRange(0,100)));
+ **/
+
+ clearIndex();
+ Map<Comparable, Doc> model = indexDocs(types, null, indexSize);
+ Map<String, Map<Comparable, Set<Comparable>>> pivots = new HashMap<String, Map<Comparable, Set<Comparable>>>();
+
+ for (int qiter=0; qiter<queryIter; qiter++) {
+ String fromField;
+ String toField;
+ if (random().nextInt(100) < 5) {
+ // pick random fields 5% of the time
+ fromField = types.get(random().nextInt(types.size())).fname;
+ // pick the same field 50% of the time we pick a random field (since other fields won't match anything)
+ toField = (random().nextInt(100) < 50) ? fromField : types.get(random().nextInt(types.size())).fname;
+ } else {
+ // otherwise, pick compatible fields that have a chance of matching indexed tokens
+ String[] group = compat[random().nextInt(compat.length)];
+ fromField = group[random().nextInt(group.length)];
+ toField = group[random().nextInt(group.length)];
+ }
+
+ Map<Comparable, Set<Comparable>> pivot = pivots.get(fromField+"/"+toField);
+ if (pivot == null) {
+ pivot = createJoinMap(model, fromField, toField);
+ pivots.put(fromField+"/"+toField, pivot);
+ }
+
+ Collection<Doc> fromDocs = model.values();
+ Set<Comparable> docs = join(fromDocs, pivot);
+ List<Doc> docList = new ArrayList<Doc>(docs.size());
+ for (Comparable id : docs) docList.add(model.get(id));
+ Collections.sort(docList, createComparator("_docid_",true,false,false,false));
+ List sortedDocs = new ArrayList();
+ for (Doc doc : docList) {
+ if (sortedDocs.size() >= 10) break;
+ sortedDocs.add(doc.toObject(h.getCore().getLatestSchema()));
+ }
+
+ Map<String,Object> resultSet = new LinkedHashMap<String,Object>();
+ resultSet.put("numFound", docList.size());
+ resultSet.put("start", 0);
+ resultSet.put("docs", sortedDocs);
+
+ // todo: use different join queries for better coverage
+
+ SolrQueryRequest req = req("wt","json","indent","true", "echoParams","all",
+ "q","{!join from="+fromField+" to="+toField
+ +" "+ (random().nextBoolean() ? "fromIndex=collection1" : "")
+ +" "+ (random().nextBoolean() ? "TESTenforceSameCoreAsAnotherOne=true" : "")
+ +" "+whateverScore()+"}*:*"
+ , "sort", "_docid_ asc"
+ );
+
+ String strResponse = h.query(req);
+
+ Object realResponse = ObjectBuilder.fromJSON(strResponse);
+ String err = JSONTestUtil.matchObj("/response", realResponse, resultSet);
+ if (err != null) {
+ final String m = "JOIN MISMATCH: " + err
+ + "\n\trequest="+req
+ + "\n\tresult="+strResponse
+ + "\n\texpected="+ JSONUtil.toJSON(resultSet)
+ ;// + "\n\tmodel="+ JSONUtil.toJSON(model);
+ log.error(m);
+ {
+ SolrQueryRequest f = req("wt","json","indent","true", "echoParams","all",
+ "q","*:*", "facet","true",
+ "facet.field", fromField
+ , "sort", "_docid_ asc"
+ ,"rows","0"
+ );
+ log.error("faceting on from field: "+h.query(f));
+ }
+ {
+ final Map<String,String> ps = ((MapSolrParams)req.getParams()).getMap();
+ final String q = ps.get("q");
+ ps.put("q", q.replaceAll("join score=none", "join"));
+ log.error("plain join: "+h.query(req));
+ ps.put("q", q);
+
+ }
+ {
+ // re-execute the request... good for putting a breakpoint here for debugging
+ final Map<String,String> ps = ((MapSolrParams)req.getParams()).getMap();
+ final String q = ps.get("q");
+ ps.put("q", q.replaceAll("\\}", " cache=false\\}"));
+ String rsp = h.query(req);
+ }
+ fail(err);
+ }
+
+ }
+ }
+ }
+
+ Map<Comparable, Set<Comparable>> createJoinMap(Map<Comparable, Doc> model, String fromField, String toField) {
+ Map<Comparable, Set<Comparable>> id_to_id = new HashMap<Comparable, Set<Comparable>>();
+
+ Map<Comparable, List<Comparable>> value_to_id = invertField(model, toField);
+
+ for (Comparable fromId : model.keySet()) {
+ Doc doc = model.get(fromId);
+ List<Comparable> vals = doc.getValues(fromField);
+ if (vals == null) continue;
+ for (Comparable val : vals) {
+ List<Comparable> toIds = value_to_id.get(val);
+ if (toIds == null) continue;
+ Set<Comparable> ids = id_to_id.get(fromId);
+ if (ids == null) {
+ ids = new HashSet<Comparable>();
+ id_to_id.put(fromId, ids);
+ }
+ for (Comparable toId : toIds)
+ ids.add(toId);
+ }
+ }
+
+ return id_to_id;
+ }
+
+
+ Set<Comparable> join(Collection<Doc> input, Map<Comparable, Set<Comparable>> joinMap) {
+ Set<Comparable> ids = new HashSet<Comparable>();
+ for (Doc doc : input) {
+ Collection<Comparable> output = joinMap.get(doc.id);
+ if (output == null) continue;
+ ids.addAll(output);
+ }
+ return ids;
+ }
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java?rev=1693092&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java Tue Jul 28 13:24:04 2015
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.join;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrCache;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+public class TestScoreJoinQPScore extends SolrTestCaseJ4 {
+
+ private static final String idField = "id";
+ private static final String toField = "movieId_s";
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
+ initCore("solrconfig.xml", "schema12.xml");
+ }
+
+ public void testSimple() throws Exception {
+ final String idField = "id";
+ final String toField = "productId_s";
+
+ clearIndex();
+
+ // 0
+ assertU(add(doc("t_description", "random text",
+ "name", "name1",
+ idField, "1")));
+
+// 1
+
+ assertU(add(doc("price_s", "10.0",
+ idField, "2",
+ toField, "1")));
+// 2
+ assertU(add(doc("price_s", "20.0",
+ idField, "3",
+ toField, "1")));
+// 3
+ assertU(add(doc("t_description", "more random text",
+ "name", "name2",
+ idField, "4")));
+// 4
+ assertU(add(doc("price_s", "10.0",
+ idField, "5",
+ toField, "4")));
+// 5
+ assertU(add(doc("price_s", "20.0",
+ idField, "6",
+ toField, "4")));
+
+ assertU(commit());
+
+ // Search for product
+ assertJQ(req("q", "{!join from=" + idField + " to=" + toField + " score=None}name:name2", "fl", "id")
+ , "/response=={'numFound':2,'start':0,'docs':[{'id':'5'},{'id':'6'}]}");
+
+ /*Query joinQuery =
+ JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name2")), indexSearcher, ScoreMode.None);
+
+ TopDocs result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(4, result.scoreDocs[0].doc);
+ assertEquals(5, result.scoreDocs[1].doc);
+ */
+ assertJQ(req("q", "{!join from=" + idField + " to=" + toField + " score=None}name:name1", "fl", "id")
+ , "/response=={'numFound':2,'start':0,'docs':[{'id':'2'},{'id':'3'}]}");
+
+ /*joinQuery = JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name1")), indexSearcher, ScoreMode.None);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(1, result.scoreDocs[0].doc);
+ assertEquals(2, result.scoreDocs[1].doc);*/
+
+ // Search for offer
+ assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=None}id:5", "fl", "id")
+ , "/response=={'numFound':1,'start':0,'docs':[{'id':'4'}]}");
+ /*joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("id", "5")), indexSearcher, ScoreMode.None);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(1, result.totalHits);
+ assertEquals(3, result.scoreDocs[0].doc);
+
+ indexSearcher.getIndexReader().close();
+ dir.close();*/
+ }
+
+ public void testSimpleWithScoring() throws Exception {
+ indexDataForScorring();
+
+ // Search for movie via subtitle
+ assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Max}title:random", "fl", "id")
+ , "/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'4'}]}");
+ //dump(req("q","{!scorejoin from="+toField+" to="+idField+" score=Max}title:random", "fl","id,score", "debug", "true"));
+ /*
+ Query joinQuery =
+ JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "random")), indexSearcher, ScoreMode.Max);
+ TopDocs result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(0, result.scoreDocs[0].doc);
+ assertEquals(3, result.scoreDocs[1].doc);*/
+
+
+ // Score mode max.
+ //dump(req("q","{!scorejoin from="+toField+" to="+idField+" score=Max}title:movie", "fl","id,score", "debug", "true"));
+
+ // dump(req("q","title:movie", "fl","id,score", "debug", "true"));
+ assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Max}title:movie", "fl", "id")
+ , "/response=={'numFound':2,'start':0,'docs':[{'id':'4'},{'id':'1'}]}");
+
+ /*joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Max);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(3, result.scoreDocs[0].doc);
+ assertEquals(0, result.scoreDocs[1].doc);*/
+
+ // Score mode total
+ assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Total}title:movie", "fl", "id")
+ , "/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'4'}]}");
+ /* joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Total);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(0, result.scoreDocs[0].doc);
+ assertEquals(3, result.scoreDocs[1].doc);
+*/
+ //Score mode avg
+ assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Avg}title:movie", "fl", "id")
+ , "/response=={'numFound':2,'start':0,'docs':[{'id':'4'},{'id':'1'}]}");
+
+ /* joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Avg);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(3, result.scoreDocs[0].doc);
+ assertEquals(0, result.scoreDocs[1].doc);*/
+
+ }
+
+ final static Comparator<String> lessFloat = new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ assertTrue(Float.parseFloat(o1) < Float.parseFloat(o2));
+ return 0;
+ }
+ };
+
+ @Ignore("SOLR-7814, also don't forget cover boost at testCacheHit()")
+ public void testBoost() throws Exception {
+ indexDataForScorring();
+ ScoreMode score = ScoreMode.values()[random().nextInt(ScoreMode.values().length)];
+
+ final SolrQueryRequest req = req("q", "{!join from=movieId_s to=id score=" + score + " b=200}title:movie", "fl", "id,score", "omitHeader", "true");
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, new SolrQueryResponse()));
+ final Query luceneQ = QParser.getParser(req.getParams().get("q"), null, req).getQuery().rewrite(req.getSearcher().getLeafReader());
+ assertEquals("" + luceneQ, Float.floatToIntBits(200), Float.floatToIntBits(luceneQ.getBoost()));
+ SolrRequestInfo.clearRequestInfo();
+ req.close();
+ }
+
+ public void testCacheHit() throws Exception {
+ indexDataForScorring();
+
+ SolrCache cache = (SolrCache) h.getCore().getInfoRegistry()
+ .get("queryResultCache");
+ {
+ final NamedList statPre = cache.getStatistics();
+ h.query(req("q", "{!join from=movieId_s to=id score=Avg}title:first", "fl", "id", "omitHeader", "true"));
+ assertHitOrInsert(cache, statPre);
+ }
+
+ {
+ final NamedList statPre = cache.getStatistics();
+ h.query(req("q", "{!join from=movieId_s to=id score=Avg}title:first", "fl", "id", "omitHeader", "true"));
+ assertHit(cache, statPre);
+ }
+
+ {
+ NamedList statPre = cache.getStatistics();
+
+ Random r = random();
+ boolean changed = false;
+ boolean x = false;
+ String from = (x = r.nextBoolean()) ? "id" : "movieId_s";
+ changed |= x;
+ String to = (x = r.nextBoolean()) ? "movieId_s" : "id";
+ changed |= x;
+ String score = (x = r.nextBoolean()) ? not(ScoreMode.Avg).name() : "Avg";
+ changed |= x;
+ /* till SOLR-7814
+ * String boost = (x = r.nextBoolean()) ? "23" : "1";
+ changed |= x; */
+ String q = (!changed) ? (r.nextBoolean() ? "title:first^67" : "title:night") : "title:first";
+
+ final String resp = h.query(req("q", "{!join from=" + from + " to=" + to +
+ " score=" + score +
+ //" b=" + boost +
+ "}" + q, "fl", "id", "omitHeader", "true")
+ );
+ assertInsert(cache, statPre);
+
+ statPre = cache.getStatistics();
+ final String repeat = h.query(req("q", "{!join from=" + from + " to=" + to + " score=" + score.toLowerCase(Locale.ROOT) +
+ //" b=" + boost
+ "}" + q, "fl", "id", "omitHeader", "true")
+ );
+ assertHit(cache, statPre);
+
+ assertEquals("lowercase shouldn't change anything", resp, repeat);
+
+ try {
+ h.query(req("q", "{!join from=" + from + " to=" + to + " score=" + score.substring(0, score.length() - 1) +
+ "}" + q, "fl", "id", "omitHeader", "true")
+ );
+ fail("excpecting exception");
+ } catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains("ScoreMode"));
+ }
+ }
+ // this queries are not overlap, with other in this test case.
+ // however it might be better to extract this method into the separate suite
+ // for a while let's nuke a cache content, in case of repetitions
+ cache.clear();
+ }
+
+ private ScoreMode not(ScoreMode s) {
+ Random r = random();
+ final List<ScoreMode> l = new ArrayList(Arrays.asList(ScoreMode.values()));
+ l.remove(s);
+ return l.get(r.nextInt(l.size()));
+ }
+
+ private void assertInsert(SolrCache cache, final NamedList statPre) {
+ assertEquals("it lookups", 1,
+ delta("lookups", cache.getStatistics(), statPre));
+ assertEquals("it doesn't hit", 0, delta("hits", cache.getStatistics(), statPre));
+ assertEquals("it inserts", 1,
+ delta("inserts", cache.getStatistics(), statPre));
+ }
+
+ private void assertHit(SolrCache cache, final NamedList statPre) {
+ assertEquals("it lookups", 1,
+ delta("lookups", cache.getStatistics(), statPre));
+ assertEquals("it hits", 1, delta("hits", cache.getStatistics(), statPre));
+ assertEquals("it doesn't insert", 0,
+ delta("inserts", cache.getStatistics(), statPre));
+ }
+
+ private void assertHitOrInsert(SolrCache cache, final NamedList statPre) {
+ assertEquals("it lookups", 1,
+ delta("lookups", cache.getStatistics(), statPre));
+ final long mayHit = delta("hits", cache.getStatistics(), statPre);
+ assertTrue("it may hit", 0 == mayHit || 1 == mayHit);
+ assertEquals("or insert on cold", 1,
+ delta("inserts", cache.getStatistics(), statPre) + mayHit);
+ }
+
+ private long delta(String key, NamedList a, NamedList b) {
+ return (Long) a.get(key) - (Long) b.get(key);
+ }
+
+ private void indexDataForScorring() {
+ clearIndex();
+// 0
+ assertU(add(doc("t_description", "A random movie",
+ "name", "Movie 1",
+ idField, "1")));
+// 1
+
+ assertU(add(doc("title", "The first subtitle of this movie",
+ idField, "2",
+ toField, "1")));
+
+
+// 2
+
+ assertU(add(doc("title", "random subtitle; random event movie",
+ idField, "3",
+ toField, "1")));
+
+// 3
+
+ assertU(add(doc("t_description", "A second random movie",
+ "name", "Movie 2",
+ idField, "4")));
+// 4
+
+ assertU(add(doc("title", "a very random event happened during christmas night",
+ idField, "5",
+ toField, "4")));
+
+
+// 5
+
+ assertU(add(doc("title", "movie end movie test 123 test 123 random",
+ idField, "6",
+ toField, "4")));
+
+
+ assertU(commit());
+ }
+}