You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2016/12/20 16:39:49 UTC
[13/24] lucene-solr:jira/solr-8593: SOLR-8542: techproducts example
now includes (disabled) learning-to-rank support (enable via
-Dsolr.ltr.enabled=true)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
index 57fc5ad..ee065c3 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
@@ -47,6 +47,16 @@ import org.apache.solr.ltr.norm.Normalizer;
}
}</pre>
* <p>
+ * Training libraries:
+ * <ul>
+ * <li> <a href="https://www.csie.ntu.edu.tw/~cjlin/liblinear/">
+ * LIBLINEAR -- A Library for Large Linear Classification</a>
+ * </ul>
+ * <ul>
+ * <li> <a href="https://www.csie.ntu.edu.tw/~cjlin/libsvm/">
+ * LIBSVM -- A Library for Support Vector Machines</a>
+ * </ul>
+ * <p>
* Background reading:
* <ul>
* <li> <a href="http://www.cs.cornell.edu/people/tj/publications/joachims_02c.pdf">
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
index 4fa595e..926fab4 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
@@ -71,6 +71,11 @@ import org.apache.solr.util.SolrPluginUtils;
}
}</pre>
* <p>
+ * Training libraries:
+ * <ul>
+ * <li> <a href="http://sourceforge.net/p/lemur/wiki/RankLib/">RankLib</a>
+ * </ul>
+ * <p>
* Background reading:
* <ul>
* <li> <a href="http://research.microsoft.com/pubs/132652/MSR-TR-2010-82.pdf">
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
index 93ebe63..354ecc2 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
@@ -17,6 +17,7 @@
package org.apache.solr.ltr.response.transform;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -27,6 +28,7 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.ltr.CSVFeatureLogger;
import org.apache.solr.ltr.FeatureLogger;
import org.apache.solr.ltr.LTRRescorer;
import org.apache.solr.ltr.LTRScoringQuery;
@@ -44,17 +46,25 @@ import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.SolrPluginUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This transformer will take care to generate and append in the response the
- * features declared in the feature store of the current model. The class is
- * useful if you are not interested in the reranking (e.g., bootstrapping a
- * machine learning framework).
- */
+ * features declared in the feature store of the current reranking model,
+ * or a specified feature store. Ex. <code>fl=id,[features store=myStore efi.user_text="ibm"]</code>
+ *
+ * <h3>Parameters</h3>
+ * <code>store</code> - The feature store to extract features from. If not provided it
+ * will default to the features used by your reranking model.<br>
+ * <code>efi.*</code> - External feature information variables required by the features
+ * you are extracting.<br>
+ * <code>format</code> - The format you want the features to be returned in. Supports (dense|sparse). Defaults to sparse.<br>
+*/
+
public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
- // used inside fl to specify the output format (csv/json) of the extracted features
- private static final String FV_RESPONSE_WRITER = "fvwt";
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// used inside fl to specify the format (dense|sparse) of the extracted features
private static final String FV_FORMAT = "format";
@@ -64,27 +74,43 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
private static String DEFAULT_LOGGING_MODEL_NAME = "logging-model";
+ private String fvCacheName;
private String loggingModelName = DEFAULT_LOGGING_MODEL_NAME;
- private String defaultFvStore;
- private String defaultFvwt;
- private String defaultFvFormat;
+ private String defaultStore;
+ private String defaultFormat;
+ private char csvKeyValueDelimiter = CSVFeatureLogger.DEFAULT_KEY_VALUE_SEPARATOR;
+ private char csvFeatureSeparator = CSVFeatureLogger.DEFAULT_FEATURE_SEPARATOR;
private LTRThreadModule threadManager = null;
+ public void setFvCacheName(String fvCacheName) {
+ this.fvCacheName = fvCacheName;
+ }
+
public void setLoggingModelName(String loggingModelName) {
this.loggingModelName = loggingModelName;
}
- public void setStore(String defaultFvStore) {
- this.defaultFvStore = defaultFvStore;
+ public void setDefaultStore(String defaultStore) {
+ this.defaultStore = defaultStore;
}
- public void setFvwt(String defaultFvwt) {
- this.defaultFvwt = defaultFvwt;
+ public void setDefaultFormat(String defaultFormat) {
+ this.defaultFormat = defaultFormat;
+ }
+
+ public void setCsvKeyValueDelimiter(String csvKeyValueDelimiter) {
+ if (csvKeyValueDelimiter.length() != 1) {
+ throw new IllegalArgumentException("csvKeyValueDelimiter must be exactly 1 character");
+ }
+ this.csvKeyValueDelimiter = csvKeyValueDelimiter.charAt(0);
}
- public void setFormat(String defaultFvFormat) {
- this.defaultFvFormat = defaultFvFormat;
+ public void setCsvFeatureSeparator(String csvFeatureSeparator) {
+ if (csvFeatureSeparator.length() != 1) {
+ throw new IllegalArgumentException("csvFeatureSeparator must be exactly 1 character");
+ }
+ this.csvFeatureSeparator = csvFeatureSeparator.charAt(0);
}
@Override
@@ -95,35 +121,62 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
}
@Override
- public DocTransformer create(String name, SolrParams params,
+ public DocTransformer create(String name, SolrParams localparams,
SolrQueryRequest req) {
// Hint to enable feature vector cache since we are requesting features
SolrQueryRequestContextUtils.setIsExtractingFeatures(req);
// Communicate which feature store we are requesting features for
- SolrQueryRequestContextUtils.setFvStoreName(req, params.get(FV_STORE, defaultFvStore));
+ SolrQueryRequestContextUtils.setFvStoreName(req, localparams.get(FV_STORE, defaultStore));
// Create and supply the feature logger to be used
SolrQueryRequestContextUtils.setFeatureLogger(req,
- FeatureLogger.createFeatureLogger(
- params.get(FV_RESPONSE_WRITER, defaultFvwt),
- params.get(FV_FORMAT, defaultFvFormat)));
+ createFeatureLogger(
+ localparams.get(FV_FORMAT, defaultFormat)));
+
+ return new FeatureTransformer(name, localparams, req);
+ }
- return new FeatureTransformer(name, params, req);
+ /**
+ * returns a FeatureLogger that logs the features
+ * 'featureFormat' param: 'dense' will write features in dense format,
+ * 'sparse' will write the features in sparse format, null or empty will
+ * default to 'sparse'
+ *
+ *
+ * @return a feature logger for the format specified.
+ */
+ private FeatureLogger createFeatureLogger(String featureFormat) {
+ final FeatureLogger.FeatureFormat f;
+ if (featureFormat == null || featureFormat.isEmpty() ||
+ featureFormat.equals("sparse")) {
+ f = FeatureLogger.FeatureFormat.SPARSE;
+ }
+ else if (featureFormat.equals("dense")) {
+ f = FeatureLogger.FeatureFormat.DENSE;
+ }
+ else {
+ f = FeatureLogger.FeatureFormat.SPARSE;
+ log.warn("unknown feature logger feature format {}", featureFormat);
+ }
+ if (fvCacheName == null) {
+ throw new IllegalArgumentException("a fvCacheName must be configured");
+ }
+ return new CSVFeatureLogger(fvCacheName, f, csvKeyValueDelimiter, csvFeatureSeparator);
}
class FeatureTransformer extends DocTransformer {
final private String name;
- final private SolrParams params;
+ final private SolrParams localparams;
final private SolrQueryRequest req;
private List<LeafReaderContext> leafContexts;
private SolrIndexSearcher searcher;
private LTRScoringQuery scoringQuery;
private LTRScoringQuery.ModelWeight modelWeight;
- private FeatureLogger<?> featureLogger;
+ private FeatureLogger featureLogger;
private boolean docsWereNotReranked;
/**
@@ -131,10 +184,10 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
* Name of the field to be added in a document representing the
* feature vectors
*/
- public FeatureTransformer(String name, SolrParams params,
+ public FeatureTransformer(String name, SolrParams localparams,
SolrQueryRequest req) {
this.name = name;
- this.params = params;
+ this.localparams = localparams;
this.req = req;
}
@@ -178,7 +231,7 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
featureStoreName, store.getFeatures());
scoringQuery = new LTRScoringQuery(lm,
- LTRQParserPlugin.extractEFIParams(params),
+ LTRQParserPlugin.extractEFIParams(localparams),
true,
threadManager); // request feature weights to be created for all features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/overview.html b/solr/contrib/ltr/src/java/overview.html
index d1a22f0..b0802d2 100644
--- a/solr/contrib/ltr/src/java/overview.html
+++ b/solr/contrib/ltr/src/java/overview.html
@@ -28,7 +28,7 @@ that sophisticated models can make more nuanced ranking decisions than standard
functions like TF-IDF or BM25.
</p>
<p>
-This module allows to plug a reranking component directly into Solr, enabling users
+This module allows to plug a reranking model directly into Solr, enabling users
to easily build their own learning to rank systems and access the rich
matching features readily available in Solr. It also provides tools to perform
feature engineering and feature extraction.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
index d6f8d8e..1e1a618 100644
--- a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
@@ -35,8 +35,9 @@
in the response. The name of the field we will be the the name of the transformer
enclosed between brackets (in this case [fv]). In order to get the feature
vector you will have to specify that you want the field (e.g., fl="*,[fv]) -->
- <transformer name="fv"
- class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory" />
+ <transformer name="fv" class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory">
+ <str name="fvCacheName">QUERY_DOC_FV</str>
+ </transformer>
<updateHandler class="solr.DirectUpdateHandler2">
<autoCommit>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
index 51971f7..b41bce1 100644
--- a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
@@ -39,8 +39,9 @@
in the response. The name of the field we will be the the name of the transformer
enclosed between brackets (in this case [fv]). In order to get the feature
vector you will have to specify that you want the field (e.g., fl="*,[fv]) -->
- <transformer name="fv"
- class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory" />
+ <transformer name="fv" class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory">
+ <str name="fvCacheName">QUERY_DOC_FV</str>
+ </transformer>
<updateHandler class="solr.DirectUpdateHandler2">
<autoCommit>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
index 5abff6f..b34be8f 100644
--- a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
@@ -22,6 +22,13 @@
<!-- Query parser used to rerank top docs with a provided model -->
<queryParser name="ltr" class="org.apache.solr.ltr.search.LTRQParserPlugin" />
+ <query>
+ <filterCache class="solr.FastLRUCache" size="4096"
+ initialSize="2048" autowarmCount="0" />
+ <cache name="QUERY_DOC_FV" class="solr.search.LRUCache" size="4096"
+ initialSize="2048" autowarmCount="4096" regenerator="solr.search.NoOpRegenerator" />
+ </query>
+
<maxBufferedDocs>1</maxBufferedDocs>
<mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
<int name="maxMergeAtOnce">10</int>
@@ -32,8 +39,9 @@
in the response. The name of the field we will be the the name of the transformer
enclosed between brackets (in this case [fv]). In order to get the feature
vector you will have to specify that you want the field (e.g., fl="*,[fv]) -->
- <transformer name="features"
- class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory" />
+ <transformer name="features" class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory">
+ <str name="fvCacheName">QUERY_DOC_FV</str>
+ </transformer>
<updateHandler class="solr.DirectUpdateHandler2">
<autoCommit>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/FeatureLoggerTestUtils.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/FeatureLoggerTestUtils.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/FeatureLoggerTestUtils.java
new file mode 100644
index 0000000..f2afd9c
--- /dev/null
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/FeatureLoggerTestUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr;
+
+public class FeatureLoggerTestUtils {
+
+ public static String toFeatureVector(String ... keysAndValues) {
+ return toFeatureVector(
+ CSVFeatureLogger.DEFAULT_KEY_VALUE_SEPARATOR,
+ CSVFeatureLogger.DEFAULT_FEATURE_SEPARATOR,
+ keysAndValues);
+ }
+
+ public static String toFeatureVector(char keyValueSeparator, char featureSeparator,
+ String ... keysAndValues) {
+ StringBuilder sb = new StringBuilder(keysAndValues.length/2 * 3);
+ for (int ii = 0; ii+1 < keysAndValues.length; ii += 2) {
+ sb.append(keysAndValues[ii])
+ .append(keyValueSeparator)
+ .append(keysAndValues[ii+1])
+ .append(featureSeparator);
+ }
+
+ final String features = (sb.length() > 0 ?
+ sb.substring(0, sb.length() - 1) : "");
+
+ return features;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
index 2e01a64..2bef197 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
@@ -92,6 +92,15 @@ public class TestLTROnSolrCloud extends TestRerankBase {
assertEquals("3", queryResponse.getResults().get(2).get("id").toString());
assertEquals("4", queryResponse.getResults().get(3).get("id").toString());
+ final String result0_features= FeatureLoggerTestUtils.toFeatureVector(
+ "powpularityS","64.0", "c3","2.0");
+ final String result1_features= FeatureLoggerTestUtils.toFeatureVector(
+ "powpularityS","49.0", "c3","2.0");
+ final String result2_features= FeatureLoggerTestUtils.toFeatureVector(
+ "powpularityS","36.0", "c3","2.0");
+ final String result3_features= FeatureLoggerTestUtils.toFeatureVector(
+ "powpularityS","25.0", "c3","2.0");
+
// Test re-rank and feature vectors returned
query.setFields("*,score,features:[fv]");
query.add("rq", "{!ltr model=powpularityS-model reRankDocs=8}");
@@ -99,16 +108,16 @@ public class TestLTROnSolrCloud extends TestRerankBase {
solrCluster.getSolrClient().query(COLLECTION,query);
assertEquals(8, queryResponse.getResults().getNumFound());
assertEquals("8", queryResponse.getResults().get(0).get("id").toString());
- assertEquals("powpularityS:64.0;c3:2.0",
+ assertEquals(result0_features,
queryResponse.getResults().get(0).get("features").toString());
assertEquals("7", queryResponse.getResults().get(1).get("id").toString());
- assertEquals("powpularityS:49.0;c3:2.0",
+ assertEquals(result1_features,
queryResponse.getResults().get(1).get("features").toString());
assertEquals("6", queryResponse.getResults().get(2).get("id").toString());
- assertEquals("powpularityS:36.0;c3:2.0",
+ assertEquals(result2_features,
queryResponse.getResults().get(2).get("features").toString());
assertEquals("5", queryResponse.getResults().get(3).get("id").toString());
- assertEquals("powpularityS:25.0;c3:2.0",
+ assertEquals(result3_features,
queryResponse.getResults().get(3).get("features").toString());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
index 68961d2..b9b3d63 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
@@ -215,6 +215,11 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
@Test
public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception {
+ final String docs0fv = FeatureLoggerTestUtils.toFeatureVector(
+ "matchedTitle","1.0", "titlePhraseMatch","0.40254828");
+ final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector(
+ "popularity","3.0", "originalScore","1.0");
+
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.add("fl", "*,score");
@@ -225,7 +230,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='matchedTitle:1.0;titlePhraseMatch:0.40254828'"); // extract all features in default store
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); // extract all features in default store
query.remove("fl");
query.remove("rq");
@@ -235,7 +240,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='popularity:3.0;originalScore:1.0'"); // extract all features from fstore4
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4
query.remove("fl");
@@ -245,7 +250,7 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); // score using fstore2 used by externalmodelstore
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7992");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='popularity:3.0;originalScore:1.0'"); // extract all features from fstore4
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
index 8c00758..15b7633 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java
@@ -17,6 +17,7 @@
package org.apache.solr.ltr.feature;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -103,6 +104,9 @@ public class TestExternalFeatures extends TestRerankBase {
query.setQuery("*:*");
query.add("rows", "1");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector(
+ "confidence","2.3", "originalScore","1.0");
+
// Features we're extracting depend on external feature info not passed in
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(), "/error/msg=='Exception from createWeight for SolrFeature [name=matchedTitle, params={q={!terms f=title}${user_query}}] SolrFeatureWeight requires efi parameter that was not passed in request.'");
@@ -110,13 +114,13 @@ public class TestExternalFeatures extends TestRerankBase {
// Adding efi in features section should make it work
query.remove("fl");
query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='confidence:2.3;originalScore:1.0'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_sparse_csv+"'");
// Adding efi in transformer + rq should still use the transformer's params for feature extraction
query.remove("fl");
query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='confidence:2.3;originalScore:1.0'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_sparse_csv+"'");
}
@Test
@@ -128,7 +132,7 @@ public class TestExternalFeatures extends TestRerankBase {
// Efi is explicitly not required, so we do not score the feature
query.remove("fl");
query.add("fl", "fvalias:[fv store=fstore2]");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='originalScore:0.0'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+FeatureLoggerTestUtils.toFeatureVector("originalScore","0.0")+"'");
}
@Test
@@ -140,7 +144,7 @@ public class TestExternalFeatures extends TestRerankBase {
// Efi is explicitly not required, so we do not score the feature
query.remove("fl");
query.add("fl", "fvalias:[fv store=fstore3]");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='originalScore:0.0'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+FeatureLoggerTestUtils.toFeatureVector("originalScore","0.0")+"'");
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalValueFeatures.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalValueFeatures.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalValueFeatures.java
index bc073cb..fc0ade2 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalValueFeatures.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalValueFeatures.java
@@ -17,6 +17,7 @@
package org.apache.solr.ltr.feature;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -55,12 +56,11 @@ public class TestExternalValueFeatures extends TestRerankBase {
query.setQuery("*:*");
query.add("fl", "*,score,features:[fv]");
query.add("rows", "3");
- query.add("fl", "[fv]");
query.add("rq", "{!ltr reRankDocs=3 model=external_model_binary_feature efi.user_device_tablet=1}");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/features=='user_device_tablet:1.0'");
+ "/response/docs/[0]/features=='"+FeatureLoggerTestUtils.toFeatureVector("user_device_tablet","1.0")+"'");
assertJQ("/query" + query.toQueryString(),
"/response/docs/[0]/score==65.0");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFeatureLogging.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFeatureLogging.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFeatureLogging.java
index 14e2903..f18c6bf 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFeatureLogging.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFeatureLogging.java
@@ -17,6 +17,7 @@
package org.apache.solr.ltr.feature;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.apache.solr.ltr.model.LinearModel;
import org.apache.solr.ltr.store.FeatureStore;
@@ -55,6 +56,13 @@ public class TestFeatureLogging extends TestRerankBase {
"c1", "c2", "c3"}, "test1",
"{\"weights\":{\"c1\":1.0,\"c2\":1.0,\"c3\":1.0}}");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector(
+ "c1","1.0",
+ "c2","2.0",
+ "c3","3.0",
+ "pop","2.0",
+ "yesmatch","1.0");
+
final SolrQuery query = new SolrQuery();
query.setQuery("title:bloomberg");
query.add("fl", "title,description,id,popularity,[fv]");
@@ -65,7 +73,7 @@ public class TestFeatureLogging extends TestRerankBase {
restTestHarness.query("/query" + query.toQueryString());
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/=={'title':'bloomberg bloomberg ', 'description':'bloomberg','id':'7', 'popularity':2, '[fv]':'c1:1.0;c2:2.0;c3:3.0;pop:2.0;yesmatch:1.0'}");
+ "/response/docs/[0]/=={'title':'bloomberg bloomberg ', 'description':'bloomberg','id':'7', 'popularity':2, '[fv]':'"+docs0fv_sparse_csv+"'}");
query.remove("fl");
query.add("fl", "[fv]");
@@ -74,15 +82,7 @@ public class TestFeatureLogging extends TestRerankBase {
restTestHarness.query("/query" + query.toQueryString());
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'c1:1.0;c2:2.0;c3:3.0;pop:2.0;yesmatch:1.0'}");
- query.remove("rq");
-
- // set logging at false but still asking for feature, and it should work anyway
- query.add("rq", "{!ltr reRankDocs=3 model=sum1}");
- assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'c1:1.0;c2:2.0;c3:3.0;pop:2.0;yesmatch:1.0'}");
-
-
+ "/response/docs/[0]/=={'[fv]':'"+docs0fv_sparse_csv+"'}");
}
@Test
@@ -108,24 +108,24 @@ public class TestFeatureLogging extends TestRerankBase {
// No store specified, use default store for extraction
query.add("fl", "fv:[fv]");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'fv':'defaultf1:1.0'}");
+ "/response/docs/[0]/=={'fv':'"+FeatureLoggerTestUtils.toFeatureVector("defaultf1","1.0")+"'}");
// Store specified, use store for extraction
query.remove("fl");
query.add("fl", "fv:[fv store=store8]");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'fv':'store8f1:2.0'}");
+ "/response/docs/[0]/=={'fv':'"+FeatureLoggerTestUtils.toFeatureVector("store8f1","2.0")+"'}");
// Store specified + model specified, use store for extraction
query.add("rq", "{!ltr reRankDocs=3 model=store9m1}");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'fv':'store8f1:2.0'}");
+ "/response/docs/[0]/=={'fv':'"+FeatureLoggerTestUtils.toFeatureVector("store8f1","2.0")+"'}");
// No store specified + model specified, use model store for extraction
query.remove("fl");
query.add("fl", "fv:[fv]");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'fv':'store9f1:3.0'}");
+ "/response/docs/[0]/=={'fv':'"+FeatureLoggerTestUtils.toFeatureVector("store9f1","3.0")+"'}");
}
@@ -157,23 +157,16 @@ public class TestFeatureLogging extends TestRerankBase {
query.add("rq", "{!ltr reRankDocs=3 model=sumgroup}");
- restTestHarness.query("/query" + query.toQueryString());
- assertJQ(
- "/query" + query.toQueryString(),
- "/grouped/title/groups/[0]/doclist/docs/[0]/=={'fv':'c1:1.0;c2:2.0;c3:3.0;pop:5.0'}");
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector(
+ "c1","1.0",
+ "c2","2.0",
+ "c3","3.0",
+ "pop","5.0");
- query.remove("fl");
- query.add("fl", "fv:[fv fvwt=json]");
restTestHarness.query("/query" + query.toQueryString());
assertJQ(
"/query" + query.toQueryString(),
- "/grouped/title/groups/[0]/doclist/docs/[0]/fv/=={'c1':1.0,'c2':2.0,'c3':3.0,'pop':5.0}");
- query.remove("fl");
- query.add("fl", "fv:[fv fvwt=json]");
-
- assertJQ(
- "/query" + query.toQueryString(),
- "/grouped/title/groups/[0]/doclist/docs/[0]/fv/=={'c1':1.0,'c2':2.0,'c3':3.0,'pop':5.0}");
+ "/grouped/title/groups/[0]/doclist/docs/[0]/=={'fv':'"+docs0fv_sparse_csv+"'}");
}
@Test
@@ -187,68 +180,46 @@ public class TestFeatureLogging extends TestRerankBase {
"match"}, "test4",
"{\"weights\":{\"match\":1.0}}");
- //json - no feature format check (default to sparse)
+ final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("match", "1.0", "c4", "1.0");
+ final String docs1fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector("c4", "1.0");
+
+ final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector("match", "1.0", "c4", "1.0");
+ final String docs1fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector("match", "0.0", "c4", "1.0");
+
final SolrQuery query = new SolrQuery();
query.setQuery("title:bloomberg");
query.add("rows", "10");
- query.add("fl", "*,score,fv:[fv store=test4 fvwt=json]");
query.add("rq", "{!ltr reRankDocs=10 model=sum4}");
- assertJQ(
- "/query" + query.toQueryString(),
- "/response/docs/[0]/fv/=={'match':1.0,'c4':1.0}");
- assertJQ(
- "/query" + query.toQueryString(),
- "/response/docs/[1]/fv/=={'c4':1.0}");
-
- //json - sparse feature format check
- query.remove("fl");
- query.add("fl", "*,score,fv:[fv store=test4 format=sparse fvwt=json]");
- assertJQ(
- "/query" + query.toQueryString(),
- "/response/docs/[0]/fv/=={'match':1.0,'c4':1.0}");
- assertJQ(
- "/query" + query.toQueryString(),
- "/response/docs/[1]/fv/=={'c4':1.0}");
-
- //json - dense feature format check
- query.remove("fl");
- query.add("fl", "*,score,fv:[fv store=test4 format=dense fvwt=json]");
- assertJQ(
- "/query" + query.toQueryString(),
- "/response/docs/[0]/fv/=={'match':1.0,'c4':1.0}");
- assertJQ(
- "/query" + query.toQueryString(),
- "/response/docs/[1]/fv/=={'match':0.0,'c4':1.0}");
//csv - no feature format check (default to sparse)
query.remove("fl");
- query.add("fl", "*,score,fv:[fv store=test4 fvwt=csv]");
+ query.add("fl", "*,score,fv:[fv store=test4]");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/fv/=='match:1.0;c4:1.0'");
+ "/response/docs/[0]/fv/=='"+docs0fv_sparse_csv+"'");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[1]/fv/=='c4:1.0'");
+ "/response/docs/[1]/fv/=='"+docs1fv_sparse_csv+"'");
//csv - sparse feature format check
query.remove("fl");
- query.add("fl", "*,score,fv:[fv store=test4 format=sparse fvwt=csv]");
+ query.add("fl", "*,score,fv:[fv store=test4 format=sparse]");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/fv/=='match:1.0;c4:1.0'");
+ "/response/docs/[0]/fv/=='"+docs0fv_sparse_csv+"'");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[1]/fv/=='c4:1.0'");
+ "/response/docs/[1]/fv/=='"+docs1fv_sparse_csv+"'");
//csv - dense feature format check
query.remove("fl");
- query.add("fl", "*,score,fv:[fv store=test4 format=dense fvwt=csv]");
+ query.add("fl", "*,score,fv:[fv store=test4 format=dense]");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[0]/fv/=='match:1.0;c4:1.0'");
+ "/response/docs/[0]/fv/=='"+docs0fv_dense_csv+"'");
assertJQ(
"/query" + query.toQueryString(),
- "/response/docs/[1]/fv/=='match:0.0;c4:1.0'");
+ "/response/docs/[1]/fv/=='"+docs1fv_dense_csv+"'");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
index 9574273..8295403 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFieldValueFeature.java
@@ -17,6 +17,7 @@
package org.apache.solr.ltr.feature;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.apache.solr.ltr.model.LinearModel;
import org.junit.AfterClass;
@@ -116,7 +117,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'popularity:"+FIELD_VALUE_FEATURE_DEFAULT_VAL+"'}");
+ "/response/docs/[0]/=={'[fv]':'"+FeatureLoggerTestUtils.toFeatureVector("popularity",Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))+"'}");
}
@@ -145,7 +146,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'popularity42:42.0'}");
+ "/response/docs/[0]/=={'[fv]':'"+FeatureLoggerTestUtils.toFeatureVector("popularity42","42.0")+"'}");
}
@@ -165,7 +166,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(), "/response/numFound/==1");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'not-existing-field:"+FIELD_VALUE_FEATURE_DEFAULT_VAL+"'}");
+ "/response/docs/[0]/=={'[fv]':'"+FeatureLoggerTestUtils.toFeatureVector("not-existing-field",Float.toString(FIELD_VALUE_FEATURE_DEFAULT_VAL))+"'}");
}
@@ -183,7 +184,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("rq", "{!ltr model=trendy-model reRankDocs=4}");
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'trendy:0.0'}");
+ "/response/docs/[0]/=={'[fv]':'"+FeatureLoggerTestUtils.toFeatureVector("trendy","0.0")+"'}");
query = new SolrQuery();
@@ -191,7 +192,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("rq", "{!ltr model=trendy-model reRankDocs=4}");
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'trendy:1.0'}");
+ "/response/docs/[0]/=={'[fv]':'"+FeatureLoggerTestUtils.toFeatureVector("trendy","1.0")+"'}");
// check default value is false
query = new SolrQuery();
@@ -199,7 +200,7 @@ public class TestFieldValueFeature extends TestRerankBase {
query.add("rq", "{!ltr model=trendy-model reRankDocs=4}");
query.add("fl", "[fv]");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/=={'[fv]':'trendy:0.0'}");
+ "/response/docs/[0]/=={'[fv]':'"+FeatureLoggerTestUtils.toFeatureVector("trendy","0.0")+"'}");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFilterSolrFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFilterSolrFeature.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFilterSolrFeature.java
index 14baefa..a6a80bd 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFilterSolrFeature.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestFilterSolrFeature.java
@@ -17,6 +17,7 @@
package org.apache.solr.ltr.feature;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.apache.solr.ltr.model.LinearModel;
import org.apache.solr.ltr.store.rest.ManagedFeatureStore;
@@ -96,10 +97,13 @@ public class TestFilterSolrFeature extends TestRerankBase {
query.add("rq", "{!ltr reRankDocs=4 model=fqmodel efi.user_query=w2}");
query.add("fl", "fv:[fv]");
+ final String docs0fv_sparse_csv= FeatureLoggerTestUtils.toFeatureVector(
+ "matchedTitle","1.0", "popularity","3.0");
+
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='2'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='matchedTitle:1.0;popularity:3.0'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_sparse_csv+"'");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestNoMatchSolrFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestNoMatchSolrFeature.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestNoMatchSolrFeature.java
index 5712687..004e314 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestNoMatchSolrFeature.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestNoMatchSolrFeature.java
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.Map;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.apache.solr.ltr.model.LinearModel;
import org.apache.solr.ltr.model.MultipleAdditiveTreesModel;
@@ -104,12 +105,11 @@ public class TestNoMatchSolrFeature extends TestRerankBase {
final Double doc0Score = (Double) ((Map<String,Object>) ((ArrayList<Object>) ((Map<String,Object>) jsonParse
.get("response")).get("docs")).get(0)).get("score");
-
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score=="
+ (doc0Score * 1.1));
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/fv=='yesmatchfeature:" + doc0Score + "'");
+ "/response/docs/[0]/fv=='"+FeatureLoggerTestUtils.toFeatureVector("yesmatchfeature", doc0Score.toString())+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='2'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/fv==''");
@@ -144,7 +144,7 @@ public class TestNoMatchSolrFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.0");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/fv=='yesmatchfeature:" + doc0Score + "'");
+ "/response/docs/[0]/fv=='"+FeatureLoggerTestUtils.toFeatureVector("yesmatchfeature", doc0Score.toString())+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/fv==''");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestOriginalScoreFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestOriginalScoreFeature.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestOriginalScoreFeature.java
index e525891..48662e6 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestOriginalScoreFeature.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestOriginalScoreFeature.java
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.Map;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.ltr.FeatureLoggerTestUtils;
import org.apache.solr.ltr.TestRerankBase;
import org.apache.solr.ltr.model.LinearModel;
import org.junit.AfterClass;
@@ -132,17 +133,17 @@ public class TestOriginalScoreFeature extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/numFound/==4");
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[0]/fv=='origScore:" + doc0Score + ";c2:2.0'");
+ "/response/docs/[0]/fv=='" + FeatureLoggerTestUtils.toFeatureVector("origScore", doc0Score, "c2", "2.0")+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='8'");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[1]/fv=='origScore:" + doc1Score + ";c2:2.0'");
+ "/response/docs/[1]/fv=='" + FeatureLoggerTestUtils.toFeatureVector("origScore", doc1Score, "c2", "2.0")+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='6'");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[2]/fv=='origScore:" + doc2Score + ";c2:2.0'");
+ "/response/docs/[2]/fv=='" + FeatureLoggerTestUtils.toFeatureVector("origScore", doc2Score, "c2", "2.0")+"'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[3]/id=='7'");
assertJQ("/query" + query.toQueryString(),
- "/response/docs/[3]/fv=='origScore:" + doc3Score + ";c2:2.0'");
+ "/response/docs/[3]/fv=='" + FeatureLoggerTestUtils.toFeatureVector("origScore", doc3Score, "c2", "2.0")+"'");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8542b2b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
index e4b0526..990a23f 100644
--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
@@ -81,6 +81,9 @@
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
+ <lib dir="${solr.install.dir:../../../..}/contrib/ltr/lib/" regex=".*\.jar" />
+ <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-ltr-\d.*\.jar" />
+
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
@@ -504,6 +507,23 @@
showItems="32" />
-->
+ <!-- Feature Values Cache
+
+ Cache used by the Learning To Rank (LTR) contrib module.
+
+ You will need to set the solr.ltr.enabled system property
+ when running solr to run with ltr enabled:
+ -Dsolr.ltr.enabled=true
+
+ https://cwiki.apache.org/confluence/display/solr/Result+Reranking
+ -->
+ <cache enable="${solr.ltr.enabled:false}" name="QUERY_DOC_FV"
+ class="solr.search.LRUCache"
+ size="4096"
+ initialSize="2048"
+ autowarmCount="4096"
+ regenerator="solr.search.NoOpRegenerator" />
+
<!-- Custom Cache
Example of a generic cache. These caches may be accessed by
@@ -1586,8 +1606,19 @@
<valueSourceParser name="myfunc"
class="com.mycompany.MyValueSourceParser" />
-->
-
-
+
+ <!-- LTR query parser
+
+ You will need to set the solr.ltr.enabled system property
+ when running solr to run with ltr enabled:
+ -Dsolr.ltr.enabled=true
+
+ https://cwiki.apache.org/confluence/display/solr/Result+Reranking
+
+ Query parser is used to rerank top docs with a provided model
+ -->
+ <queryParser enable="${solr.ltr.enabled:false}" name="ltr" class="org.apache.solr.ltr.search.LTRQParserPlugin"/>
+
<!-- Document Transformers
http://wiki.apache.org/solr/DocTransformers
-->
@@ -1611,5 +1642,22 @@
EditorialMarkerFactory will do exactly that:
<transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" />
-->
-
+
+ <!--
+ LTR Transformer will encode the document features in the response. For each document the transformer
+ will add the features as an extra field in the response. The name of the field will be the
+ name of the transformer enclosed between brackets (in this case [features]).
+ In order to get the feature vector you will have to specify that you
+ want the field (e.g., fl="*,[features])
+
+ You will need to set the solr.ltr.enabled system property
+ when running solr to run with ltr enabled:
+ -Dsolr.ltr.enabled=true
+
+ https://cwiki.apache.org/confluence/display/solr/Result+Reranking
+ -->
+ <transformer enable="${solr.ltr.enabled:false}" name="features" class="org.apache.solr.ltr.response.transform.LTRFeatureLoggerTransformerFactory">
+ <str name="fvCacheName">QUERY_DOC_FV</str>
+ </transformer>
+
</config>