You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2016/12/08 18:44:05 UTC
[06/13] lucene-solr:branch_6x: SOLR-8542: Adds Solr Learning to Rank
(LTR) plugin for reranking results with machine learning models. (Michael
Nilsson, Diego Ceccarelli, Joshua Pantony, Jon Dorando, Naveen Santhapuri,
Alessandro Benedetti, David Grohmann
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java
new file mode 100644
index 0000000..cb7c1a0
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.feature;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SyntaxError;
+/**
+ * This feature allows you to reuse any Solr query as a feature. The value
+ * of the feature will be the score of the given query for the current document.
+ * See <a href="https://cwiki.apache.org/confluence/display/solr/Other+Parsers">Solr documentation of other parsers</a> you can use as a feature.
+ * Example configurations:
+ * <pre>[{ "name": "isBook",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params":{ "fq": ["{!terms f=category}book"] }
+},
+{
+ "name": "documentRecency",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": {
+ "q": "{!func}recip( ms(NOW,publish_date), 3.16e-11, 1, 1)"
+ }
+}]</pre>
+ **/
+public class SolrFeature extends Feature {
+
+ private String df;
+ private String q;
+ private List<String> fq;
+
+ public String getDf() {
+ return df;
+ }
+
+ public void setDf(String df) {
+ this.df = df;
+ }
+
+ public String getQ() {
+ return q;
+ }
+
+ public void setQ(String q) {
+ this.q = q;
+ }
+
+ public List<String> getFq() {
+ return fq;
+ }
+
+ public void setFq(List<String> fq) {
+ this.fq = fq;
+ }
+
+ public SolrFeature(String name, Map<String,Object> params) {
+ super(name, params);
+ }
+
+ @Override
+ public LinkedHashMap<String,Object> paramsToMap() {
+ final LinkedHashMap<String,Object> params = new LinkedHashMap<>(3, 1.0f);
+ if (df != null) {
+ params.put("df", df);
+ }
+ if (q != null) {
+ params.put("q", q);
+ }
+ if (fq != null) {
+ params.put("fq", fq);
+ }
+ return params;
+ }
+
+ @Override
+ public FeatureWeight createWeight(IndexSearcher searcher, boolean needsScores,
+ SolrQueryRequest request, Query originalQuery, Map<String,String[]> efi)
+ throws IOException {
+ return new SolrFeatureWeight(searcher, request, originalQuery, efi);
+ }
+
+ @Override
+ protected void validate() throws FeatureException {
+ if ((q == null || q.isEmpty()) &&
+ ((fq == null) || fq.isEmpty())) {
+ throw new FeatureException(getClass().getSimpleName()+
+ ": Q or FQ must be provided");
+ }
+ }
+ /**
+ * Weight for a SolrFeature
+ **/
+ public class SolrFeatureWeight extends FeatureWeight {
+ Weight solrQueryWeight;
+ Query query;
+ List<Query> queryAndFilters;
+
+ public SolrFeatureWeight(IndexSearcher searcher,
+ SolrQueryRequest request, Query originalQuery, Map<String,String[]> efi) throws IOException {
+ super(SolrFeature.this, searcher, request, originalQuery, efi);
+ try {
+ String solrQuery = q;
+ final List<String> fqs = fq;
+
+ if ((solrQuery == null) || solrQuery.isEmpty()) {
+ solrQuery = "*:*";
+ }
+
+ solrQuery = macroExpander.expand(solrQuery);
+ if (solrQuery == null) {
+ throw new FeatureException(this.getClass().getSimpleName()+" requires efi parameter that was not passed in request.");
+ }
+
+ final SolrQueryRequest req = makeRequest(request.getCore(), solrQuery,
+ fqs, df);
+ if (req == null) {
+ throw new IOException("ERROR: No parameters provided");
+ }
+
+ // Build the filter queries
+ queryAndFilters = new ArrayList<Query>(); // If there are no fqs we just want an empty list
+ if (fqs != null) {
+ for (String fq : fqs) {
+ if ((fq != null) && (fq.trim().length() != 0)) {
+ fq = macroExpander.expand(fq);
+ final QParser fqp = QParser.getParser(fq, req);
+ final Query filterQuery = fqp.getQuery();
+ if (filterQuery != null) {
+ queryAndFilters.add(filterQuery);
+ }
+ }
+ }
+ }
+
+ final QParser parser = QParser.getParser(solrQuery, req);
+ query = parser.parse();
+
+ // Query can be null if there was no input to parse, for instance if you
+ // make a phrase query with "to be", and the analyzer removes all the
+ // words
+ // leaving nothing for the phrase query to parse.
+ if (query != null) {
+ queryAndFilters.add(query);
+ solrQueryWeight = searcher.createNormalizedWeight(query, true);
+ }
+ } catch (final SyntaxError e) {
+ throw new FeatureException("Failed to parse feature query.", e);
+ }
+ }
+
+ private LocalSolrQueryRequest makeRequest(SolrCore core, String solrQuery,
+ List<String> fqs, String df) {
+ final NamedList<String> returnList = new NamedList<String>();
+ if ((solrQuery != null) && !solrQuery.isEmpty()) {
+ returnList.add(CommonParams.Q, solrQuery);
+ }
+ if (fqs != null) {
+ for (final String fq : fqs) {
+ returnList.add(CommonParams.FQ, fq);
+ }
+ }
+ if ((df != null) && !df.isEmpty()) {
+ returnList.add(CommonParams.DF, df);
+ }
+ if (returnList.size() > 0) {
+ return new LocalSolrQueryRequest(core, returnList);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public FeatureScorer scorer(LeafReaderContext context) throws IOException {
+ Scorer solrScorer = null;
+ if (solrQueryWeight != null) {
+ solrScorer = solrQueryWeight.scorer(context);
+ }
+
+ final DocIdSetIterator idItr = getDocIdSetIteratorFromQueries(
+ queryAndFilters, context);
+ if (idItr != null) {
+ return solrScorer == null ? new ValueFeatureScorer(this, 1f, idItr)
+ : new SolrFeatureScorer(this, solrScorer,
+ new SolrFeatureScorerIterator(idItr, solrScorer.iterator()));
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Given a list of Solr filters/queries, return a doc iterator that
+ * traverses over the documents that matched all the criteria of the
+ * queries.
+ *
+ * @param queries
+ * Filtering criteria to match documents against
+ * @param context
+ * Index reader
+ * @return DocIdSetIterator to traverse documents that matched all filter
+ * criteria
+ */
+ private DocIdSetIterator getDocIdSetIteratorFromQueries(List<Query> queries,
+ LeafReaderContext context) throws IOException {
+ final SolrIndexSearcher.ProcessedFilter pf = ((SolrIndexSearcher) searcher)
+ .getProcessedFilter(null, queries);
+ final Bits liveDocs = context.reader().getLiveDocs();
+
+ DocIdSetIterator idIter = null;
+ if (pf.filter != null) {
+ final DocIdSet idSet = pf.filter.getDocIdSet(context, liveDocs);
+ if (idSet != null) {
+ idIter = idSet.iterator();
+ }
+ }
+
+ return idIter;
+ }
+
+ /**
+ * Scorer for a SolrFeature
+ **/
+ public class SolrFeatureScorer extends FeatureScorer {
+ final private Scorer solrScorer;
+
+ public SolrFeatureScorer(FeatureWeight weight, Scorer solrScorer,
+ SolrFeatureScorerIterator itr) {
+ super(weight, itr);
+ this.solrScorer = solrScorer;
+ }
+
+ @Override
+ public float score() throws IOException {
+ try {
+ return solrScorer.score();
+ } catch (UnsupportedOperationException e) {
+ throw new FeatureException(
+ e.toString() + ": " +
+ "Unable to extract feature for "
+ + name, e);
+ }
+ }
+ }
+
+ /**
+ * An iterator that allows to iterate only on the documents for which a feature has
+ * a value.
+ **/
+ public class SolrFeatureScorerIterator extends DocIdSetIterator {
+
+ final private DocIdSetIterator filterIterator;
+ final private DocIdSetIterator scorerFilter;
+
+ SolrFeatureScorerIterator(DocIdSetIterator filterIterator,
+ DocIdSetIterator scorerFilter) {
+ this.filterIterator = filterIterator;
+ this.scorerFilter = scorerFilter;
+ }
+
+ @Override
+ public int docID() {
+ return filterIterator.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ int docID = filterIterator.nextDoc();
+ scorerFilter.advance(docID);
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ // We use iterator to catch the scorer up since
+ // that checks if the target id is in the query + all the filters
+ int docID = filterIterator.advance(target);
+ scorerFilter.advance(docID);
+ return docID;
+ }
+
+ @Override
+ public long cost() {
+ return filterIterator.cost() + scorerFilter.cost();
+ }
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/ValueFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/ValueFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/ValueFeature.java
new file mode 100644
index 0000000..61aa9e5
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/ValueFeature.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.feature;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.solr.request.SolrQueryRequest;
+/**
+ * This feature allows to return a constant given value for the current document.
+ *
+ * Example configuration:
+ * <pre>{
+ "name" : "userFromMobile",
+ "class" : "org.apache.solr.ltr.feature.ValueFeature",
+ "params" : { "value" : "${userFromMobile}", "required":true }
+ }</pre>
+ *
+ *You can place a constant value like "1.3f" in the value params, but many times you
+ *would want to pass in external information to use per request. For instance, maybe
+ *you want to rank things differently if the search came from a mobile device, or maybe
+ *you want to use your external query intent system as a feature.
+ *In the rerank request you can pass in rq={... efi.userFromMobile=1}, and the above
+ *feature will return 1 for all the docs for that request. If required is set to true,
+ *the request will return an error since you failed to pass in the efi, otherwise if will
+ *just skip the feature and use a default value of 0 instead.
+ **/
+public class ValueFeature extends Feature {
+ private float configValue = -1f;
+ private String configValueStr = null;
+
+ private Object value = null;
+ private Boolean required = null;
+
+ public Object getValue() {
+ return value;
+ }
+
+ public void setValue(Object value) {
+ this.value = value;
+ if (value instanceof String) {
+ configValueStr = (String) value;
+ } else if (value instanceof Double) {
+ configValue = ((Double) value).floatValue();
+ } else if (value instanceof Float) {
+ configValue = ((Float) value).floatValue();
+ } else if (value instanceof Integer) {
+ configValue = ((Integer) value).floatValue();
+ } else if (value instanceof Long) {
+ configValue = ((Long) value).floatValue();
+ } else {
+ throw new FeatureException("Invalid type for 'value' in params for " + this);
+ }
+ }
+
+ public boolean isRequired() {
+ return Boolean.TRUE.equals(required);
+ }
+
+ public void setRequired(boolean required) {
+ this.required = required;
+ }
+
+ @Override
+ public LinkedHashMap<String,Object> paramsToMap() {
+ final LinkedHashMap<String,Object> params = new LinkedHashMap<>(2, 1.0f);
+ params.put("value", value);
+ if (required != null) {
+ params.put("required", required);
+ }
+ return params;
+ }
+
+ @Override
+ protected void validate() throws FeatureException {
+ if (configValueStr != null && configValueStr.trim().isEmpty()) {
+ throw new FeatureException("Empty field 'value' in params for " + this);
+ }
+ }
+
+ public ValueFeature(String name, Map<String,Object> params) {
+ super(name, params);
+ }
+
+ @Override
+ public FeatureWeight createWeight(IndexSearcher searcher, boolean needsScores,
+ SolrQueryRequest request, Query originalQuery, Map<String,String[]> efi)
+ throws IOException {
+ return new ValueFeatureWeight(searcher, request, originalQuery, efi);
+ }
+
+ public class ValueFeatureWeight extends FeatureWeight {
+
+ final protected Float featureValue;
+
+ public ValueFeatureWeight(IndexSearcher searcher,
+ SolrQueryRequest request, Query originalQuery, Map<String,String[]> efi) {
+ super(ValueFeature.this, searcher, request, originalQuery, efi);
+ if (configValueStr != null) {
+ final String expandedValue = macroExpander.expand(configValueStr);
+ if (expandedValue != null) {
+ featureValue = Float.parseFloat(expandedValue);
+ } else if (isRequired()) {
+ throw new FeatureException(this.getClass().getSimpleName() + " requires efi parameter that was not passed in request.");
+ } else {
+ featureValue=null;
+ }
+ } else {
+ featureValue = configValue;
+ }
+ }
+
+ @Override
+ public FeatureScorer scorer(LeafReaderContext context) throws IOException {
+ if(featureValue!=null) {
+ return new ValueFeatureScorer(this, featureValue,
+ DocIdSetIterator.all(DocIdSetIterator.NO_MORE_DOCS));
+ } else {
+ return null;
+ }
+ }
+
+
+
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/package-info.java
new file mode 100644
index 0000000..456fffc
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains Feature related classes
+ */
+package org.apache.solr.ltr.feature;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
new file mode 100644
index 0000000..9edcfe5
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.model;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Explanation;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.feature.FeatureException;
+import org.apache.solr.ltr.norm.IdentityNormalizer;
+import org.apache.solr.ltr.norm.Normalizer;
+import org.apache.solr.util.SolrPluginUtils;
+
+/**
+ * A scoring model computes scores that can be used to rerank documents.
+ * <p>
+ * A scoring model consists of
+ * <ul>
+ * <li> a list of features ({@link Feature}) and
+ * <li> a list of normalizers ({@link Normalizer}) plus
+ * <li> parameters or configuration to represent the scoring algorithm.
+ * </ul>
+ * <p>
+ * Example configuration (snippet):
+ * <pre>{
+ "class" : "...",
+ "name" : "myModelName",
+ "features" : [
+ {
+ "name" : "isBook"
+ },
+ {
+ "name" : "originalScore",
+ "norm": {
+ "class" : "org.apache.solr.ltr.norm.StandardNormalizer",
+ "params" : { "avg":"100", "std":"10" }
+ }
+ },
+ {
+ "name" : "price",
+ "norm": {
+ "class" : "org.apache.solr.ltr.norm.MinMaxNormalizer",
+ "params" : { "min":"0", "max":"1000" }
+ }
+ }
+ ],
+ "params" : {
+ ...
+ }
+}</pre>
+ * <p>
+ * {@link LTRScoringModel} is an abstract class and concrete classes must
+ * implement the {@link #score(float[])} and
+ * {@link #explain(LeafReaderContext, int, float, List)} methods.
+ */
+public abstract class LTRScoringModel {
+
+ protected final String name;
+ private final String featureStoreName;
+ protected final List<Feature> features;
+ private final List<Feature> allFeatures;
+ private final Map<String,Object> params;
+ private final List<Normalizer> norms;
+
+ public static LTRScoringModel getInstance(SolrResourceLoader solrResourceLoader,
+ String className, String name, List<Feature> features,
+ List<Normalizer> norms,
+ String featureStoreName, List<Feature> allFeatures,
+ Map<String,Object> params) throws ModelException {
+ final LTRScoringModel model;
+ try {
+ // create an instance of the model
+ model = solrResourceLoader.newInstance(
+ className,
+ LTRScoringModel.class,
+ new String[0], // no sub packages
+ new Class[] { String.class, List.class, List.class, String.class, List.class, Map.class },
+ new Object[] { name, features, norms, featureStoreName, allFeatures, params });
+ if (params != null) {
+ SolrPluginUtils.invokeSetters(model, params.entrySet());
+ }
+ } catch (final Exception e) {
+ throw new ModelException("Model type does not exist " + className, e);
+ }
+ model.validate();
+ return model;
+ }
+
+ public LTRScoringModel(String name, List<Feature> features,
+ List<Normalizer> norms,
+ String featureStoreName, List<Feature> allFeatures,
+ Map<String,Object> params) {
+ this.name = name;
+ this.features = features;
+ this.featureStoreName = featureStoreName;
+ this.allFeatures = allFeatures;
+ this.params = params;
+ this.norms = norms;
+ }
+
+ /**
+ * Validate that settings make sense and throws
+ * {@link ModelException} if they do not make sense.
+ */
+ protected void validate() throws ModelException {
+ if (features.isEmpty()) {
+ throw new ModelException("no features declared for model "+name);
+ }
+ final HashSet<String> featureNames = new HashSet<>();
+ for (final Feature feature : features) {
+ final String featureName = feature.getName();
+ if (!featureNames.add(featureName)) {
+ throw new ModelException("duplicated feature "+featureName+" in model "+name);
+ }
+ }
+ if (features.size() != norms.size()) {
+ throw new ModelException("counted "+features.size()+" features and "+norms.size()+" norms in model "+name);
+ }
+ }
+
+ /**
+ * @return the norms
+ */
+ public List<Normalizer> getNorms() {
+ return Collections.unmodifiableList(norms);
+ }
+
+ /**
+ * @return the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return the features
+ */
+ public List<Feature> getFeatures() {
+ return Collections.unmodifiableList(features);
+ }
+
+ public Map<String,Object> getParams() {
+ return params;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = (prime * result) + ((features == null) ? 0 : features.hashCode());
+ result = (prime * result) + ((name == null) ? 0 : name.hashCode());
+ result = (prime * result) + ((params == null) ? 0 : params.hashCode());
+ result = (prime * result) + ((norms == null) ? 0 : norms.hashCode());
+ result = (prime * result) + ((featureStoreName == null) ? 0 : featureStoreName.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final LTRScoringModel other = (LTRScoringModel) obj;
+ if (features == null) {
+ if (other.features != null) {
+ return false;
+ }
+ } else if (!features.equals(other.features)) {
+ return false;
+ }
+ if (norms == null) {
+ if (other.norms != null) {
+ return false;
+ }
+ } else if (!norms.equals(other.norms)) {
+ return false;
+ }
+ if (name == null) {
+ if (other.name != null) {
+ return false;
+ }
+ } else if (!name.equals(other.name)) {
+ return false;
+ }
+ if (params == null) {
+ if (other.params != null) {
+ return false;
+ }
+ } else if (!params.equals(other.params)) {
+ return false;
+ }
+ if (featureStoreName == null) {
+ if (other.featureStoreName != null) {
+ return false;
+ }
+ } else if (!featureStoreName.equals(other.featureStoreName)) {
+ return false;
+ }
+
+
+ return true;
+ }
+
+ public boolean hasParams() {
+ return !((params == null) || params.isEmpty());
+ }
+
+ public Collection<Feature> getAllFeatures() {
+ return allFeatures;
+ }
+
+ public String getFeatureStoreName() {
+ return featureStoreName;
+ }
+
+ /**
+ * Given a list of normalized values for all features a scoring algorithm
+ * cares about, calculate and return a score.
+ *
+ * @param modelFeatureValuesNormalized
+ * List of normalized feature values. Each feature is identified by
+ * its id, which is the index in the array
+ * @return The final score for a document
+ */
+ public abstract float score(float[] modelFeatureValuesNormalized);
+
+ /**
+ * Similar to the score() function, except it returns an explanation of how
+ * the features were used to calculate the score.
+ *
+ * @param context
+ * Context the document is in
+ * @param doc
+ * Document to explain
+ * @param finalScore
+ * Original score
+ * @param featureExplanations
+ * Explanations for each feature calculation
+ * @return Explanation for the scoring of a document
+ */
+ public abstract Explanation explain(LeafReaderContext context, int doc,
+ float finalScore, List<Explanation> featureExplanations);
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(name="+getName()+")";
+ }
+
+ /**
+ * Goes through all the stored feature values, and calculates the normalized
+ * values for all the features that will be used for scoring.
+ */
+ public void normalizeFeaturesInPlace(float[] modelFeatureValues) {
+ float[] modelFeatureValuesNormalized = modelFeatureValues;
+ if (modelFeatureValues.length != norms.size()) {
+ throw new FeatureException("Must have normalizer for every feature");
+ }
+ for(int idx = 0; idx < modelFeatureValuesNormalized.length; ++idx) {
+ modelFeatureValuesNormalized[idx] =
+ norms.get(idx).normalize(modelFeatureValuesNormalized[idx]);
+ }
+ }
+
+ public Explanation getNormalizerExplanation(Explanation e, int idx) {
+ Normalizer n = norms.get(idx);
+ if (n != IdentityNormalizer.INSTANCE) {
+ return n.explain(e);
+ }
+ return e;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
new file mode 100644
index 0000000..57fc5ad
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LinearModel.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Explanation;
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.norm.Normalizer;
+
+/**
+ * A scoring model that computes scores using a dot product.
+ * Example models are RankSVM and Pranking.
+ * <p>
+ * Example configuration:
+ * <pre>{
+ "class" : "org.apache.solr.ltr.model.LinearModel",
+ "name" : "myModelName",
+ "features" : [
+ { "name" : "userTextTitleMatch" },
+ { "name" : "originalScore" },
+ { "name" : "isBook" }
+ ],
+ "params" : {
+ "weights" : {
+ "userTextTitleMatch" : 1.0,
+ "originalScore" : 0.5,
+ "isBook" : 0.1
+ }
+ }
+}</pre>
+ * <p>
+ * Background reading:
+ * <ul>
+ * <li> <a href="http://www.cs.cornell.edu/people/tj/publications/joachims_02c.pdf">
+ * Thorsten Joachims. Optimizing Search Engines Using Clickthrough Data.
+ * Proceedings of the ACM Conference on Knowledge Discovery and Data Mining (KDD), ACM, 2002.</a>
+ * </ul>
+ * <ul>
+ * <li> <a href="https://papers.nips.cc/paper/2023-pranking-with-ranking.pdf">
+ * Koby Crammer and Yoram Singer. Pranking with Ranking.
+ * Advances in Neural Information Processing Systems (NIPS), 2001.</a>
+ * </ul>
+ */
+public class LinearModel extends LTRScoringModel {
+
+ protected Float[] featureToWeight;
+
+ public void setWeights(Object weights) {
+ final Map<String,Double> modelWeights = (Map<String,Double>) weights;
+ for (int ii = 0; ii < features.size(); ++ii) {
+ final String key = features.get(ii).getName();
+ final Double val = modelWeights.get(key);
+ featureToWeight[ii] = (val == null ? null : new Float(val.floatValue()));
+ }
+ }
+
+ public LinearModel(String name, List<Feature> features,
+ List<Normalizer> norms,
+ String featureStoreName, List<Feature> allFeatures,
+ Map<String,Object> params) {
+ super(name, features, norms, featureStoreName, allFeatures, params);
+ featureToWeight = new Float[features.size()];
+ }
+
+ @Override
+ protected void validate() throws ModelException {
+ super.validate();
+
+ final ArrayList<String> missingWeightFeatureNames = new ArrayList<String>();
+ for (int i = 0; i < features.size(); ++i) {
+ if (featureToWeight[i] == null) {
+ missingWeightFeatureNames.add(features.get(i).getName());
+ }
+ }
+ if (missingWeightFeatureNames.size() == features.size()) {
+ throw new ModelException("Model " + name + " doesn't contain any weights");
+ }
+ if (!missingWeightFeatureNames.isEmpty()) {
+ throw new ModelException("Model " + name + " lacks weight(s) for "+missingWeightFeatureNames);
+ }
+ }
+
+ @Override
+ public float score(float[] modelFeatureValuesNormalized) {
+ float score = 0;
+ for (int i = 0; i < modelFeatureValuesNormalized.length; ++i) {
+ score += modelFeatureValuesNormalized[i] * featureToWeight[i];
+ }
+ return score;
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc,
+ float finalScore, List<Explanation> featureExplanations) {
+ final List<Explanation> details = new ArrayList<>();
+ int index = 0;
+
+ for (final Explanation featureExplain : featureExplanations) {
+ final List<Explanation> featureDetails = new ArrayList<>();
+ featureDetails.add(Explanation.match(featureToWeight[index],
+ "weight on feature"));
+ featureDetails.add(featureExplain);
+
+ details.add(Explanation.match(featureExplain.getValue()
+ * featureToWeight[index], "prod of:", featureDetails));
+ index++;
+ }
+
+ return Explanation.match(finalScore, toString()
+ + " model applied to features, sum of:", details);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(getClass().getSimpleName());
+ sb.append("(name=").append(getName());
+ sb.append(",featureWeights=[");
+ for (int ii = 0; ii < features.size(); ++ii) {
+ if (ii>0) {
+ sb.append(',');
+ }
+ final String key = features.get(ii).getName();
+ sb.append(key).append('=').append(featureToWeight[ii]);
+ }
+ sb.append("])");
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/ModelException.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/ModelException.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/ModelException.java
new file mode 100644
index 0000000..de8786d
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/ModelException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.model;
+
+public class ModelException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public ModelException(String message) {
+ super(message);
+ }
+
+ public ModelException(String message, Exception cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
new file mode 100644
index 0000000..4fa595e
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Explanation;
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.norm.Normalizer;
+import org.apache.solr.util.SolrPluginUtils;
+
+/**
+ * A scoring model that computes scores based on the summation of multiple weighted trees.
+ * Example models are LambdaMART and Gradient Boosted Regression Trees (GBRT) .
+ * <p>
+ * Example configuration:
+<pre>{
+ "class" : "org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name" : "multipleadditivetreesmodel",
+ "features":[
+ { "name" : "userTextTitleMatch"},
+ { "name" : "originalScore"}
+ ],
+ "params" : {
+ "trees" : [
+ {
+ "weight" : 1,
+ "root": {
+ "feature" : "userTextTitleMatch",
+ "threshold" : 0.5,
+ "left" : {
+ "value" : -100
+ },
+ "right" : {
+ "feature" : "originalScore",
+ "threshold" : 10.0,
+ "left" : {
+ "value" : 50
+ },
+ "right" : {
+ "value" : 75
+ }
+ }
+ }
+ },
+ {
+ "weight" : 2,
+ "root" : {
+ "value" : -10
+ }
+ }
+ ]
+ }
+}</pre>
+ * <p>
+ * Background reading:
+ * <ul>
+ * <li> <a href="http://research.microsoft.com/pubs/132652/MSR-TR-2010-82.pdf">
+ * Christopher J.C. Burges. From RankNet to LambdaRank to LambdaMART: An Overview.
+ * Microsoft Research Technical Report MSR-TR-2010-82.</a>
+ * </ul>
+ * <ul>
+ * <li> <a href="https://papers.nips.cc/paper/3305-a-general-boosting-method-and-its-application-to-learning-ranking-functions-for-web-search.pdf">
+ * Z. Zheng, H. Zha, T. Zhang, O. Chapelle, K. Chen, and G. Sun. A General Boosting Method and its Application to Learning Ranking Functions for Web Search.
+ * Advances in Neural Information Processing Systems (NIPS), 2007.</a>
+ * </ul>
+ */
+public class MultipleAdditiveTreesModel extends LTRScoringModel {
+
+ private final HashMap<String,Integer> fname2index;
+ private List<RegressionTree> trees;
+
+ private RegressionTree createRegressionTree(Map<String,Object> map) {
+ final RegressionTree rt = new RegressionTree();
+ if (map != null) {
+ SolrPluginUtils.invokeSetters(rt, map.entrySet());
+ }
+ return rt;
+ }
+
+ private RegressionTreeNode createRegressionTreeNode(Map<String,Object> map) {
+ final RegressionTreeNode rtn = new RegressionTreeNode();
+ if (map != null) {
+ SolrPluginUtils.invokeSetters(rtn, map.entrySet());
+ }
+ return rtn;
+ }
+
+ public class RegressionTreeNode {
+ private static final float NODE_SPLIT_SLACK = 1E-6f;
+
+ private float value = 0f;
+ private String feature;
+ private int featureIndex = -1;
+ private Float threshold;
+ private RegressionTreeNode left;
+ private RegressionTreeNode right;
+
+ public void setValue(float value) {
+ this.value = value;
+ }
+
+ public void setValue(String value) {
+ this.value = Float.parseFloat(value);
+ }
+
+ public void setFeature(String feature) {
+ this.feature = feature;
+ final Integer idx = fname2index.get(this.feature);
+ // this happens if the tree specifies a feature that does not exist
+ // this could be due to lambdaSmart building off of pre-existing trees
+ // that use a feature that is no longer output during feature extraction
+ featureIndex = (idx == null) ? -1 : idx;
+ }
+
+ public void setThreshold(float threshold) {
+ this.threshold = threshold + NODE_SPLIT_SLACK;
+ }
+
+ public void setThreshold(String threshold) {
+ this.threshold = Float.parseFloat(threshold) + NODE_SPLIT_SLACK;
+ }
+
+ public void setLeft(Object left) {
+ this.left = createRegressionTreeNode((Map<String,Object>) left);
+ }
+
+ public void setRight(Object right) {
+ this.right = createRegressionTreeNode((Map<String,Object>) right);
+ }
+
+ public boolean isLeaf() {
+ return feature == null;
+ }
+
+ public float score(float[] featureVector) {
+ if (isLeaf()) {
+ return value;
+ }
+
+ // unsupported feature (tree is looking for a feature that does not exist)
+ if ((featureIndex < 0) || (featureIndex >= featureVector.length)) {
+ return 0f;
+ }
+
+ if (featureVector[featureIndex] <= threshold) {
+ return left.score(featureVector);
+ } else {
+ return right.score(featureVector);
+ }
+ }
+
+ public String explain(float[] featureVector) {
+ if (isLeaf()) {
+ return "val: " + value;
+ }
+
+ // unsupported feature (tree is looking for a feature that does not exist)
+ if ((featureIndex < 0) || (featureIndex >= featureVector.length)) {
+ return "'" + feature + "' does not exist in FV, Return Zero";
+ }
+
+ // could store extra information about how much training data supported
+ // each branch and report
+ // that here
+
+ if (featureVector[featureIndex] <= threshold) {
+ String rval = "'" + feature + "':" + featureVector[featureIndex] + " <= "
+ + threshold + ", Go Left | ";
+ return rval + left.explain(featureVector);
+ } else {
+ String rval = "'" + feature + "':" + featureVector[featureIndex] + " > "
+ + threshold + ", Go Right | ";
+ return rval + right.explain(featureVector);
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ if (isLeaf()) {
+ sb.append(value);
+ } else {
+ sb.append("(feature=").append(feature);
+ sb.append(",threshold=").append(threshold.floatValue()-NODE_SPLIT_SLACK);
+ sb.append(",left=").append(left);
+ sb.append(",right=").append(right);
+ sb.append(')');
+ }
+ return sb.toString();
+ }
+
+ public RegressionTreeNode() {
+ }
+
+ public void validate() throws ModelException {
+ if (isLeaf()) {
+ if (left != null || right != null) {
+ throw new ModelException("MultipleAdditiveTreesModel tree node is leaf with left="+left+" and right="+right);
+ }
+ return;
+ }
+ if (null == threshold) {
+ throw new ModelException("MultipleAdditiveTreesModel tree node is missing threshold");
+ }
+ if (null == left) {
+ throw new ModelException("MultipleAdditiveTreesModel tree node is missing left");
+ } else {
+ left.validate();
+ }
+ if (null == right) {
+ throw new ModelException("MultipleAdditiveTreesModel tree node is missing right");
+ } else {
+ right.validate();
+ }
+ }
+
+ }
+
+ public class RegressionTree {
+
+ private Float weight;
+ private RegressionTreeNode root;
+
+ public void setWeight(float weight) {
+ this.weight = new Float(weight);
+ }
+
+ public void setWeight(String weight) {
+ this.weight = new Float(weight);
+ }
+
+ public void setRoot(Object root) {
+ this.root = createRegressionTreeNode((Map<String,Object>)root);
+ }
+
+ public float score(float[] featureVector) {
+ return weight.floatValue() * root.score(featureVector);
+ }
+
+ public String explain(float[] featureVector) {
+ return root.explain(featureVector);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append("(weight=").append(weight);
+ sb.append(",root=").append(root);
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public RegressionTree() {
+ }
+
+ public void validate() throws ModelException {
+ if (weight == null) {
+ throw new ModelException("MultipleAdditiveTreesModel tree doesn't contain a weight");
+ }
+ if (root == null) {
+ throw new ModelException("MultipleAdditiveTreesModel tree doesn't contain a tree");
+ } else {
+ root.validate();
+ }
+ }
+ }
+
+ public void setTrees(Object trees) {
+ this.trees = new ArrayList<RegressionTree>();
+ for (final Object o : (List<Object>) trees) {
+ final RegressionTree rt = createRegressionTree((Map<String,Object>) o);
+ this.trees.add(rt);
+ }
+ }
+
+ public MultipleAdditiveTreesModel(String name, List<Feature> features,
+ List<Normalizer> norms,
+ String featureStoreName, List<Feature> allFeatures,
+ Map<String,Object> params) {
+ super(name, features, norms, featureStoreName, allFeatures, params);
+
+ fname2index = new HashMap<String,Integer>();
+ for (int i = 0; i < features.size(); ++i) {
+ final String key = features.get(i).getName();
+ fname2index.put(key, i);
+ }
+ }
+
+ @Override
+ protected void validate() throws ModelException {
+ super.validate();
+ if (trees == null) {
+ throw new ModelException("no trees declared for model "+name);
+ }
+ for (RegressionTree tree : trees) {
+ tree.validate();
+ }
+ }
+
+ @Override
+ public float score(float[] modelFeatureValuesNormalized) {
+ float score = 0;
+ for (final RegressionTree t : trees) {
+ score += t.score(modelFeatureValuesNormalized);
+ }
+ return score;
+ }
+
+ // /////////////////////////////////////////
+ // produces a string that looks like:
+ // 40.0 = multipleadditivetreesmodel [ org.apache.solr.ltr.model.MultipleAdditiveTreesModel ]
+ // model applied to
+ // features, sum of:
+ // 50.0 = tree 0 | 'matchedTitle':1.0 > 0.500001, Go Right |
+ // 'this_feature_doesnt_exist' does not
+ // exist in FV, Go Left | val: 50.0
+ // -10.0 = tree 1 | val: -10.0
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc,
+ float finalScore, List<Explanation> featureExplanations) {
+ final float[] fv = new float[featureExplanations.size()];
+ int index = 0;
+ for (final Explanation featureExplain : featureExplanations) {
+ fv[index] = featureExplain.getValue();
+ index++;
+ }
+
+ final List<Explanation> details = new ArrayList<>();
+ index = 0;
+
+ for (final RegressionTree t : trees) {
+ final float score = t.score(fv);
+ final Explanation p = Explanation.match(score, "tree " + index + " | "
+ + t.explain(fv));
+ details.add(p);
+ index++;
+ }
+
+ return Explanation.match(finalScore, toString()
+ + " model applied to features, sum of:", details);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(getClass().getSimpleName());
+ sb.append("(name=").append(getName());
+ sb.append(",trees=[");
+ for (int ii = 0; ii < trees.size(); ++ii) {
+ if (ii>0) {
+ sb.append(',');
+ }
+ sb.append(trees.get(ii));
+ }
+ sb.append("])");
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/package-info.java
new file mode 100644
index 0000000..32bd626
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains Model related classes
+ */
+package org.apache.solr.ltr.model;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/IdentityNormalizer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/IdentityNormalizer.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/IdentityNormalizer.java
new file mode 100644
index 0000000..a3d1026
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/IdentityNormalizer.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.norm;
+
+import java.util.LinkedHashMap;
+
+/**
+ * A Normalizer that normalizes a feature value to itself. This is the
+ * default normalizer class, if no normalizer is configured then the
+ * IdentityNormalizer will be used.
+ */
+public class IdentityNormalizer extends Normalizer {
+
+ public static final IdentityNormalizer INSTANCE = new IdentityNormalizer();
+
+ public IdentityNormalizer() {
+
+ }
+
+ @Override
+ public float normalize(float value) {
+ return value;
+ }
+
+ @Override
+ public LinkedHashMap<String,Object> paramsToMap() {
+ return null;
+ }
+
+ @Override
+ protected void validate() throws NormalizerException {
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/MinMaxNormalizer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/MinMaxNormalizer.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/MinMaxNormalizer.java
new file mode 100644
index 0000000..92e233c
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/MinMaxNormalizer.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.norm;
+
+import java.util.LinkedHashMap;
+
+/**
+ * A Normalizer to scale a feature value using a (min,max) range.
+ * <p>
+ * Example configuration:
+<pre>
+"norm" : {
+ "class" : "org.apache.solr.ltr.norm.MinMaxNormalizer",
+ "params" : { "min":"0", "max":"50" }
+}
+</pre>
+ * Example normalizations:
+ * <ul>
+ * <li>-5 will be normalized to -0.1
+ * <li>55 will be normalized to 1.1
+ * <li>+5 will be normalized to +0.1
+ * </ul>
+ */
+public class MinMaxNormalizer extends Normalizer {
+
+ private float min = Float.NEGATIVE_INFINITY;
+ private float max = Float.POSITIVE_INFINITY;
+ private float delta = max - min;
+
+ private void updateDelta() {
+ delta = max - min;
+ }
+
+ public float getMin() {
+ return min;
+ }
+
+ public void setMin(float min) {
+ this.min = min;
+ updateDelta();
+ }
+
+ public void setMin(String min) {
+ this.min = Float.parseFloat(min);
+ updateDelta();
+ }
+
+ public float getMax() {
+ return max;
+ }
+
+ public void setMax(float max) {
+ this.max = max;
+ updateDelta();
+ }
+
+ public void setMax(String max) {
+ this.max = Float.parseFloat(max);
+ updateDelta();
+ }
+
+ @Override
+ protected void validate() throws NormalizerException {
+ if (delta == 0f) {
+ throw
+ new NormalizerException("MinMax Normalizer delta must not be zero " +
+ "| min = " + min + ",max = " + max + ",delta = " + delta);
+ }
+ }
+
+ @Override
+ public float normalize(float value) {
+ return (value - min) / delta;
+ }
+
+ @Override
+ public LinkedHashMap<String,Object> paramsToMap() {
+ final LinkedHashMap<String,Object> params = new LinkedHashMap<>(2, 1.0f);
+ params.put("min", min);
+ params.put("max", max);
+ return params;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(64); // default initialCapacity of 16 won't be enough
+ sb.append(getClass().getSimpleName()).append('(');
+ sb.append("min=").append(min);
+ sb.append(",max=").append(max).append(')');
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/Normalizer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/Normalizer.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/Normalizer.java
new file mode 100644
index 0000000..2b311f8
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/Normalizer.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.norm;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.lucene.search.Explanation;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.util.SolrPluginUtils;
+
+/**
+ * A normalizer normalizes the value of a feature. After the feature values
+ * have been computed, the {@link Normalizer#normalize(float)} methods will
+ * be called and the resulting values will be used by the model.
+ */
+public abstract class Normalizer {
+
+
+ public abstract float normalize(float value);
+
+ public abstract LinkedHashMap<String,Object> paramsToMap();
+
+ public Explanation explain(Explanation explain) {
+ final float normalized = normalize(explain.getValue());
+ final String explainDesc = "normalized using " + toString();
+
+ return Explanation.match(normalized, explainDesc, explain);
+ }
+
+ public static Normalizer getInstance(SolrResourceLoader solrResourceLoader,
+ String className, Map<String,Object> params) {
+ final Normalizer f = solrResourceLoader.newInstance(className, Normalizer.class);
+ if (params != null) {
+ SolrPluginUtils.invokeSetters(f, params.entrySet());
+ }
+ f.validate();
+ return f;
+ }
+
+ /**
+ * As part of creation of a normalizer instance, this function confirms
+ * that the normalizer parameters are valid.
+ *
+ * @throws NormalizerException
+ * Normalizer Exception
+ */
+ protected abstract void validate() throws NormalizerException;
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/NormalizerException.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/NormalizerException.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/NormalizerException.java
new file mode 100644
index 0000000..5b33f05
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/NormalizerException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.norm;
+
+public class NormalizerException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public NormalizerException(String message) {
+ super(message);
+ }
+
+ public NormalizerException(String message, Exception cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/StandardNormalizer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/StandardNormalizer.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/StandardNormalizer.java
new file mode 100644
index 0000000..7ab525c
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/StandardNormalizer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.norm;
+
+import java.util.LinkedHashMap;
+
+/**
+ * A Normalizer to scale a feature value around an average-and-standard-deviation distribution.
+ * <p>
+ * Example configuration:
+<pre>
+"norm" : {
+ "class" : "org.apache.solr.ltr.norm.StandardNormalizer",
+ "params" : { "avg":"42", "std":"6" }
+}
+</pre>
+ * <p>
+ * Example normalizations:
+ * <ul>
+ * <li>39 will be normalized to -0.5
+ * <li>42 will be normalized to 0
+ * <li>45 will be normalized to +0.5
+ * </ul>
+ */
+public class StandardNormalizer extends Normalizer {
+
+ private float avg = 0f;
+ private float std = 1f;
+
+ public float getAvg() {
+ return avg;
+ }
+
+ public void setAvg(float avg) {
+ this.avg = avg;
+ }
+
+ public float getStd() {
+ return std;
+ }
+
+ public void setStd(float std) {
+ this.std = std;
+ }
+
+ public void setAvg(String avg) {
+ this.avg = Float.parseFloat(avg);
+ }
+
+ public void setStd(String std) {
+ this.std = Float.parseFloat(std);
+ }
+
+ @Override
+ public float normalize(float value) {
+ return (value - avg) / std;
+ }
+
+ @Override
+ protected void validate() throws NormalizerException {
+ if (std <= 0f) {
+ throw
+ new NormalizerException("Standard Normalizer standard deviation must "
+ + "be positive | avg = " + avg + ",std = " + std);
+ }
+ }
+
+ @Override
+ public LinkedHashMap<String,Object> paramsToMap() {
+ final LinkedHashMap<String,Object> params = new LinkedHashMap<>(2, 1.0f);
+ params.put("avg", avg);
+ params.put("std", std);
+ return params;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(64); // default initialCapacity of 16 won't be enough
+ sb.append(getClass().getSimpleName()).append('(');
+ sb.append("avg=").append(avg);
+ sb.append(",std=").append(avg).append(')');
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/package-info.java
new file mode 100644
index 0000000..164b425
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/norm/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A normalizer normalizes the value of a feature. Once that the feature values
+ * will be computed, the normalizer will be applied and the resulting values
+ * will be received by the model.
+ */
+package org.apache.solr.ltr.norm;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/package-info.java
new file mode 100644
index 0000000..59aebe8
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/package-info.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <p>
+ * This package contains the main logic for performing the reranking using
+ * a Learning to Rank model.
+ * </p>
+ * <p>
+ * A model will be applied on each document through a {@link org.apache.solr.ltr.LTRScoringQuery}, a
+ * subclass of {@link org.apache.lucene.search.Query}. As a normal query,
+ * the learned model will produce a new score
+ * for each document reranked.
+ * </p>
+ * <p>
+ * A {@link org.apache.solr.ltr.LTRScoringQuery} is created by providing an instance of
+ * {@link org.apache.solr.ltr.model.LTRScoringModel}. An instance of
+ * {@link org.apache.solr.ltr.model.LTRScoringModel}
+ * defines how to combine the features in order to create a new
+ * score for a document. A new Learning to Rank model is plugged
+ * into the framework by extending {@link org.apache.solr.ltr.model.LTRScoringModel},
+ * (see for example {@link org.apache.solr.ltr.model.MultipleAdditiveTreesModel} and {@link org.apache.solr.ltr.model.LinearModel}).
+ * </p>
+ * <p>
+ * The {@link org.apache.solr.ltr.LTRScoringQuery} will take care of computing the values of
+ * all the features (see {@link org.apache.solr.ltr.feature.Feature}) and then will delegate the final score
+ * generation to the {@link org.apache.solr.ltr.model.LTRScoringModel}, by calling the method
+ * {@link org.apache.solr.ltr.model.LTRScoringModel#score(float[] modelFeatureValuesNormalized) score(float[] modelFeatureValuesNormalized)}.
+ * </p>
+ */
+package org.apache.solr.ltr;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/FeatureStore.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/FeatureStore.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/FeatureStore.java
new file mode 100644
index 0000000..ab2595f
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/FeatureStore.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.store;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.feature.FeatureException;
+
+public class FeatureStore {
+
+ /** the name of the default feature store **/
+ public static final String DEFAULT_FEATURE_STORE_NAME = "_DEFAULT_";
+
+ private final LinkedHashMap<String,Feature> store = new LinkedHashMap<>(); // LinkedHashMap because we need predictable iteration order
+ private final String name;
+
+ public FeatureStore(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Feature get(String name) {
+ return store.get(name);
+ }
+
+ public void add(Feature feature) {
+ final String name = feature.getName();
+ if (store.containsKey(name)) {
+ throw new FeatureException(name
+ + " already contained in the store, please use a different name");
+ }
+ feature.setIndex(store.size());
+ store.put(name, feature);
+ }
+
+ public List<Feature> getFeatures() {
+ final List<Feature> storeValues = new ArrayList<Feature>(store.values());
+ return Collections.unmodifiableList(storeValues);
+ }
+
+ @Override
+ public String toString() {
+ return "FeatureStore [features=" + store.keySet() + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/ModelStore.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/ModelStore.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/ModelStore.java
new file mode 100644
index 0000000..dbb065f
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/ModelStore.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.store;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.ltr.model.LTRScoringModel;
+import org.apache.solr.ltr.model.ModelException;
+
+/**
+ * Contains the model and features declared.
+ */
+public class ModelStore {
+
+ private final Map<String,LTRScoringModel> availableModels;
+
+ public ModelStore() {
+ availableModels = new HashMap<>();
+ }
+
+ public synchronized LTRScoringModel getModel(String name) {
+ return availableModels.get(name);
+ }
+
+ public void clear() {
+ availableModels.clear();
+ }
+
+ public List<LTRScoringModel> getModels() {
+ final List<LTRScoringModel> availableModelsValues =
+ new ArrayList<LTRScoringModel>(availableModels.values());
+ return Collections.unmodifiableList(availableModelsValues);
+ }
+
+ @Override
+ public String toString() {
+ return "ModelStore [availableModels=" + availableModels.keySet() + "]";
+ }
+
+ public LTRScoringModel delete(String modelName) {
+ return availableModels.remove(modelName);
+ }
+
+ public synchronized void addModel(LTRScoringModel modeldata)
+ throws ModelException {
+ final String name = modeldata.getName();
+
+ if (availableModels.containsKey(name)) {
+ throw new ModelException("model '" + name
+ + "' already exists. Please use a different name");
+ }
+
+ availableModels.put(modeldata.getName(), modeldata);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/package-info.java
new file mode 100644
index 0000000..1ed9bff
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains feature and model store related classes.
+ */
+package org.apache.solr.ltr.store;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
new file mode 100644
index 0000000..beb217c
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.store.rest;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.store.FeatureStore;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.rest.BaseSolrResource;
+import org.apache.solr.rest.ManagedResource;
+import org.apache.solr.rest.ManagedResourceObserver;
+import org.apache.solr.rest.ManagedResourceStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Managed resource for a storing a feature.
+ */
+public class ManagedFeatureStore extends ManagedResource implements ManagedResource.ChildResourceSupport {
+
+ public static void registerManagedFeatureStore(SolrResourceLoader solrResourceLoader,
+ ManagedResourceObserver managedResourceObserver) {
+ solrResourceLoader.getManagedResourceRegistry().registerManagedResource(
+ REST_END_POINT,
+ ManagedFeatureStore.class,
+ managedResourceObserver);
+ }
+
+ public static ManagedFeatureStore getManagedFeatureStore(SolrCore core) {
+ return (ManagedFeatureStore) core.getRestManager()
+ .getManagedResource(REST_END_POINT);
+ }
+
+ /** the feature store rest endpoint **/
+ public static final String REST_END_POINT = "/schema/feature-store";
+ // TODO: reduce from public to package visibility (once tests no longer need public access)
+
+ /** name of the attribute containing the feature class **/
+ static final String CLASS_KEY = "class";
+ /** name of the attribute containing the feature name **/
+ static final String NAME_KEY = "name";
+ /** name of the attribute containing the feature params **/
+ static final String PARAMS_KEY = "params";
+ /** name of the attribute containing the feature store used **/
+ static final String FEATURE_STORE_NAME_KEY = "store";
+
+ private final Map<String,FeatureStore> stores = new HashMap<>();
+
+ /**
+ * Managed feature store: the name of the attribute containing all the feature
+ * stores
+ **/
+ private static final String FEATURE_STORE_JSON_FIELD = "featureStores";
+
+ /**
+ * Managed feature store: the name of the attribute containing all the
+ * features of a feature store
+ **/
+ private static final String FEATURES_JSON_FIELD = "features";
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public ManagedFeatureStore(String resourceId, SolrResourceLoader loader,
+ ManagedResourceStorage.StorageIO storageIO) throws SolrException {
+ super(resourceId, loader, storageIO);
+
+ }
+
+ public synchronized FeatureStore getFeatureStore(String name) {
+ if (name == null) {
+ name = FeatureStore.DEFAULT_FEATURE_STORE_NAME;
+ }
+ if (!stores.containsKey(name)) {
+ stores.put(name, new FeatureStore(name));
+ }
+ return stores.get(name);
+ }
+
+ @Override
+ protected void onManagedDataLoadedFromStorage(NamedList<?> managedInitArgs,
+ Object managedData) throws SolrException {
+
+ stores.clear();
+ log.info("------ managed feature ~ loading ------");
+ if (managedData instanceof List) {
+ @SuppressWarnings("unchecked")
+ final List<Map<String,Object>> up = (List<Map<String,Object>>) managedData;
+ for (final Map<String,Object> u : up) {
+ final String featureStore = (String) u.get(FEATURE_STORE_NAME_KEY);
+ addFeature(u, featureStore);
+ }
+ }
+ }
+
+ public synchronized void addFeature(Map<String,Object> map, String featureStore) {
+ log.info("register feature based on {}", map);
+ final FeatureStore fstore = getFeatureStore(featureStore);
+ final Feature feature = fromFeatureMap(solrResourceLoader, map);
+ fstore.add(feature);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Object applyUpdatesToManagedData(Object updates) {
+ if (updates instanceof List) {
+ final List<Map<String,Object>> up = (List<Map<String,Object>>) updates;
+ for (final Map<String,Object> u : up) {
+ final String featureStore = (String) u.get(FEATURE_STORE_NAME_KEY);
+ addFeature(u, featureStore);
+ }
+ }
+
+ if (updates instanceof Map) {
+ // a unique feature
+ Map<String,Object> updatesMap = (Map<String,Object>) updates;
+ final String featureStore = (String) updatesMap.get(FEATURE_STORE_NAME_KEY);
+ addFeature(updatesMap, featureStore);
+ }
+
+ final List<Object> features = new ArrayList<>();
+ for (final FeatureStore fs : stores.values()) {
+ features.addAll(featuresAsManagedResources(fs));
+ }
+ return features;
+ }
+
+ @Override
+ public synchronized void doDeleteChild(BaseSolrResource endpoint, String childId) {
+ if (childId.equals("*")) {
+ stores.clear();
+ }
+ if (stores.containsKey(childId)) {
+ stores.remove(childId);
+ }
+ storeManagedData(applyUpdatesToManagedData(null));
+ }
+
+ /**
+ * Called to retrieve a named part (the given childId) of the resource at the
+ * given endpoint. Note: since we have a unique child feature store we ignore
+ * the childId.
+ */
+ @Override
+ public void doGet(BaseSolrResource endpoint, String childId) {
+ final SolrQueryResponse response = endpoint.getSolrResponse();
+
+ // If no feature store specified, show all the feature stores available
+ if (childId == null) {
+ response.add(FEATURE_STORE_JSON_FIELD, stores.keySet());
+ } else {
+ final FeatureStore store = getFeatureStore(childId);
+ if (store == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "missing feature store [" + childId + "]");
+ }
+ response.add(FEATURES_JSON_FIELD,
+ featuresAsManagedResources(store));
+ }
+ }
+
+ private static List<Object> featuresAsManagedResources(FeatureStore store) {
+ final List<Feature> storedFeatures = store.getFeatures();
+ final List<Object> features = new ArrayList<Object>(storedFeatures.size());
+ for (final Feature f : storedFeatures) {
+ final LinkedHashMap<String,Object> m = toFeatureMap(f);
+ m.put(FEATURE_STORE_NAME_KEY, store.getName());
+ features.add(m);
+ }
+ return features;
+ }
+
+ private static LinkedHashMap<String,Object> toFeatureMap(Feature feat) {
+ final LinkedHashMap<String,Object> o = new LinkedHashMap<>(4, 1.0f); // 1 extra for caller to add store
+ o.put(NAME_KEY, feat.getName());
+ o.put(CLASS_KEY, feat.getClass().getCanonicalName());
+ o.put(PARAMS_KEY, feat.paramsToMap());
+ return o;
+ }
+
+ private static Feature fromFeatureMap(SolrResourceLoader solrResourceLoader,
+ Map<String,Object> featureMap) {
+ final String className = (String) featureMap.get(CLASS_KEY);
+
+ final String name = (String) featureMap.get(NAME_KEY);
+
+ @SuppressWarnings("unchecked")
+ final Map<String,Object> params = (Map<String,Object>) featureMap.get(PARAMS_KEY);
+
+ return Feature.getInstance(solrResourceLoader, className, name, params);
+ }
+}