You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2016/12/08 18:44:04 UTC
[05/13] lucene-solr:branch_6x: SOLR-8542: Adds Solr Learning to Rank
(LTR) plugin for reranking results with machine learning models. (Michael
Nilsson, Diego Ceccarelli, Joshua Pantony, Jon Dorando, Naveen Santhapuri,
Alessandro Benedetti, David Grohmann
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
new file mode 100644
index 0000000..97aaa40
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.ltr.store.rest;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.model.LTRScoringModel;
+import org.apache.solr.ltr.model.ModelException;
+import org.apache.solr.ltr.norm.IdentityNormalizer;
+import org.apache.solr.ltr.norm.Normalizer;
+import org.apache.solr.ltr.store.FeatureStore;
+import org.apache.solr.ltr.store.ModelStore;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.rest.BaseSolrResource;
+import org.apache.solr.rest.ManagedResource;
+import org.apache.solr.rest.ManagedResourceObserver;
+import org.apache.solr.rest.ManagedResourceStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Menaged resource for storing a model
+ */
+public class ManagedModelStore extends ManagedResource implements ManagedResource.ChildResourceSupport {
+
+ public static void registerManagedModelStore(SolrResourceLoader solrResourceLoader,
+ ManagedResourceObserver managedResourceObserver) {
+ solrResourceLoader.getManagedResourceRegistry().registerManagedResource(
+ REST_END_POINT,
+ ManagedModelStore.class,
+ managedResourceObserver);
+ }
+
+ public static ManagedModelStore getManagedModelStore(SolrCore core) {
+ return (ManagedModelStore) core.getRestManager()
+ .getManagedResource(REST_END_POINT);
+ }
+
+ /** the model store rest endpoint **/
+ public static final String REST_END_POINT = "/schema/model-store";
+ // TODO: reduce from public to package visibility (once tests no longer need public access)
+
+ /**
+ * Managed model store: the name of the attribute containing all the models of
+ * a model store
+ **/
+ private static final String MODELS_JSON_FIELD = "models";
+
+ /** name of the attribute containing a class **/
+ static final String CLASS_KEY = "class";
+ /** name of the attribute containing the features **/
+ static final String FEATURES_KEY = "features";
+ /** name of the attribute containing a name **/
+ static final String NAME_KEY = "name";
+ /** name of the attribute containing a normalizer **/
+ static final String NORM_KEY = "norm";
+ /** name of the attribute containing parameters **/
+ static final String PARAMS_KEY = "params";
+ /** name of the attribute containing a store **/
+ static final String STORE_KEY = "store";
+
+ private final ModelStore store;
+ private ManagedFeatureStore managedFeatureStore;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public ManagedModelStore(String resourceId, SolrResourceLoader loader,
+ ManagedResourceStorage.StorageIO storageIO) throws SolrException {
+ super(resourceId, loader, storageIO);
+ store = new ModelStore();
+ }
+
+ public void setManagedFeatureStore(ManagedFeatureStore managedFeatureStore) {
+ log.info("INIT model store");
+ this.managedFeatureStore = managedFeatureStore;
+ }
+
+ public ManagedFeatureStore getManagedFeatureStore() {
+ return managedFeatureStore;
+ }
+
+ private Object managedData;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void onManagedDataLoadedFromStorage(NamedList<?> managedInitArgs,
+ Object managedData) throws SolrException {
+ store.clear();
+ // the managed models on the disk or on zookeeper will be loaded in a lazy
+ // way, since we need to set the managed features first (unfortunately
+ // managed resources do not
+ // decouple the creation of a managed resource with the reading of the data
+ // from the storage)
+ this.managedData = managedData;
+
+ }
+
+ public void loadStoredModels() {
+ log.info("------ managed models ~ loading ------");
+
+ if ((managedData != null) && (managedData instanceof List)) {
+ final List<Map<String,Object>> up = (List<Map<String,Object>>) managedData;
+ for (final Map<String,Object> u : up) {
+ try {
+ final LTRScoringModel algo = fromLTRScoringModelMap(solrResourceLoader, u, managedFeatureStore);
+ addModel(algo);
+ } catch (final ModelException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+ }
+ }
+ }
+
+ public synchronized void addModel(LTRScoringModel ltrScoringModel) throws ModelException {
+ try {
+ log.info("adding model {}", ltrScoringModel.getName());
+ store.addModel(ltrScoringModel);
+ } catch (final ModelException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected Object applyUpdatesToManagedData(Object updates) {
+ if (updates instanceof List) {
+ final List<Map<String,Object>> up = (List<Map<String,Object>>) updates;
+ for (final Map<String,Object> u : up) {
+ try {
+ final LTRScoringModel algo = fromLTRScoringModelMap(solrResourceLoader, u, managedFeatureStore);
+ addModel(algo);
+ } catch (final ModelException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+ }
+ }
+
+ if (updates instanceof Map) {
+ final Map<String,Object> map = (Map<String,Object>) updates;
+ try {
+ final LTRScoringModel algo = fromLTRScoringModelMap(solrResourceLoader, map, managedFeatureStore);
+ addModel(algo);
+ } catch (final ModelException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+ }
+
+ return modelsAsManagedResources(store.getModels());
+ }
+
+ @Override
+ public synchronized void doDeleteChild(BaseSolrResource endpoint, String childId) {
+ if (childId.equals("*")) {
+ store.clear();
+ } else {
+ store.delete(childId);
+ }
+ storeManagedData(applyUpdatesToManagedData(null));
+ }
+
+ /**
+ * Called to retrieve a named part (the given childId) of the resource at the
+ * given endpoint. Note: since we have a unique child managed store we ignore
+ * the childId.
+ */
+ @Override
+ public void doGet(BaseSolrResource endpoint, String childId) {
+
+ final SolrQueryResponse response = endpoint.getSolrResponse();
+ response.add(MODELS_JSON_FIELD,
+ modelsAsManagedResources(store.getModels()));
+ }
+
+ public LTRScoringModel getModel(String modelName) {
+ // this function replicates getModelStore().getModel(modelName), but
+ // it simplifies the testing (we can avoid to mock also a ModelStore).
+ return store.getModel(modelName);
+ }
+
+ @Override
+ public String toString() {
+ return "ManagedModelStore [store=" + store + ", featureStores="
+ + managedFeatureStore + "]";
+ }
+
+ /**
+ * Returns the available models as a list of Maps objects. After an update the
+ * managed resources needs to return the resources in this format in order to
+ * store in json somewhere (zookeeper, disk...)
+ *
+ *
+ * @return the available models as a list of Maps objects
+ */
+ private static List<Object> modelsAsManagedResources(List<LTRScoringModel> models) {
+ final List<Object> list = new ArrayList<>(models.size());
+ for (final LTRScoringModel model : models) {
+ list.add(toLTRScoringModelMap(model));
+ }
+ return list;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static LTRScoringModel fromLTRScoringModelMap(SolrResourceLoader solrResourceLoader,
+ Map<String,Object> modelMap, ManagedFeatureStore managedFeatureStore) {
+
+ final FeatureStore featureStore =
+ managedFeatureStore.getFeatureStore((String) modelMap.get(STORE_KEY));
+
+ final List<Feature> features = new ArrayList<>();
+ final List<Normalizer> norms = new ArrayList<>();
+
+ final List<Object> featureList = (List<Object>) modelMap.get(FEATURES_KEY);
+ if (featureList != null) {
+ for (final Object feature : featureList) {
+ final Map<String,Object> featureMap = (Map<String,Object>) feature;
+ features.add(lookupFeatureFromFeatureMap(featureMap, featureStore));
+ norms.add(createNormalizerFromFeatureMap(solrResourceLoader, featureMap));
+ }
+ }
+
+ return LTRScoringModel.getInstance(solrResourceLoader,
+ (String) modelMap.get(CLASS_KEY), // modelClassName
+ (String) modelMap.get(NAME_KEY), // modelName
+ features,
+ norms,
+ featureStore.getName(),
+ featureStore.getFeatures(),
+ (Map<String,Object>) modelMap.get(PARAMS_KEY));
+ }
+
+ private static LinkedHashMap<String,Object> toLTRScoringModelMap(LTRScoringModel model) {
+ final LinkedHashMap<String,Object> modelMap = new LinkedHashMap<>(5, 1.0f);
+
+ modelMap.put(NAME_KEY, model.getName());
+ modelMap.put(CLASS_KEY, model.getClass().getCanonicalName());
+ modelMap.put(STORE_KEY, model.getFeatureStoreName());
+
+ final List<Map<String,Object>> features = new ArrayList<>();
+ final List<Feature> featuresList = model.getFeatures();
+ final List<Normalizer> normsList = model.getNorms();
+ for (int ii=0; ii<featuresList.size(); ++ii) {
+ features.add(toFeatureMap(featuresList.get(ii), normsList.get(ii)));
+ }
+ modelMap.put(FEATURES_KEY, features);
+
+ modelMap.put(PARAMS_KEY, model.getParams());
+
+ return modelMap;
+ }
+
+ private static Feature lookupFeatureFromFeatureMap(Map<String,Object> featureMap,
+ FeatureStore featureStore) {
+ final String featureName = (String)featureMap.get(NAME_KEY);
+ return (featureName == null ? null
+ : featureStore.get(featureName));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Normalizer createNormalizerFromFeatureMap(SolrResourceLoader solrResourceLoader,
+ Map<String,Object> featureMap) {
+ final Map<String,Object> normMap = (Map<String,Object>)featureMap.get(NORM_KEY);
+ return (normMap == null ? IdentityNormalizer.INSTANCE
+ : fromNormalizerMap(solrResourceLoader, normMap));
+ }
+
+ private static LinkedHashMap<String,Object> toFeatureMap(Feature feature, Normalizer norm) {
+ final LinkedHashMap<String,Object> map = new LinkedHashMap<String,Object>(2, 1.0f);
+ map.put(NAME_KEY, feature.getName());
+ map.put(NORM_KEY, toNormalizerMap(norm));
+ return map;
+ }
+
+ private static Normalizer fromNormalizerMap(SolrResourceLoader solrResourceLoader,
+ Map<String,Object> normMap) {
+ final String className = (String) normMap.get(CLASS_KEY);
+
+ @SuppressWarnings("unchecked")
+ final Map<String,Object> params = (Map<String,Object>) normMap.get(PARAMS_KEY);
+
+ return Normalizer.getInstance(solrResourceLoader, className, params);
+ }
+
+ private static LinkedHashMap<String,Object> toNormalizerMap(Normalizer norm) {
+ final LinkedHashMap<String,Object> normalizer = new LinkedHashMap<>(2, 1.0f);
+
+ normalizer.put(CLASS_KEY, norm.getClass().getCanonicalName());
+
+ final LinkedHashMap<String,Object> params = norm.paramsToMap();
+ if (params != null) {
+ normalizer.put(PARAMS_KEY, params);
+ }
+
+ return normalizer;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/package-info.java
new file mode 100644
index 0000000..fbf7029
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/store/rest/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the {@link org.apache.solr.rest.ManagedResource} that encapsulate
+ * the feature and the model stores.
+ */
+package org.apache.solr.ltr.store.rest;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/response/transform/LTRFeatureLoggerTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/response/transform/LTRFeatureLoggerTransformerFactory.java b/solr/contrib/ltr/src/java/org/apache/solr/response/transform/LTRFeatureLoggerTransformerFactory.java
new file mode 100644
index 0000000..d144292
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/response/transform/LTRFeatureLoggerTransformerFactory.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.response.transform;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Explanation;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.ltr.FeatureLogger;
+import org.apache.solr.ltr.LTRRescorer;
+import org.apache.solr.ltr.LTRScoringQuery;
+import org.apache.solr.ltr.LTRThreadModule;
+import org.apache.solr.ltr.SolrQueryRequestContextUtils;
+import org.apache.solr.ltr.feature.Feature;
+import org.apache.solr.ltr.model.LTRScoringModel;
+import org.apache.solr.ltr.norm.Normalizer;
+import org.apache.solr.ltr.store.FeatureStore;
+import org.apache.solr.ltr.store.rest.ManagedFeatureStore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.ResultContext;
+import org.apache.solr.search.LTRQParserPlugin;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.SolrPluginUtils;
+
+/**
+ * This transformer will take care to generate and append in the response the
+ * features declared in the feature store of the current model. The class is
+ * useful if you are not interested in the reranking (e.g., bootstrapping a
+ * machine learning framework).
+ */
+public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
+
+ // used inside fl to specify the output format (csv/json) of the extracted features
+ private static final String FV_RESPONSE_WRITER = "fvwt";
+
+ // used inside fl to specify the format (dense|sparse) of the extracted features
+ private static final String FV_FORMAT = "format";
+
+ // used inside fl to specify the feature store to use for the feature extraction
+ private static final String FV_STORE = "store";
+
+ private static String DEFAULT_LOGGING_MODEL_NAME = "logging-model";
+
+ private String loggingModelName = DEFAULT_LOGGING_MODEL_NAME;
+ private String defaultFvStore;
+ private String defaultFvwt;
+ private String defaultFvFormat;
+
+ private LTRThreadModule threadManager = null;
+
+ public void setLoggingModelName(String loggingModelName) {
+ this.loggingModelName = loggingModelName;
+ }
+
+ public void setStore(String defaultFvStore) {
+ this.defaultFvStore = defaultFvStore;
+ }
+
+ public void setFvwt(String defaultFvwt) {
+ this.defaultFvwt = defaultFvwt;
+ }
+
+ public void setFormat(String defaultFvFormat) {
+ this.defaultFvFormat = defaultFvFormat;
+ }
+
+ @Override
+ public void init(@SuppressWarnings("rawtypes") NamedList args) {
+ super.init(args);
+ threadManager = LTRThreadModule.getInstance(args);
+ SolrPluginUtils.invokeSetters(this, args);
+ }
+
+ @Override
+ public DocTransformer create(String name, SolrParams params,
+ SolrQueryRequest req) {
+
+ // Hint to enable feature vector cache since we are requesting features
+ SolrQueryRequestContextUtils.setIsExtractingFeatures(req);
+
+ // Communicate which feature store we are requesting features for
+ SolrQueryRequestContextUtils.setFvStoreName(req, params.get(FV_STORE, defaultFvStore));
+
+ // Create and supply the feature logger to be used
+ SolrQueryRequestContextUtils.setFeatureLogger(req,
+ FeatureLogger.createFeatureLogger(
+ params.get(FV_RESPONSE_WRITER, defaultFvwt),
+ params.get(FV_FORMAT, defaultFvFormat)));
+
+ return new FeatureTransformer(name, params, req);
+ }
+
+ class FeatureTransformer extends DocTransformer {
+
+ final private String name;
+ final private SolrParams params;
+ final private SolrQueryRequest req;
+
+ private List<LeafReaderContext> leafContexts;
+ private SolrIndexSearcher searcher;
+ private LTRScoringQuery scoringQuery;
+ private LTRScoringQuery.ModelWeight modelWeight;
+ private FeatureLogger<?> featureLogger;
+ private boolean docsWereNotReranked;
+
+ /**
+ * @param name
+ * Name of the field to be added in a document representing the
+ * feature vectors
+ */
+ public FeatureTransformer(String name, SolrParams params,
+ SolrQueryRequest req) {
+ this.name = name;
+ this.params = params;
+ this.req = req;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setContext(ResultContext context) {
+ super.setContext(context);
+ if (context == null) {
+ return;
+ }
+ if (context.getRequest() == null) {
+ return;
+ }
+
+ searcher = context.getSearcher();
+ if (searcher == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "searcher is null");
+ }
+ leafContexts = searcher.getTopReaderContext().leaves();
+
+ // Setup LTRScoringQuery
+ scoringQuery = SolrQueryRequestContextUtils.getScoringQuery(req);
+ docsWereNotReranked = (scoringQuery == null);
+ String featureStoreName = SolrQueryRequestContextUtils.getFvStoreName(req);
+ if (docsWereNotReranked || (featureStoreName != null && (!featureStoreName.equals(scoringQuery.getScoringModel().getFeatureStoreName())))) {
+ // if store is set in the transformer we should overwrite the logger
+
+ final ManagedFeatureStore fr = ManagedFeatureStore.getManagedFeatureStore(req.getCore());
+
+ final FeatureStore store = fr.getFeatureStore(featureStoreName);
+ featureStoreName = store.getName(); // if featureStoreName was null before this gets actual name
+
+ try {
+ final LoggingModel lm = new LoggingModel(loggingModelName,
+ featureStoreName, store.getFeatures());
+
+ scoringQuery = new LTRScoringQuery(lm,
+ LTRQParserPlugin.extractEFIParams(params),
+ true,
+ threadManager); // request feature weights to be created for all features
+
+ // Local transformer efi if provided
+ scoringQuery.setOriginalQuery(context.getQuery());
+
+ }catch (final Exception e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "retrieving the feature store "+featureStoreName, e);
+ }
+ }
+
+ if (scoringQuery.getFeatureLogger() == null){
+ scoringQuery.setFeatureLogger( SolrQueryRequestContextUtils.getFeatureLogger(req) );
+ }
+ scoringQuery.setRequest(req);
+
+ featureLogger = scoringQuery.getFeatureLogger();
+
+ try {
+ modelWeight = scoringQuery.createWeight(searcher, true, 1f);
+ } catch (final IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e.getMessage(), e);
+ }
+ if (modelWeight == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "error logging the features, model weight is null");
+ }
+ }
+
+ @Override
+ public void transform(SolrDocument doc, int docid, float score)
+ throws IOException {
+ Object fv = featureLogger.getFeatureVector(docid, scoringQuery, searcher);
+ if (fv == null) { // FV for this document was not in the cache
+ fv = featureLogger.makeFeatureVector(
+ LTRRescorer.extractFeaturesInfo(
+ modelWeight,
+ docid,
+ (docsWereNotReranked ? new Float(score) : null),
+ leafContexts));
+ }
+
+ doc.addField(name, fv);
+ }
+
+ }
+
+ private static class LoggingModel extends LTRScoringModel {
+
+ public LoggingModel(String name, String featureStoreName, List<Feature> allFeatures){
+ this(name, Collections.emptyList(), Collections.emptyList(),
+ featureStoreName, allFeatures, Collections.emptyMap());
+ }
+
+ protected LoggingModel(String name, List<Feature> features,
+ List<Normalizer> norms, String featureStoreName,
+ List<Feature> allFeatures, Map<String,Object> params) {
+ super(name, features, norms, featureStoreName, allFeatures, params);
+ }
+
+ @Override
+ public float score(float[] modelFeatureValuesNormalized) {
+ return 0;
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc, float finalScore, List<Explanation> featureExplanations) {
+ return Explanation.match(finalScore, toString()
+ + " logging model, used only for logging the features");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/response/transform/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/response/transform/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/response/transform/package-info.java
new file mode 100644
index 0000000..bab3ebf
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/response/transform/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * APIs and implementations of {@link org.apache.solr.response.transform.DocTransformer} for modifying documents in Solr request responses
+ */
+package org.apache.solr.response.transform;
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/search/LTRQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/search/LTRQParserPlugin.java b/solr/contrib/ltr/src/java/org/apache/solr/search/LTRQParserPlugin.java
new file mode 100644
index 0000000..40cbaa9
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/search/LTRQParserPlugin.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.ltr.LTRRescorer;
+import org.apache.solr.ltr.LTRScoringQuery;
+import org.apache.solr.ltr.LTRThreadModule;
+import org.apache.solr.ltr.SolrQueryRequestContextUtils;
+import org.apache.solr.ltr.model.LTRScoringModel;
+import org.apache.solr.ltr.store.rest.ManagedFeatureStore;
+import org.apache.solr.ltr.store.rest.ManagedModelStore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.rest.ManagedResource;
+import org.apache.solr.rest.ManagedResourceObserver;
+import org.apache.solr.util.SolrPluginUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Plug into solr a rerank model.
+ *
+ * Learning to Rank Query Parser Syntax: rq={!ltr model=6029760550880411648 reRankDocs=300
+ * efi.myCompanyQueryIntent=0.98}
+ *
+ */
+public class LTRQParserPlugin extends QParserPlugin implements ResourceLoaderAware, ManagedResourceObserver {
+ public static final String NAME = "ltr";
+ private static Query defaultQuery = new MatchAllDocsQuery();
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ // params for setting custom external info that features can use, like query
+ // intent
+ static final String EXTERNAL_FEATURE_INFO = "efi.";
+
+ private ManagedFeatureStore fr = null;
+ private ManagedModelStore mr = null;
+
+ private LTRThreadModule threadManager = null;
+
+ /** query parser plugin: the name of the attribute for setting the model **/
+ public static final String MODEL = "model";
+
+ /** query parser plugin: default number of documents to rerank **/
+ public static final int DEFAULT_RERANK_DOCS = 200;
+
+ /**
+ * query parser plugin:the param that will select how the number of document
+ * to rerank
+ **/
+ public static final String RERANK_DOCS = "reRankDocs";
+
+ @Override
+ public void init(@SuppressWarnings("rawtypes") NamedList args) {
+ super.init(args);
+ threadManager = LTRThreadModule.getInstance(args);
+ SolrPluginUtils.invokeSetters(this, args);
+ }
+
+ @Override
+ public QParser createParser(String qstr, SolrParams localParams,
+ SolrParams params, SolrQueryRequest req) {
+ return new LTRQParser(qstr, localParams, params, req);
+ }
+
+ /**
+ * Given a set of local SolrParams, extract all of the efi.key=value params into a map
+ * @param localParams Local request parameters that might conatin efi params
+ * @return Map of efi params, where the key is the name of the efi param, and the
+ * value is the value of the efi param
+ */
+ public static Map<String,String[]> extractEFIParams(SolrParams localParams) {
+ final Map<String,String[]> externalFeatureInfo = new HashMap<>();
+ for (final Iterator<String> it = localParams.getParameterNamesIterator(); it
+ .hasNext();) {
+ final String name = it.next();
+ if (name.startsWith(EXTERNAL_FEATURE_INFO)) {
+ externalFeatureInfo.put(
+ name.substring(EXTERNAL_FEATURE_INFO.length()),
+ new String[] {localParams.get(name)});
+ }
+ }
+ return externalFeatureInfo;
+ }
+
+
+ @Override
+ public void inform(ResourceLoader loader) throws IOException {
+ final SolrResourceLoader solrResourceLoader = (SolrResourceLoader) loader;
+ ManagedFeatureStore.registerManagedFeatureStore(solrResourceLoader, this);
+ ManagedModelStore.registerManagedModelStore(solrResourceLoader, this);
+ }
+
+ @Override
+ public void onManagedResourceInitialized(NamedList<?> args, ManagedResource res) throws SolrException {
+ if (res instanceof ManagedFeatureStore) {
+ fr = (ManagedFeatureStore)res;
+ }
+ if (res instanceof ManagedModelStore){
+ mr = (ManagedModelStore)res;
+ }
+ if (mr != null && fr != null){
+ mr.setManagedFeatureStore(fr);
+ // now we can safely load the models
+ mr.loadStoredModels();
+
+ }
+ }
+
+ public class LTRQParser extends QParser {
+
+ public LTRQParser(String qstr, SolrParams localParams, SolrParams params,
+ SolrQueryRequest req) {
+ super(qstr, localParams, params, req);
+ }
+
+ @Override
+ public Query parse() throws SyntaxError {
+ // ReRanking Model
+ final String modelName = localParams.get(LTRQParserPlugin.MODEL);
+ if ((modelName == null) || modelName.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Must provide model in the request");
+ }
+
+ final LTRScoringModel ltrScoringModel = mr.getModel(modelName);
+ if (ltrScoringModel == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "cannot find " + LTRQParserPlugin.MODEL + " " + modelName);
+ }
+
+ final String modelFeatureStoreName = ltrScoringModel.getFeatureStoreName();
+ final boolean extractFeatures = SolrQueryRequestContextUtils.isExtractingFeatures(req);
+ final String fvStoreName = SolrQueryRequestContextUtils.getFvStoreName(req);
+ // Check if features are requested and if the model feature store and feature-transform feature store are the same
+ final boolean featuresRequestedFromSameStore = (modelFeatureStoreName.equals(fvStoreName) || fvStoreName == null) ? extractFeatures:false;
+
+ final LTRScoringQuery scoringQuery = new LTRScoringQuery(ltrScoringModel,
+ extractEFIParams(localParams),
+ featuresRequestedFromSameStore, threadManager);
+
+ // Enable the feature vector caching if we are extracting features, and the features
+ // we requested are the same ones we are reranking with
+ if (featuresRequestedFromSameStore) {
+ scoringQuery.setFeatureLogger( SolrQueryRequestContextUtils.getFeatureLogger(req) );
+ }
+ SolrQueryRequestContextUtils.setScoringQuery(req, scoringQuery);
+
+ int reRankDocs = localParams.getInt(RERANK_DOCS, DEFAULT_RERANK_DOCS);
+ reRankDocs = Math.max(1, reRankDocs);
+
+ // External features
+ scoringQuery.setRequest(req);
+
+ return new LTRQuery(scoringQuery, reRankDocs);
+ }
+ }
+
+ /**
+ * A learning to rank Query, will incapsulate a learning to rank model, and delegate to it the rescoring
+ * of the documents.
+ **/
+ public class LTRQuery extends AbstractReRankQuery {
+ private final LTRScoringQuery scoringQuery;
+
+ public LTRQuery(LTRScoringQuery scoringQuery, int reRankDocs) {
+ super(defaultQuery, reRankDocs, new LTRRescorer(scoringQuery));
+ this.scoringQuery = scoringQuery;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * classHash() + (mainQuery.hashCode() + scoringQuery.hashCode() + reRankDocs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return sameClassAs(o) && equalsTo(getClass().cast(o));
+ }
+
+ private boolean equalsTo(LTRQuery other) {
+ return (mainQuery.equals(other.mainQuery)
+ && scoringQuery.equals(other.scoringQuery) && (reRankDocs == other.reRankDocs));
+ }
+
+ @Override
+ public RankQuery wrap(Query _mainQuery) {
+ super.wrap(_mainQuery);
+ scoringQuery.setOriginalQuery(_mainQuery);
+ return this;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "{!ltr mainQuery='" + mainQuery.toString() + "' scoringQuery='"
+ + scoringQuery.toString() + "' reRankDocs=" + reRankDocs + "}";
+ }
+
+ @Override
+ protected Query rewrite(Query rewrittenMainQuery) throws IOException {
+ return new LTRQuery(scoringQuery, reRankDocs).wrap(rewrittenMainQuery);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/org/apache/solr/search/package-info.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/search/package-info.java b/solr/contrib/ltr/src/java/org/apache/solr/search/package-info.java
new file mode 100644
index 0000000..2286a93
--- /dev/null
+++ b/solr/contrib/ltr/src/java/org/apache/solr/search/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * APIs and classes for {@linkplain org.apache.solr.search.QParserPlugin parsing} and {@linkplain org.apache.solr.search.SolrIndexSearcher processing} search requests
+ */
+package org.apache.solr.search;
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/java/overview.html
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/overview.html b/solr/contrib/ltr/src/java/overview.html
new file mode 100644
index 0000000..ccae361
--- /dev/null
+++ b/solr/contrib/ltr/src/java/overview.html
@@ -0,0 +1,91 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<body>
+Apache Solr Search Server: Learning to Rank Contrib
+
+<p>
+This module contains a logic <strong>to plug machine learned ranking modules into Solr</strong>.
+</p>
+<p>
+In information retrieval systems, Learning to Rank is used to re-rank the top X
+retrieved documents using trained machine learning models. The hope is
+that sophisticated models can make more nuanced ranking decisions than standard ranking
+functions like TF-IDF or BM25.
+</p>
+<p>
+This module allows to plug a reranking component directly into Solr, enabling users
+to easily build their own learning to rank systems and access the rich
+matching features readily available in Solr. It also provides tools to perform
+feature engineering and feature extraction.
+</p>
+<h2> Code structure </h2>
+<p>
+A Learning to Rank model is plugged into the ranking through the {@link org.apache.solr.search.LTRQParserPlugin},
+a {@link org.apache.solr.search.QParserPlugin}. The plugin will
+read from the request the model (instance of {@link org.apache.solr.ltr.model.LTRScoringModel})
+used to perform the request plus other
+parameters. The plugin will generate a {@link org.apache.solr.search.LTRQParserPlugin.LTRQuery LTRQuery}:
+a particular {@link org.apache.solr.search.RankQuery}
+that will encapsulate the given model and use it to
+rescore and rerank the document (by using an {@link org.apache.solr.ltr.LTRRescorer}).
+</p>
+<p>
+A model will be applied on each document through a {@link org.apache.solr.ltr.LTRScoringQuery}, a
+subclass of {@link org.apache.lucene.search.Query}. As a normal query,
+the learned model will produce a new score
+for each document reranked.
+</p>
+<p>
+A {@link org.apache.solr.ltr.LTRScoringQuery} is created by providing an instance of
+{@link org.apache.solr.ltr.model.LTRScoringModel}. An instance of
+{@link org.apache.solr.ltr.model.LTRScoringModel}
+defines how to combine the features in order to create a new
+score for a document. A new learning to rank model is plugged
+into the framework by extending {@link org.apache.solr.ltr.model.LTRScoringModel},
+(see for example {@link org.apache.solr.ltr.model.MultipleAdditiveTreesModel} and {@link org.apache.solr.ltr.model.LinearModel}).
+</p>
+<p>
+The {@link org.apache.solr.ltr.LTRScoringQuery} will take care of computing the values of
+all the features (see {@link org.apache.solr.ltr.feature.Feature}) and then will delegate the final score
+generation to the {@link org.apache.solr.ltr.model.LTRScoringModel}, by calling the method
+{@link org.apache.solr.ltr.model.LTRScoringModel#score(float[] modelFeatureValuesNormalized)}.
+</p>
+<p>
+A {@link org.apache.solr.ltr.feature.Feature} will produce a particular value for each document, so
+it is modeled as a {@link org.apache.lucene.search.Query}. The package
+{@link org.apache.solr.ltr.feature} contains several examples
+of features. One benefit of extending the Query object is that we can reuse
+Query as a feature, see for example {@link org.apache.solr.ltr.feature.SolrFeature}.
+Features for a document can also be returned in the response by
+using the FeatureTransformer (a {@link org.apache.solr.response.transform.DocTransformer DocTransformer})
+provided by {@link org.apache.solr.response.transform.LTRFeatureLoggerTransformerFactory}.
+</p>
+<p>
+{@link org.apache.solr.ltr.store} contains all the logic to store all the features and the models.
+Models are registered into a unique {@link org.apache.solr.ltr.store.ModelStore ModelStore},
+and each model specifies a particular {@link org.apache.solr.ltr.store.FeatureStore FeatureStore} that
+will contain a particular subset of features.
+<p>
+</p>
+Features and models can be managed through a REST API, provided by the
+{@link org.apache.solr.rest.ManagedResource Managed Resources}
+{@link org.apache.solr.ltr.store.rest.ManagedFeatureStore ManagedFeatureStore}
+and {@link org.apache.solr.ltr.store.rest.ManagedModelStore ManagedModelStore}.
+</p>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/comp_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/comp_features.json b/solr/contrib/ltr/src/test-files/featureExamples/comp_features.json
new file mode 100644
index 0000000..8d75739
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/comp_features.json
@@ -0,0 +1,37 @@
+[
+{ "name":"origScore",
+ "class":"org.apache.solr.ltr.feature.OriginalScoreFeature",
+ "params":{},
+ "store": "feature-store-6"
+},
+{
+ "name": "descriptionTermFreq",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": { "q" : "{!func}termfreq(description,${user_text})" },
+ "store": "feature-store-6"
+},
+{
+ "name": "popularity",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": { "q" : "{!func}normHits"},
+ "store": "feature-store-6"
+},
+{
+ "name": "isPopular",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": {"fq" : ["{!field f=popularity}201"] },
+ "store": "feature-store-6"
+},
+{
+ "name": "queryPartialMatch2",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": {"q": "{!dismax qf=description mm=2}${user_text}" },
+ "store": "feature-store-6"
+},
+{
+ "name": "queryPartialMatch2.1",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": {"q": "{!dismax qf=description mm=2}${user_text}" },
+ "store": "feature-store-6"
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
new file mode 100644
index 0000000..6c0cfa6
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json
@@ -0,0 +1,51 @@
+[ {
+ "name" : "matchedTitle",
+ "class" : "org.apache.solr.ltr.feature.SolrFeature",
+ "params" : {
+ "q" : "{!terms f=title}${user_query}"
+ }
+}, {
+ "name" : "confidence",
+ "class" : "org.apache.solr.ltr.feature.ValueFeature",
+ "store": "fstore2",
+ "params" : {
+ "value" : "${myconf}"
+ }
+}, {
+ "name":"originalScore",
+ "class":"org.apache.solr.ltr.feature.OriginalScoreFeature",
+ "store": "fstore2",
+ "params":{}
+}, {
+ "name" : "occurrences",
+ "class" : "org.apache.solr.ltr.feature.ValueFeature",
+ "store": "fstore3",
+ "params" : {
+ "value" : "${myOcc}",
+ "required" : false
+ }
+}, {
+ "name":"originalScore",
+ "class":"org.apache.solr.ltr.feature.OriginalScoreFeature",
+ "store": "fstore3",
+ "params":{}
+}, {
+ "name" : "popularity",
+ "class" : "org.apache.solr.ltr.feature.ValueFeature",
+ "store": "fstore4",
+ "params" : {
+ "value" : "${myPop}",
+ "required" : true
+ }
+}, {
+ "name":"originalScore",
+ "class":"org.apache.solr.ltr.feature.OriginalScoreFeature",
+ "store": "fstore4",
+ "params":{}
+}, {
+ "name" : "titlePhraseMatch",
+ "class" : "org.apache.solr.ltr.feature.SolrFeature",
+ "params" : {
+ "q" : "{!field f=title}${user_query}"
+ }
+} ]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/external_features_for_sparse_processing.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/external_features_for_sparse_processing.json b/solr/contrib/ltr/src/test-files/featureExamples/external_features_for_sparse_processing.json
new file mode 100644
index 0000000..52bab27
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/external_features_for_sparse_processing.json
@@ -0,0 +1,18 @@
+[{
+ "name" : "user_device_smartphone",
+ "class":"org.apache.solr.ltr.feature.ValueFeature",
+ "params" : {
+ "value": "${user_device_smartphone}"
+ }
+},
+ {
+ "name" : "user_device_tablet",
+ "class":"org.apache.solr.ltr.feature.ValueFeature",
+ "params" : {
+ "value": "${user_device_tablet}"
+ }
+ }
+
+
+
+]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/features-linear-efi.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/features-linear-efi.json b/solr/contrib/ltr/src/test-files/featureExamples/features-linear-efi.json
new file mode 100644
index 0000000..e05542a
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/features-linear-efi.json
@@ -0,0 +1,17 @@
+[
+ {
+ "name": "sampleConstant",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 5
+ }
+ },
+ {
+ "name" : "search_number_of_nights",
+ "class":"org.apache.solr.ltr.feature.ValueFeature",
+ "params" : {
+ "value": "${search_number_of_nights}"
+ }
+ }
+
+]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/features-linear.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/features-linear.json b/solr/contrib/ltr/src/test-files/featureExamples/features-linear.json
new file mode 100644
index 0000000..8cc2996
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/features-linear.json
@@ -0,0 +1,51 @@
+[
+ {
+ "name": "title",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 1
+ }
+ },
+ {
+ "name": "description",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 2
+ }
+ },
+ {
+ "name": "keywords",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 2
+ }
+ },
+ {
+ "name": "popularity",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 3
+ }
+ },
+ {
+ "name": "text",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 4
+ }
+ },
+ {
+ "name": "queryIntentPerson",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 5
+ }
+ },
+ {
+ "name": "queryIntentCompany",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 5
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/features-store-test-model.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/features-store-test-model.json b/solr/contrib/ltr/src/test-files/featureExamples/features-store-test-model.json
new file mode 100644
index 0000000..69aad84
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/features-store-test-model.json
@@ -0,0 +1,51 @@
+[
+ {
+ "name": "constant1",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "store":"test",
+ "params": {
+ "value": 1
+ }
+ },
+ {
+ "name": "constant2",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "store":"test",
+ "params": {
+ "value": 2
+ }
+ },
+ {
+ "name": "constant3",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "store":"test",
+ "params": {
+ "value": 3
+ }
+ },
+ {
+ "name": "constant4",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "store":"test",
+ "params": {
+ "value": 4
+ }
+ },
+ {
+ "name": "constant5",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "store":"test",
+ "params": {
+ "value": 5
+ }
+ },
+ {
+ "name": "pop",
+ "class": "org.apache.solr.ltr.feature.FieldValueFeature",
+ "store":"test",
+ "params": {
+ "field": "popularity"
+ }
+ }
+
+]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/fq_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/fq_features.json b/solr/contrib/ltr/src/test-files/featureExamples/fq_features.json
new file mode 100644
index 0000000..13968f9
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/fq_features.json
@@ -0,0 +1,16 @@
+[
+ {
+ "name": "matchedTitle",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": {
+ "q": "{!terms f=title}${user_query}"
+ }
+ },
+ {
+ "name": "popularity",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 3
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/featureExamples/multipleadditivetreesmodel_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/featureExamples/multipleadditivetreesmodel_features.json b/solr/contrib/ltr/src/test-files/featureExamples/multipleadditivetreesmodel_features.json
new file mode 100644
index 0000000..92f3861
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/featureExamples/multipleadditivetreesmodel_features.json
@@ -0,0 +1,16 @@
+[
+ {
+ "name": "matchedTitle",
+ "class": "org.apache.solr.ltr.feature.SolrFeature",
+ "params": {
+ "q": "{!terms f=title}${user_query}"
+ }
+ },
+ {
+ "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs",
+ "class": "org.apache.solr.ltr.feature.ValueFeature",
+ "params": {
+ "value": 1
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/log4j.properties
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/log4j.properties b/solr/contrib/ltr/src/test-files/log4j.properties
new file mode 100644
index 0000000..d86c698
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/log4j.properties
@@ -0,0 +1,32 @@
+# Logging level
+log4j.rootLogger=INFO, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Target=System.err
+log4j.appender.CONSOLE.layout=org.apache.log4j.EnhancedPatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r %-5p (%t) [%X{node_name} %X{collection} %X{shard} %X{replica} %X{core}] %c{1.} %m%n
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.directory=WARN
+log4j.logger.org.apache.solr.hadoop=INFO
+log4j.logger.org.apache.solr.client.solrj.embedded.JettySolrRunner=DEBUG
+org.apache.solr.client.solrj.embedded.JettySolrRunner=DEBUG
+
+#log4j.logger.org.apache.solr.update.processor.LogUpdateProcessor=DEBUG
+#log4j.logger.org.apache.solr.update.processor.DistributedUpdateProcessor=DEBUG
+#log4j.logger.org.apache.solr.update.PeerSync=DEBUG
+#log4j.logger.org.apache.solr.core.CoreContainer=DEBUG
+#log4j.logger.org.apache.solr.cloud.RecoveryStrategy=DEBUG
+#log4j.logger.org.apache.solr.cloud.SyncStrategy=DEBUG
+#log4j.logger.org.apache.solr.handler.admin.CoreAdminHandler=DEBUG
+#log4j.logger.org.apache.solr.cloud.ZkController=DEBUG
+#log4j.logger.org.apache.solr.update.DefaultSolrCoreState=DEBUG
+#log4j.logger.org.apache.solr.common.cloud.ConnectionManager=DEBUG
+#log4j.logger.org.apache.solr.update.UpdateLog=DEBUG
+#log4j.logger.org.apache.solr.cloud.ChaosMonkey=DEBUG
+#log4j.logger.org.apache.solr.update.TransactionLog=DEBUG
+#log4j.logger.org.apache.solr.handler.ReplicationHandler=DEBUG
+#log4j.logger.org.apache.solr.handler.IndexFetcher=DEBUG
+
+#log4j.logger.org.apache.solr.common.cloud.ClusterStateUtil=DEBUG
+#log4j.logger.org.apache.solr.cloud.OverseerAutoReplicaFailoverThread=DEBUG
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
new file mode 100644
index 0000000..04ab229
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json
@@ -0,0 +1,12 @@
+{
+ "class":"org.apache.solr.ltr.model.LinearModel",
+ "name":"externalmodel",
+ "features":[
+ { "name": "matchedTitle"}
+ ],
+ "params":{
+ "weights": {
+ "matchedTitle": 0.999
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json
new file mode 100644
index 0000000..f8e6648
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json
@@ -0,0 +1,13 @@
+{
+ "class":"org.apache.solr.ltr.model.LinearModel",
+ "name":"externalmodelstore",
+ "store": "fstore2",
+ "features":[
+ { "name": "confidence"}
+ ],
+ "params":{
+ "weights": {
+ "confidence": 0.999
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/fq-model.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/fq-model.json b/solr/contrib/ltr/src/test-files/modelExamples/fq-model.json
new file mode 100644
index 0000000..b5d631f
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/fq-model.json
@@ -0,0 +1,20 @@
+{
+ "class":"org.apache.solr.ltr.model.LinearModel",
+ "name":"fqmodel",
+ "features":[
+ {
+ "name":"matchedTitle",
+ "norm": {
+ "class":"org.apache.solr.ltr.norm.MinMaxNormalizer",
+ "params":{ "min":"0.0f", "max":"10.0f" }
+ }
+ },
+ { "name":"popularity"}
+ ],
+ "params":{
+ "weights": {
+ "matchedTitle": 0.5,
+ "popularity": 0.5
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/linear-model-efi.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/linear-model-efi.json b/solr/contrib/ltr/src/test-files/modelExamples/linear-model-efi.json
new file mode 100644
index 0000000..018466e
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/linear-model-efi.json
@@ -0,0 +1,14 @@
+{
+ "class":"org.apache.solr.ltr.model.LinearModel",
+ "name":"linear-efi",
+ "features":[
+ {"name":"sampleConstant"},
+ {"name":"search_number_of_nights"}
+ ],
+ "params":{
+ "weights":{
+ "sampleConstant":1.0,
+ "search_number_of_nights":2.0
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/linear-model.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/linear-model.json b/solr/contrib/ltr/src/test-files/modelExamples/linear-model.json
new file mode 100644
index 0000000..6b46dca
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/linear-model.json
@@ -0,0 +1,30 @@
+{
+ "class":"org.apache.solr.ltr.model.LinearModel",
+ "name":"6029760550880411648",
+ "features":[
+ {"name":"title"},
+ {"name":"description"},
+ {"name":"keywords"},
+ {
+ "name":"popularity",
+ "norm": {
+ "class":"org.apache.solr.ltr.norm.MinMaxNormalizer",
+ "params":{ "min":"0.0f", "max":"10.0f" }
+ }
+ },
+ {"name":"text"},
+ {"name":"queryIntentPerson"},
+ {"name":"queryIntentCompany"}
+ ],
+ "params":{
+ "weights": {
+ "title": 0.0000000000,
+ "description": 0.1000000000,
+ "keywords": 0.2000000000,
+ "popularity": 0.3000000000,
+ "text": 0.4000000000,
+ "queryIntentPerson":0.1231231,
+ "queryIntentCompany":0.12121211
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel.json
new file mode 100644
index 0000000..37551a0
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel.json
@@ -0,0 +1,38 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "1f",
+ "root": {
+ "feature": "matchedTitle",
+ "threshold": "0.5f",
+ "left" : {
+ "value" : "-100"
+ },
+ "right": {
+ "feature" : "this_feature_doesnt_exist",
+ "threshold": "10.0f",
+ "left" : {
+ "value" : "50"
+ },
+ "right" : {
+ "value" : "75"
+ }
+ }
+ }
+ },
+ {
+ "weight" : "2f",
+ "root": {
+ "value" : "-10"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_external_binary_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_external_binary_features.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_external_binary_features.json
new file mode 100644
index 0000000..cb8996e
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_external_binary_features.json
@@ -0,0 +1,38 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"external_model_binary_feature",
+ "features":[
+ { "name": "user_device_smartphone"},
+ { "name": "user_device_tablet"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "1f",
+ "root": {
+ "feature": "user_device_smartphone",
+ "threshold": "0.5f",
+ "left" : {
+ "value" : "0"
+ },
+ "right" : {
+ "value" : "50"
+ }
+
+ }},
+ {
+ "weight" : "1f",
+ "root": {
+ "feature": "user_device_tablet",
+ "threshold": "0.5f",
+ "left" : {
+ "value" : "0"
+ },
+ "right" : {
+ "value" : "65"
+ }
+
+ }}
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_feature.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_feature.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_feature.json
new file mode 100644
index 0000000..2919f07
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_feature.json
@@ -0,0 +1,24 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_feature",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "1f",
+ "root": {
+ "threshold": "0.5f",
+ "left" : {
+ "value" : "-100"
+ },
+ "right": {
+ "value" : "75"
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_features.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_features.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_features.json
new file mode 100644
index 0000000..ec4c37f
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_features.json
@@ -0,0 +1,14 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_features",
+ "params":{
+ "trees": [
+ {
+ "weight" : "2f",
+ "root": {
+ "value" : "-10"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_left.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_left.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_left.json
new file mode 100644
index 0000000..653d2ff
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_left.json
@@ -0,0 +1,22 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_left",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "1f",
+ "root": {
+ "feature": "matchedTitle",
+ "threshold": "0.5f",
+ "right": {
+ "value" : "75"
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_params.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_params.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_params.json
new file mode 100644
index 0000000..4d50c4e
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_params.json
@@ -0,0 +1,8 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_params",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ]
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_right.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_right.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_right.json
new file mode 100644
index 0000000..acd2d83
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_right.json
@@ -0,0 +1,22 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_right",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "1f",
+ "root": {
+ "feature": "matchedTitle",
+ "threshold": "0.5f",
+ "left" : {
+ "value" : "-100"
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_threshold.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_threshold.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_threshold.json
new file mode 100644
index 0000000..d0fc381
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_threshold.json
@@ -0,0 +1,24 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_threshold",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "1f",
+ "root": {
+ "feature": "matchedTitle",
+ "left" : {
+ "value" : "-100"
+ },
+ "right": {
+ "value" : "75"
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_tree.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_tree.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_tree.json
new file mode 100644
index 0000000..507def3
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_tree.json
@@ -0,0 +1,15 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_tree",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "weight" : "2f"
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_trees.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_trees.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_trees.json
new file mode 100644
index 0000000..bb360dd
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_trees.json
@@ -0,0 +1,10 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_trees",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_weight.json
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_weight.json b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_weight.json
new file mode 100644
index 0000000..9048e6c
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/modelExamples/multipleadditivetreesmodel_no_weight.json
@@ -0,0 +1,24 @@
+{
+ "class":"org.apache.solr.ltr.model.MultipleAdditiveTreesModel",
+ "name":"multipleadditivetreesmodel_no_weight",
+ "features":[
+ { "name": "matchedTitle"},
+ { "name": "constantScoreToForceMultipleAdditiveTreesScoreAllDocs"}
+ ],
+ "params":{
+ "trees": [
+ {
+ "root": {
+ "feature": "matchedTitle",
+ "threshold": "0.5f",
+ "left" : {
+ "value" : "-100"
+ },
+ "right": {
+ "value" : "75"
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/schema.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/schema.xml
new file mode 100644
index 0000000..15cf140
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/schema.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="example" version="1.5">
+ <fields>
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+ <field name="title" type="text_general" indexed="true" stored="true"/>
+ <field name="description" type="text_general" indexed="true" stored="true"/>
+ <field name="keywords" type="text_general" indexed="true" stored="true" multiValued="true"/>
+ <field name="popularity" type="int" indexed="true" stored="true" />
+ <field name="normHits" type="float" indexed="true" stored="true" />
+ <field name="text" type="text_general" indexed="true" stored="false" multiValued="true"/>
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+
+ <dynamicField name="*_s" type="string" indexed="true" stored="true" />
+ <dynamicField name="*_t" type="text_general" indexed="true" stored="true"/>
+ </fields>
+
+ <uniqueKey>id</uniqueKey>
+
+ <copyField source="title" dest="text"/>
+ <copyField source="description" dest="text"/>
+
+ <types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldtype name="binary" class="solr.BinaryField"/>
+
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="text_lc" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ </fieldType>
+ </types>
+
+ <!-- Similarity is the scoring routine for each document vs. a query.
+ A custom Similarity or SimilarityFactory may be specified here, but
+ the default is fine for most applications.
+ For more info: http://wiki.apache.org/solr/SchemaXml#Similarity
+ -->
+ <!--
+ <similarity class="com.example.solr.CustomSimilarityFactory">
+ <str name="paramkey">param value</str>
+ </similarity>
+ -->
+
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
new file mode 100644
index 0000000..1a18471
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<config>
+ <luceneMatchVersion>6.0.0</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.RAMDirectoryFactory}" />
+
+ <schemaFactory class="ClassicIndexSchemaFactory" />
+
+
+ <!-- Query parser used to rerank top docs with a provided model -->
+ <queryParser name="ltr"
+ class="org.apache.solr.search.LTRQParserPlugin" />
+
+ <query>
+ <filterCache class="solr.FastLRUCache" size="4096"
+ initialSize="2048" autowarmCount="0" />
+ <cache name="QUERY_DOC_FV" class="solr.search.LRUCache" size="4096"
+ initialSize="2048" autowarmCount="4096" regenerator="solr.search.NoOpRegenerator" />
+ </query>
+
+ <!-- add a transformer that will encode the document features in the response.
+ For each document the transformer will add the features as an extra field
+ in the response. The name of the field we will be the the name of the transformer
+ enclosed between brackets (in this case [fv]). In order to get the feature
+ vector you will have to specify that you want the field (e.g., fl="*,[fv]) -->
+ <transformer name="fv"
+ class="org.apache.solr.response.transform.LTRFeatureLoggerTransformerFactory" />
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <autoCommit>
+ <maxTime>15000</maxTime>
+ <openSearcher>false</openSearcher>
+ </autoCommit>
+ <autoSoftCommit>
+ <maxTime>1000</maxTime>
+ </autoSoftCommit>
+ <updateLog>
+ <str name="dir">${solr.data.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
+ <!-- Query request handler managing models and features -->
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">id</str>
+ </lst>
+ </requestHandler>
+
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
new file mode 100644
index 0000000..fd0940a
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<config>
+ <luceneMatchVersion>6.0.0</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.RAMDirectoryFactory}" />
+
+ <schemaFactory class="ClassicIndexSchemaFactory" />
+
+
+ <!-- Query parser used to rerank top docs with a provided model -->
+ <queryParser name="ltr" class="org.apache.solr.search.LTRQParserPlugin" >
+ <int name="threadModule.totalPoolThreads">10</int> <!-- Maximum threads to use for all queries -->
+ <int name="threadModule.numThreadsPerRequest">10</int> <!-- Maximum threads to use for a single query-->
+ </queryParser>
+
+
+
+ <query>
+ <filterCache class="solr.FastLRUCache" size="4096"
+ initialSize="2048" autowarmCount="0" />
+ <cache name="QUERY_DOC_FV" class="solr.search.LRUCache" size="4096"
+ initialSize="2048" autowarmCount="4096" regenerator="solr.search.NoOpRegenerator" />
+ </query>
+
+ <!-- add a transformer that will encode the document features in the response.
+ For each document the transformer will add the features as an extra field
+ in the response. The name of the field we will be the the name of the transformer
+ enclosed between brackets (in this case [fv]). In order to get the feature
+ vector you will have to specify that you want the field (e.g., fl="*,[fv]) -->
+ <transformer name="fv"
+ class="org.apache.solr.response.transform.LTRFeatureLoggerTransformerFactory" />
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <autoCommit>
+ <maxTime>15000</maxTime>
+ <openSearcher>false</openSearcher>
+ </autoCommit>
+ <autoSoftCommit>
+ <maxTime>1000</maxTime>
+ </autoSoftCommit>
+ <updateLog>
+ <str name="dir">${solr.data.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
+ <!-- Query request handler managing models and features -->
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">id</str>
+ </lst>
+ </requestHandler>
+
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
new file mode 100644
index 0000000..a36c1df
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<config>
+ <luceneMatchVersion>6.0.0</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.RAMDirectoryFactory}" />
+
+ <schemaFactory class="ClassicIndexSchemaFactory" />
+
+
+ <!-- Query parser used to rerank top docs with a provided model -->
+ <queryParser name="ltr" class="org.apache.solr.search.LTRQParserPlugin" />
+
+ <maxBufferedDocs>1</maxBufferedDocs>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">10</int>
+ <int name="segmentsPerTier">1000</int>
+ </mergePolicyFactory>
+ <!-- add a transformer that will encode the document features in the response.
+ For each document the transformer will add the features as an extra field
+ in the response. The name of the field we will be the the name of the transformer
+ enclosed between brackets (in this case [fv]). In order to get the feature
+ vector you will have to specify that you want the field (e.g., fl="*,[fv]) -->
+ <transformer name="features"
+ class="org.apache.solr.response.transform.LTRFeatureLoggerTransformerFactory" />
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <autoCommit>
+ <maxTime>15000</maxTime>
+ <openSearcher>false</openSearcher>
+ </autoCommit>
+ <autoSoftCommit>
+ <maxTime>1000</maxTime>
+ </autoSoftCommit>
+ <updateLog>
+ <str name="dir">${solr.data.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
+ <!-- Query request handler managing models and features -->
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">id</str>
+ </lst>
+ </requestHandler>
+
+</config>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/solr/collection1/conf/stopwords.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/stopwords.txt b/solr/contrib/ltr/src/test-files/solr/collection1/conf/stopwords.txt
new file mode 100644
index 0000000..eabae3b
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/stopwords.txt
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+a
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a511b30a/solr/contrib/ltr/src/test-files/solr/collection1/conf/synonyms.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test-files/solr/collection1/conf/synonyms.txt b/solr/contrib/ltr/src/test-files/solr/collection1/conf/synonyms.txt
new file mode 100644
index 0000000..0ef0e8d
--- /dev/null
+++ b/solr/contrib/ltr/src/test-files/solr/collection1/conf/synonyms.txt
@@ -0,0 +1,28 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+#some test synonym mappings unlikely to appear in real input text
+aaafoo => aaabar
+bbbfoo => bbbfoo bbbbar
+cccfoo => cccbar cccbaz
+fooaaa,baraaa,bazaaa
+
+# Some synonym groups specific to this example
+GB,gib,gigabyte,gigabytes
+MB,mib,megabyte,megabytes
+Television, Televisions, TV, TVs
+#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
+#after us won't split it into two words.
+
+# Synonym mappings can be used for spelling correction too
+pixima => pixma