You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:50:03 UTC
[29/50] [abbrv] incubator-lens git commit: Lens-465 : Refactor ml
packages. (sharad)
Lens-465 : Refactor ml packages. (sharad)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/0f5ea4c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/0f5ea4c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/0f5ea4c7
Branch: refs/heads/current-release-line
Commit: 0f5ea4c7827fc4740c1c2ba0fb9527903a2b910c
Parents: 278e0e8
Author: Sharad Agarwal <sh...@flipkarts-MacBook-Pro.local>
Authored: Sun Apr 5 10:48:32 2015 +0530
Committer: Sharad Agarwal <sh...@flipkarts-MacBook-Pro.local>
Committed: Sun Apr 5 10:48:32 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/api/ml/ModelMetadata.java | 118 ---
.../java/org/apache/lens/api/ml/TestReport.java | 125 ----
.../org/apache/lens/client/LensMLClient.java | 12 +-
.../apache/lens/client/LensMLJerseyClient.java | 4 +-
.../java/org/apache/lens/ml/AlgoArgParser.java | 114 ---
.../main/java/org/apache/lens/ml/AlgoParam.java | 53 --
.../main/java/org/apache/lens/ml/Algorithm.java | 46 --
.../java/org/apache/lens/ml/Algorithms.java | 87 ---
.../org/apache/lens/ml/ClassifierBaseModel.java | 46 --
.../java/org/apache/lens/ml/ExampleUtils.java | 101 ---
.../org/apache/lens/ml/ForecastingModel.java | 93 ---
.../main/java/org/apache/lens/ml/HiveMLUDF.java | 136 ----
.../org/apache/lens/ml/LabelledPrediction.java | 32 -
.../main/java/org/apache/lens/ml/LensML.java | 159 ----
.../java/org/apache/lens/ml/LensMLImpl.java | 734 ------------------
.../main/java/org/apache/lens/ml/MLAlgo.java | 53 --
.../main/java/org/apache/lens/ml/MLDriver.java | 71 --
.../main/java/org/apache/lens/ml/MLModel.java | 79 --
.../main/java/org/apache/lens/ml/MLRunner.java | 173 -----
.../java/org/apache/lens/ml/MLTestMetric.java | 28 -
.../java/org/apache/lens/ml/MLTestReport.java | 95 ---
.../main/java/org/apache/lens/ml/MLUtils.java | 60 --
.../java/org/apache/lens/ml/ModelLoader.java | 239 ------
.../org/apache/lens/ml/MultiPrediction.java | 28 -
.../java/org/apache/lens/ml/QueryRunner.java | 56 --
.../org/apache/lens/ml/TableTestingSpec.java | 325 --------
.../org/apache/lens/ml/algo/api/AlgoParam.java | 53 ++
.../org/apache/lens/ml/algo/api/Algorithm.java | 46 ++
.../org/apache/lens/ml/algo/api/MLAlgo.java | 53 ++
.../org/apache/lens/ml/algo/api/MLDriver.java | 71 ++
.../org/apache/lens/ml/algo/api/MLModel.java | 79 ++
.../apache/lens/ml/algo/lib/AlgoArgParser.java | 117 +++
.../org/apache/lens/ml/algo/lib/Algorithms.java | 89 +++
.../lens/ml/algo/lib/ClassifierBaseModel.java | 48 ++
.../lens/ml/algo/lib/ForecastingModel.java | 95 +++
.../lens/ml/algo/lib/LabelledPrediction.java | 32 +
.../lens/ml/algo/lib/MultiPrediction.java | 28 +
.../lens/ml/algo/spark/BaseSparkAlgo.java | 287 +++++++
.../spark/BaseSparkClassificationModel.java | 65 ++
.../ml/algo/spark/ColumnFeatureFunction.java | 102 +++
.../lens/ml/algo/spark/DoubleValueMapper.java | 39 +
.../lens/ml/algo/spark/FeatureFunction.java | 40 +
.../lens/ml/algo/spark/FeatureValueMapper.java | 36 +
.../apache/lens/ml/algo/spark/HiveTableRDD.java | 63 ++
.../lens/ml/algo/spark/SparkMLDriver.java | 278 +++++++
.../lens/ml/algo/spark/TableTrainingSpec.java | 433 +++++++++++
.../lens/ml/algo/spark/dt/DecisionTreeAlgo.java | 108 +++
.../dt/DecisionTreeClassificationModel.java | 37 +
.../algo/spark/dt/SparkDecisionTreeModel.java | 75 ++
.../lens/ml/algo/spark/kmeans/KMeansAlgo.java | 163 ++++
.../spark/kmeans/KMeansClusteringModel.java | 67 ++
.../algo/spark/lr/LogisticRegressionAlgo.java | 86 +++
.../lr/LogitRegressionClassificationModel.java | 39 +
.../lens/ml/algo/spark/nb/NaiveBayesAlgo.java | 73 ++
.../spark/nb/NaiveBayesClassificationModel.java | 39 +
.../apache/lens/ml/algo/spark/svm/SVMAlgo.java | 90 +++
.../algo/spark/svm/SVMClassificationModel.java | 39 +
.../java/org/apache/lens/ml/api/LensML.java | 161 ++++
.../org/apache/lens/ml/api/MLTestReport.java | 95 +++
.../org/apache/lens/ml/api/ModelMetadata.java | 118 +++
.../java/org/apache/lens/ml/api/TestReport.java | 125 ++++
.../java/org/apache/lens/ml/dao/MLDBUtils.java | 6 +-
.../java/org/apache/lens/ml/impl/HiveMLUDF.java | 138 ++++
.../org/apache/lens/ml/impl/LensMLImpl.java | 744 +++++++++++++++++++
.../java/org/apache/lens/ml/impl/MLRunner.java | 172 +++++
.../java/org/apache/lens/ml/impl/MLTask.java | 285 +++++++
.../java/org/apache/lens/ml/impl/MLUtils.java | 62 ++
.../org/apache/lens/ml/impl/ModelLoader.java | 242 ++++++
.../org/apache/lens/ml/impl/QueryRunner.java | 56 ++
.../apache/lens/ml/impl/TableTestingSpec.java | 325 ++++++++
.../java/org/apache/lens/ml/server/MLApp.java | 60 ++
.../org/apache/lens/ml/server/MLService.java | 27 +
.../apache/lens/ml/server/MLServiceImpl.java | 329 ++++++++
.../lens/ml/server/MLServiceResource.java | 427 +++++++++++
.../lens/ml/spark/ColumnFeatureFunction.java | 102 ---
.../apache/lens/ml/spark/DoubleValueMapper.java | 39 -
.../apache/lens/ml/spark/FeatureFunction.java | 40 -
.../lens/ml/spark/FeatureValueMapper.java | 36 -
.../org/apache/lens/ml/spark/HiveTableRDD.java | 63 --
.../org/apache/lens/ml/spark/SparkMLDriver.java | 275 -------
.../apache/lens/ml/spark/TableTrainingSpec.java | 433 -----------
.../lens/ml/spark/algos/BaseSparkAlgo.java | 290 --------
.../lens/ml/spark/algos/DecisionTreeAlgo.java | 109 ---
.../apache/lens/ml/spark/algos/KMeansAlgo.java | 163 ----
.../ml/spark/algos/LogisticRegressionAlgo.java | 86 ---
.../lens/ml/spark/algos/NaiveBayesAlgo.java | 73 --
.../org/apache/lens/ml/spark/algos/SVMAlgo.java | 90 ---
.../models/BaseSparkClassificationModel.java | 65 --
.../models/DecisionTreeClassificationModel.java | 35 -
.../ml/spark/models/KMeansClusteringModel.java | 67 --
.../LogitRegressionClassificationModel.java | 37 -
.../models/NaiveBayesClassificationModel.java | 37 -
.../ml/spark/models/SVMClassificationModel.java | 37 -
.../ml/spark/models/SparkDecisionTreeModel.java | 75 --
.../java/org/apache/lens/ml/task/MLTask.java | 286 -------
.../java/org/apache/lens/rdd/LensRDDClient.java | 2 +-
.../java/org/apache/lens/server/ml/MLApp.java | 60 --
.../org/apache/lens/server/ml/MLService.java | 27 -
.../apache/lens/server/ml/MLServiceImpl.java | 324 --------
.../lens/server/ml/MLServiceResource.java | 415 -----------
.../java/org/apache/lens/ml/ExampleUtils.java | 101 +++
.../java/org/apache/lens/ml/TestMLResource.java | 15 +-
.../java/org/apache/lens/ml/TestMLRunner.java | 7 +-
lens-ml-lib/src/test/resources/lens-site.xml | 6 +-
tools/conf-pseudo-distr/server/lens-site.xml | 6 +-
105 files changed, 6367 insertions(+), 6343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/api/ml/ModelMetadata.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/api/ml/ModelMetadata.java b/lens-ml-lib/src/main/java/org/apache/lens/api/ml/ModelMetadata.java
deleted file mode 100644
index 0f072bf..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/api/ml/ModelMetadata.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.ml;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-/**
- * The Class ModelMetadata.
- */
-@XmlRootElement
-/**
- * Instantiates a new model metadata.
- *
- * @param modelID
- * the model id
- * @param table
- * the table
- * @param algorithm
- * the algorithm
- * @param params
- * the params
- * @param createdAt
- * the created at
- * @param modelPath
- * the model path
- * @param labelColumn
- * the label column
- * @param features
- * the features
- */
-@AllArgsConstructor
-/**
- * Instantiates a new model metadata.
- */
-@NoArgsConstructor
-public class ModelMetadata {
-
- /** The model id. */
- @XmlElement
- @Getter
- private String modelID;
-
- /** The table. */
- @XmlElement
- @Getter
- private String table;
-
- /** The algorithm. */
- @XmlElement
- @Getter
- private String algorithm;
-
- /** The params. */
- @XmlElement
- @Getter
- private String params;
-
- /** The created at. */
- @XmlElement
- @Getter
- private String createdAt;
-
- /** The model path. */
- @XmlElement
- @Getter
- private String modelPath;
-
- /** The label column. */
- @XmlElement
- @Getter
- private String labelColumn;
-
- /** The features. */
- @XmlElement
- @Getter
- private String features;
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#toString()
- */
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
-
- builder.append("Algorithm: ").append(algorithm).append('\n');
- builder.append("Model ID: ").append(modelID).append('\n');
- builder.append("Training table: ").append(table).append('\n');
- builder.append("Features: ").append(features).append('\n');
- builder.append("Labelled Column: ").append(labelColumn).append('\n');
- builder.append("Training params: ").append(params).append('\n');
- builder.append("Created on: ").append(createdAt).append('\n');
- builder.append("Model saved at: ").append(modelPath).append('\n');
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java b/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
deleted file mode 100644
index 2ae384b..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.ml;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-/**
- * The Class TestReport.
- */
-@XmlRootElement
-/**
- * Instantiates a new test report.
- *
- * @param testTable
- * the test table
- * @param outputTable
- * the output table
- * @param outputColumn
- * the output column
- * @param labelColumn
- * the label column
- * @param featureColumns
- * the feature columns
- * @param algorithm
- * the algorithm
- * @param modelID
- * the model id
- * @param reportID
- * the report id
- * @param queryID
- * the query id
- */
-@AllArgsConstructor
-/**
- * Instantiates a new test report.
- */
-@NoArgsConstructor
-public class TestReport {
-
- /** The test table. */
- @XmlElement
- @Getter
- private String testTable;
-
- /** The output table. */
- @XmlElement
- @Getter
- private String outputTable;
-
- /** The output column. */
- @XmlElement
- @Getter
- private String outputColumn;
-
- /** The label column. */
- @XmlElement
- @Getter
- private String labelColumn;
-
- /** The feature columns. */
- @XmlElement
- @Getter
- private String featureColumns;
-
- /** The algorithm. */
- @XmlElement
- @Getter
- private String algorithm;
-
- /** The model id. */
- @XmlElement
- @Getter
- private String modelID;
-
- /** The report id. */
- @XmlElement
- @Getter
- private String reportID;
-
- /** The query id. */
- @XmlElement
- @Getter
- private String queryID;
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#toString()
- */
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("Input test table: ").append(testTable).append('\n');
- builder.append("Algorithm: ").append(algorithm).append('\n');
- builder.append("Report id: ").append(reportID).append('\n');
- builder.append("Model id: ").append(modelID).append('\n');
- builder.append("Lens Query id: ").append(queryID).append('\n');
- builder.append("Feature columns: ").append(featureColumns).append('\n');
- builder.append("Labelled column: ").append(labelColumn).append('\n');
- builder.append("Predicted column: ").append(outputColumn).append('\n');
- builder.append("Test output table: ").append(outputTable).append('\n');
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
index d9ec314..4d4521e 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
@@ -32,12 +32,12 @@ import javax.ws.rs.core.Form;
import org.apache.lens.api.LensException;
import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.ml.ModelMetadata;
-import org.apache.lens.api.ml.TestReport;
-import org.apache.lens.ml.LensML;
-import org.apache.lens.ml.MLAlgo;
-import org.apache.lens.ml.MLModel;
-import org.apache.lens.ml.MLTestReport;
+import org.apache.lens.ml.algo.api.MLAlgo;
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.api.LensML;
+import org.apache.lens.ml.api.MLTestReport;
+import org.apache.lens.ml.api.ModelMetadata;
+import org.apache.lens.ml.api.TestReport;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
index af47a41..c68dd12 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
@@ -31,8 +31,8 @@ import javax.ws.rs.core.MediaType;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
-import org.apache.lens.api.ml.ModelMetadata;
-import org.apache.lens.api.ml.TestReport;
+import org.apache.lens.ml.api.ModelMetadata;
+import org.apache.lens.ml.api.TestReport;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoArgParser.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoArgParser.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoArgParser.java
deleted file mode 100644
index 20da083..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoArgParser.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * The Class AlgoArgParser.
- */
-public final class AlgoArgParser {
- private AlgoArgParser() {
- }
-
- /**
- * The Class CustomArgParser.
- *
- * @param <E> the element type
- */
- public abstract static class CustomArgParser<E> {
-
- /**
- * Parses the.
- *
- * @param value the value
- * @return the e
- */
- public abstract E parse(String value);
- }
-
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(AlgoArgParser.class);
-
- /**
- * Extracts feature names. If the algo has any parameters associated with @AlgoParam annotation, those are set
- * as well.
- *
- * @param algo the algo
- * @param args the args
- * @return List of feature column names.
- */
- public static List<String> parseArgs(MLAlgo algo, String[] args) {
- List<String> featureColumns = new ArrayList<String>();
- Class<? extends MLAlgo> algoClass = algo.getClass();
- // Get param fields
- Map<String, Field> fieldMap = new HashMap<String, Field>();
-
- for (Field fld : algoClass.getDeclaredFields()) {
- fld.setAccessible(true);
- AlgoParam paramAnnotation = fld.getAnnotation(AlgoParam.class);
- if (paramAnnotation != null) {
- fieldMap.put(paramAnnotation.name(), fld);
- }
- }
-
- for (int i = 0; i < args.length; i += 2) {
- String key = args[i].trim();
- String value = args[i + 1].trim();
-
- try {
- if ("feature".equalsIgnoreCase(key)) {
- featureColumns.add(value);
- } else if (fieldMap.containsKey(key)) {
- Field f = fieldMap.get(key);
- if (String.class.equals(f.getType())) {
- f.set(algo, value);
- } else if (Integer.TYPE.equals(f.getType())) {
- f.setInt(algo, Integer.parseInt(value));
- } else if (Double.TYPE.equals(f.getType())) {
- f.setDouble(algo, Double.parseDouble(value));
- } else if (Long.TYPE.equals(f.getType())) {
- f.setLong(algo, Long.parseLong(value));
- } else {
- // check if the algo provides a deserializer for this param
- String customParserClass = algo.getConf().getProperties().get("lens.ml.args." + key);
- if (customParserClass != null) {
- Class<? extends CustomArgParser<?>> clz = (Class<? extends CustomArgParser<?>>) Class
- .forName(customParserClass);
- CustomArgParser<?> parser = clz.newInstance();
- f.set(algo, parser.parse(value));
- } else {
- LOG.warn("Ignored param " + key + "=" + value + " as no parser found");
- }
- }
- }
- } catch (Exception exc) {
- LOG.error("Error while setting param " + key + " to " + value + " for algo " + algo);
- }
- }
- return featureColumns;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoParam.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoParam.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoParam.java
deleted file mode 100644
index 5836f51..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/AlgoParam.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * The Interface AlgoParam.
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.FIELD)
-public @interface AlgoParam {
-
- /**
- * Name.
- *
- * @return the string
- */
- String name();
-
- /**
- * Help.
- *
- * @return the string
- */
- String help();
-
- /**
- * Default value.
- *
- * @return the string
- */
- String defaultValue() default "None";
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithm.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithm.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithm.java
deleted file mode 100644
index 7025d7b..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithm.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * The Interface Algorithm.
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.TYPE)
-public @interface Algorithm {
-
- /**
- * Name.
- *
- * @return the string
- */
- String name();
-
- /**
- * Description.
- *
- * @return the string
- */
- String description();
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithms.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithms.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithms.java
deleted file mode 100644
index c1b7212..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/Algorithms.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.lang.reflect.Constructor;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lens.api.LensException;
-
-/**
- * The Class Algorithms.
- */
-public class Algorithms {
-
- /** The algorithm classes. */
- private final Map<String, Class<? extends MLAlgo>> algorithmClasses
- = new HashMap<String, Class<? extends MLAlgo>>();
-
- /**
- * Register.
- *
- * @param algoClass the algo class
- */
- public void register(Class<? extends MLAlgo> algoClass) {
- if (algoClass != null && algoClass.getAnnotation(Algorithm.class) != null) {
- algorithmClasses.put(algoClass.getAnnotation(Algorithm.class).name(), algoClass);
- } else {
- throw new IllegalArgumentException("Not a valid algorithm class: " + algoClass);
- }
- }
-
- /**
- * Gets the algo for name.
- *
- * @param name the name
- * @return the algo for name
- * @throws LensException the lens exception
- */
- public MLAlgo getAlgoForName(String name) throws LensException {
- Class<? extends MLAlgo> algoClass = algorithmClasses.get(name);
- if (algoClass == null) {
- return null;
- }
- Algorithm algoAnnotation = algoClass.getAnnotation(Algorithm.class);
- String description = algoAnnotation.description();
- try {
- Constructor<? extends MLAlgo> algoConstructor = algoClass.getConstructor(String.class, String.class);
- return algoConstructor.newInstance(name, description);
- } catch (Exception exc) {
- throw new LensException("Unable to get algo: " + name, exc);
- }
- }
-
- /**
- * Checks if is algo supported.
- *
- * @param name the name
- * @return true, if is algo supported
- */
- public boolean isAlgoSupported(String name) {
- return algorithmClasses.containsKey(name);
- }
-
- public List<String> getAlgorithmNames() {
- return new ArrayList<String>(algorithmClasses.keySet());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/ClassifierBaseModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/ClassifierBaseModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/ClassifierBaseModel.java
deleted file mode 100644
index 68008fe..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/ClassifierBaseModel.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-/**
- * Return a single double value as a prediction. This is useful in classifiers where the classifier returns a single
- * class label as a prediction.
- */
-public abstract class ClassifierBaseModel extends MLModel<Double> {
-
- /**
- * Gets the feature vector.
- *
- * @param args the args
- * @return the feature vector
- */
- public final double[] getFeatureVector(Object[] args) {
- double[] features = new double[args.length];
- for (int i = 0; i < args.length; i++) {
- if (args[i] instanceof Double) {
- features[i] = (Double) args[i];
- } else if (args[i] instanceof String) {
- features[i] = Double.parseDouble((String) args[i]);
- } else {
- features[i] = 0.0;
- }
- }
- return features;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
deleted file mode 100644
index 9fe1ea0..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-/**
- * The Class ExampleUtils.
- */
-public final class ExampleUtils {
- private ExampleUtils() {
- }
-
- private static final Log LOG = LogFactory.getLog(ExampleUtils.class);
-
- /**
- * Creates the example table.
- *
- * @param conf the conf
- * @param database the database
- * @param tableName the table name
- * @param sampleDataFile the sample data file
- * @param labelColumn the label column
- * @param features the features
- * @throws HiveException the hive exception
- */
- public static void createTable(HiveConf conf, String database, String tableName, String sampleDataFile,
- String labelColumn, Map<String, String> tableParams, String... features) throws HiveException {
-
- Path dataFilePath = new Path(sampleDataFile);
- Path partDir = dataFilePath.getParent();
-
- // Create table
- List<FieldSchema> columns = new ArrayList<FieldSchema>();
-
- // Label is optional. Not used for unsupervised models.
- // If present, label will be the first column, followed by features
- if (labelColumn != null) {
- columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
- }
-
- for (String feature : features) {
- columns.add(new FieldSchema(feature, "double", "Feature " + feature));
- }
-
- Table tbl = Hive.get(conf).newTable(database + "." + tableName);
- tbl.setTableType(TableType.MANAGED_TABLE);
- tbl.getTTable().getSd().setCols(columns);
- tbl.getTTable().getParameters().putAll(tableParams);
- tbl.setInputFormatClass(TextInputFormat.class);
- tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
- tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");
-
- List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
- partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
- tbl.setPartCols(partCols);
-
- Hive.get(conf).createTable(tbl, false);
- LOG.info("Created table " + tableName);
-
- // Add partition for the data file
- AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName, false);
- Map<String, String> partSpec = new HashMap<String, String>();
- partSpec.put("dummy_partition_col", "dummy_val");
- partitionDesc.addPartition(partSpec, partDir.toUri().toString());
- Hive.get(conf).createPartitions(partitionDesc);
- LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/ForecastingModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/ForecastingModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/ForecastingModel.java
deleted file mode 100644
index 5163db5..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/ForecastingModel.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.util.List;
-
-/**
- * The Class ForecastingModel.
- */
-public class ForecastingModel extends MLModel<MultiPrediction> {
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.MLModel#predict(java.lang.Object[])
- */
- @Override
- public MultiPrediction predict(Object... args) {
- return new ForecastingPredictions(null);
- }
-
- /**
- * The Class ForecastingPredictions.
- */
- public static class ForecastingPredictions implements MultiPrediction {
-
- /** The values. */
- private final List<LabelledPrediction> values;
-
- /**
- * Instantiates a new forecasting predictions.
- *
- * @param values the values
- */
- public ForecastingPredictions(List<LabelledPrediction> values) {
- this.values = values;
- }
-
- @Override
- public List<LabelledPrediction> getPredictions() {
- return values;
- }
- }
-
- /**
- * The Class ForecastingLabel.
- */
- public static class ForecastingLabel implements LabelledPrediction<Long, Double> {
-
- /** The timestamp. */
- private final Long timestamp;
-
- /** The value. */
- private final double value;
-
- /**
- * Instantiates a new forecasting label.
- *
- * @param timestamp the timestamp
- * @param value the value
- */
- public ForecastingLabel(long timestamp, double value) {
- this.timestamp = timestamp;
- this.value = value;
- }
-
- @Override
- public Long getLabel() {
- return timestamp;
- }
-
- @Override
- public Double getPrediction() {
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/HiveMLUDF.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/HiveMLUDF.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/HiveMLUDF.java
deleted file mode 100644
index 687ca54..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/HiveMLUDF.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.MapredContext;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.hive.serde2.lazy.LazyDouble;
-import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.mapred.JobConf;
-
-/**
- * Generic UDF to laod ML Models saved in HDFS and apply the model on list of columns passed as argument.
- */
-@Description(name = "predict",
- value = "_FUNC_(algorithm, modelID, features...) - Run prediction algorithm with given "
- + "algorithm name, model ID and input feature columns")
-public final class HiveMLUDF extends GenericUDF {
- private HiveMLUDF() {
- }
-
- /** The Constant UDF_NAME. */
- public static final String UDF_NAME = "predict";
-
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(HiveMLUDF.class);
-
- /** The conf. */
- private JobConf conf;
-
- /** The soi. */
- private StringObjectInspector soi;
-
- /** The doi. */
- private LazyDoubleObjectInspector doi;
-
- /** The model. */
- private MLModel model;
-
- /**
- * Currently we only support double as the return value.
- *
- * @param objectInspectors the object inspectors
- * @return the object inspector
- * @throws UDFArgumentException the UDF argument exception
- */
- @Override
- public ObjectInspector initialize(ObjectInspector[] objectInspectors) throws UDFArgumentException {
- // We require algo name, model id and at least one feature
- if (objectInspectors.length < 3) {
- throw new UDFArgumentLengthException("Algo name, model ID and at least one feature should be passed to "
- + UDF_NAME);
- }
- LOG.info(UDF_NAME + " initialized");
- return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hive.ql.udf.generic.GenericUDF#evaluate(org.apache.hadoop.hive.ql.udf.generic.GenericUDF.
- * DeferredObject[])
- */
- @Override
- public Object evaluate(DeferredObject[] deferredObjects) throws HiveException {
- String algorithm = soi.getPrimitiveJavaObject(deferredObjects[0].get());
- String modelId = soi.getPrimitiveJavaObject(deferredObjects[1].get());
-
- Double[] features = new Double[deferredObjects.length - 2];
- for (int i = 2; i < deferredObjects.length; i++) {
- LazyDouble lazyDouble = (LazyDouble) deferredObjects[i].get();
- features[i - 2] = (lazyDouble == null) ? 0d : doi.get(lazyDouble);
- }
-
- try {
- if (model == null) {
- model = ModelLoader.loadModel(conf, algorithm, modelId);
- }
- } catch (IOException e) {
- throw new HiveException(e);
- }
-
- return model.predict(features);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hive.ql.udf.generic.GenericUDF#getDisplayString(java.lang.String[])
- */
- @Override
- public String getDisplayString(String[] strings) {
- return UDF_NAME;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hive.ql.udf.generic.GenericUDF#configure(org.apache.hadoop.hive.ql.exec.MapredContext)
- */
- @Override
- public void configure(MapredContext context) {
- super.configure(context);
- conf = context.getJobConf();
- soi = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- doi = LazyPrimitiveObjectInspectorFactory.LAZY_DOUBLE_OBJECT_INSPECTOR;
- LOG.info(UDF_NAME + " configured. Model base dir path: " + conf.get(ModelLoader.MODEL_PATH_BASE_DIR));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/LabelledPrediction.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/LabelledPrediction.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/LabelledPrediction.java
deleted file mode 100644
index 6e7f677..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/LabelledPrediction.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-/**
- * Prediction type used when the model prediction is of complex types. For example, in forecasting the predictions are a
- * series of timestamp, and value pairs.
- *
- * @param <LABELTYPE> the generic type
- * @param <PREDICTIONTYPE> the generic type
- */
-public interface LabelledPrediction<LABELTYPE, PREDICTIONTYPE> {
- LABELTYPE getLabel();
-
- PREDICTIONTYPE getPrediction();
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
deleted file mode 100644
index cdf28dd..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.LensSessionHandle;
-
-/**
- * Lens's machine learning interface used by client code as well as Lens ML service.
- */
-public interface LensML {
-
- /** Name of ML service */
- String NAME = "ml";
-
- /**
- * Get list of available machine learning algorithms
- *
- * @return
- */
- List<String> getAlgorithms();
-
- /**
- * Get user friendly information about parameters accepted by the algorithm.
- *
- * @param algorithm the algorithm
- * @return map of param key to its help message
- */
- Map<String, String> getAlgoParamDescription(String algorithm);
-
- /**
- * Get a algo object instance which could be used to generate a model of the given algorithm.
- *
- * @param algorithm the algorithm
- * @return the algo for name
- * @throws LensException the lens exception
- */
- MLAlgo getAlgoForName(String algorithm) throws LensException;
-
- /**
- * Create a model using the given HCatalog table as input. The arguments should contain information needeed to
- * generate the model.
- *
- * @param table the table
- * @param algorithm the algorithm
- * @param args the args
- * @return Unique ID of the model created after training is complete
- * @throws LensException the lens exception
- */
- String train(String table, String algorithm, String[] args) throws LensException;
-
- /**
- * Get model IDs for the given algorithm.
- *
- * @param algorithm the algorithm
- * @return the models
- * @throws LensException the lens exception
- */
- List<String> getModels(String algorithm) throws LensException;
-
- /**
- * Get a model instance given the algorithm name and model ID.
- *
- * @param algorithm the algorithm
- * @param modelId the model id
- * @return the model
- * @throws LensException the lens exception
- */
- MLModel getModel(String algorithm, String modelId) throws LensException;
-
- /**
- * Get the FS location where model instance is saved.
- *
- * @param algorithm the algorithm
- * @param modelID the model id
- * @return the model path
- */
- String getModelPath(String algorithm, String modelID);
-
- /**
- * Evaluate model by running it against test data contained in the given table.
- *
- * @param session the session
- * @param table the table
- * @param algorithm the algorithm
- * @param modelID the model id
- * @return Test report object containing test output table, and various evaluation metrics
- * @throws LensException the lens exception
- */
- MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
- String outputTable) throws LensException;
-
- /**
- * Get test reports for an algorithm.
- *
- * @param algorithm the algorithm
- * @return the test reports
- * @throws LensException the lens exception
- */
- List<String> getTestReports(String algorithm) throws LensException;
-
- /**
- * Get a test report by ID.
- *
- * @param algorithm the algorithm
- * @param reportID the report id
- * @return the test report
- * @throws LensException the lens exception
- */
- MLTestReport getTestReport(String algorithm, String reportID) throws LensException;
-
- /**
- * Online predict call given a model ID, algorithm name and sample feature values.
- *
- * @param algorithm the algorithm
- * @param modelID the model id
- * @param features the features
- * @return prediction result
- * @throws LensException the lens exception
- */
- Object predict(String algorithm, String modelID, Object[] features) throws LensException;
-
- /**
- * Permanently delete a model instance.
- *
- * @param algorithm the algorithm
- * @param modelID the model id
- * @throws LensException the lens exception
- */
- void deleteModel(String algorithm, String modelID) throws LensException;
-
- /**
- * Permanently delete a test report instance.
- *
- * @param algorithm the algorithm
- * @param reportID the report id
- * @throws LensException the lens exception
- */
- void deleteTestReport(String algorithm, String reportID) throws LensException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
deleted file mode 100644
index b45f7f2..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
+++ /dev/null
@@ -1,734 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.Entity;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.MediaType;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.query.LensQuery;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.ml.spark.SparkMLDriver;
-import org.apache.lens.ml.spark.algos.BaseSparkAlgo;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.session.SessionService;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.spark.api.java.JavaSparkContext;
-
-import org.glassfish.jersey.media.multipart.FormDataBodyPart;
-import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.glassfish.jersey.media.multipart.FormDataMultiPart;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
-
-/**
- * The Class LensMLImpl.
- */
-public class LensMLImpl implements LensML {
-
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensMLImpl.class);
-
- /** The drivers. */
- protected List<MLDriver> drivers;
-
- /** The conf. */
- private HiveConf conf;
-
- /** The spark context. */
- private JavaSparkContext sparkContext;
-
- /** Check if the predict UDF has been registered for a user */
- private final Map<LensSessionHandle, Boolean> predictUdfStatus;
- /** Background thread to periodically check if we need to clear expire status for a session */
- private ScheduledExecutorService udfStatusExpirySvc;
-
- /**
- * Instantiates a new lens ml impl.
- *
- * @param conf the conf
- */
- public LensMLImpl(HiveConf conf) {
- this.conf = conf;
- this.predictUdfStatus = new ConcurrentHashMap<LensSessionHandle, Boolean>();
- }
-
- public HiveConf getConf() {
- return conf;
- }
-
- /**
- * Use an existing Spark context. Useful in case of
- *
- * @param jsc JavaSparkContext instance
- */
- public void setSparkContext(JavaSparkContext jsc) {
- this.sparkContext = jsc;
- }
-
- public List<String> getAlgorithms() {
- List<String> algos = new ArrayList<String>();
- for (MLDriver driver : drivers) {
- algos.addAll(driver.getAlgoNames());
- }
- return algos;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getAlgoForName(java.lang.String)
- */
- public MLAlgo getAlgoForName(String algorithm) throws LensException {
- for (MLDriver driver : drivers) {
- if (driver.isAlgoSupported(algorithm)) {
- return driver.getAlgoInstance(algorithm);
- }
- }
- throw new LensException("Algo not supported " + algorithm);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#train(java.lang.String, java.lang.String, java.lang.String[])
- */
- public String train(String table, String algorithm, String[] args) throws LensException {
- MLAlgo algo = getAlgoForName(algorithm);
-
- String modelId = UUID.randomUUID().toString();
-
- LOG.info("Begin training model " + modelId + ", algo=" + algorithm + ", table=" + table + ", params="
- + Arrays.toString(args));
-
- String database = null;
- if (SessionState.get() != null) {
- database = SessionState.get().getCurrentDatabase();
- } else {
- database = "default";
- }
-
- MLModel model = algo.train(toLensConf(conf), database, table, modelId, args);
-
- LOG.info("Done training model: " + modelId);
-
- model.setCreatedAt(new Date());
- model.setAlgoName(algorithm);
-
- Path modelLocation = null;
- try {
- modelLocation = persistModel(model);
- LOG.info("Model saved: " + modelId + ", algo: " + algorithm + ", path: " + modelLocation);
- return model.getId();
- } catch (IOException e) {
- throw new LensException("Error saving model " + modelId + " for algo " + algorithm, e);
- }
- }
-
- /**
- * Gets the algo dir.
- *
- * @param algoName the algo name
- * @return the algo dir
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private Path getAlgoDir(String algoName) throws IOException {
- String modelSaveBaseDir = conf.get(ModelLoader.MODEL_PATH_BASE_DIR, ModelLoader.MODEL_PATH_BASE_DIR_DEFAULT);
- return new Path(new Path(modelSaveBaseDir), algoName);
- }
-
- /**
- * Persist model.
- *
- * @param model the model
- * @return the path
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private Path persistModel(MLModel model) throws IOException {
- // Get model save path
- Path algoDir = getAlgoDir(model.getAlgoName());
- FileSystem fs = algoDir.getFileSystem(conf);
-
- if (!fs.exists(algoDir)) {
- fs.mkdirs(algoDir);
- }
-
- Path modelSavePath = new Path(algoDir, model.getId());
- ObjectOutputStream outputStream = null;
-
- try {
- outputStream = new ObjectOutputStream(fs.create(modelSavePath, false));
- outputStream.writeObject(model);
- outputStream.flush();
- } catch (IOException io) {
- LOG.error("Error saving model " + model.getId() + " reason: " + io.getMessage());
- throw io;
- } finally {
- IOUtils.closeQuietly(outputStream);
- }
- return modelSavePath;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getModels(java.lang.String)
- */
- public List<String> getModels(String algorithm) throws LensException {
- try {
- Path algoDir = getAlgoDir(algorithm);
- FileSystem fs = algoDir.getFileSystem(conf);
- if (!fs.exists(algoDir)) {
- return null;
- }
-
- List<String> models = new ArrayList<String>();
-
- for (FileStatus stat : fs.listStatus(algoDir)) {
- models.add(stat.getPath().getName());
- }
-
- if (models.isEmpty()) {
- return null;
- }
-
- return models;
- } catch (IOException ioex) {
- throw new LensException(ioex);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getModel(java.lang.String, java.lang.String)
- */
- public MLModel getModel(String algorithm, String modelId) throws LensException {
- try {
- return ModelLoader.loadModel(conf, algorithm, modelId);
- } catch (IOException e) {
- throw new LensException(e);
- }
- }
-
- /**
- * Inits the.
- *
- * @param hiveConf the hive conf
- */
- public synchronized void init(HiveConf hiveConf) {
- this.conf = hiveConf;
-
- // Get all the drivers
- String[] driverClasses = hiveConf.getStrings("lens.ml.drivers");
-
- if (driverClasses == null || driverClasses.length == 0) {
- throw new RuntimeException("No ML Drivers specified in conf");
- }
-
- LOG.info("Loading drivers " + Arrays.toString(driverClasses));
- drivers = new ArrayList<MLDriver>(driverClasses.length);
-
- for (String driverClass : driverClasses) {
- Class<?> cls;
- try {
- cls = Class.forName(driverClass);
- } catch (ClassNotFoundException e) {
- LOG.error("Driver class not found " + driverClass);
- continue;
- }
-
- if (!MLDriver.class.isAssignableFrom(cls)) {
- LOG.warn("Not a driver class " + driverClass);
- continue;
- }
-
- try {
- Class<? extends MLDriver> mlDriverClass = (Class<? extends MLDriver>) cls;
- MLDriver driver = mlDriverClass.newInstance();
- driver.init(toLensConf(conf));
- drivers.add(driver);
- LOG.info("Added driver " + driverClass);
- } catch (Exception e) {
- LOG.error("Failed to create driver " + driverClass + " reason: " + e.getMessage(), e);
- }
- }
- if (drivers.isEmpty()) {
- throw new RuntimeException("No ML drivers loaded");
- }
-
- LOG.info("Inited ML service");
- }
-
- /**
- * Start.
- */
- public synchronized void start() {
- for (MLDriver driver : drivers) {
- try {
- if (driver instanceof SparkMLDriver && sparkContext != null) {
- ((SparkMLDriver) driver).useSparkContext(sparkContext);
- }
- driver.start();
- } catch (LensException e) {
- LOG.error("Failed to start driver " + driver, e);
- }
- }
-
- udfStatusExpirySvc = Executors.newSingleThreadScheduledExecutor();
- udfStatusExpirySvc.scheduleAtFixedRate(new UDFStatusExpiryRunnable(), 60, 60, TimeUnit.SECONDS);
-
- LOG.info("Started ML service");
- }
-
- /**
- * Stop.
- */
- public synchronized void stop() {
- for (MLDriver driver : drivers) {
- try {
- driver.stop();
- } catch (LensException e) {
- LOG.error("Failed to stop driver " + driver, e);
- }
- }
- drivers.clear();
- udfStatusExpirySvc.shutdownNow();
- LOG.info("Stopped ML service");
- }
-
- public synchronized HiveConf getHiveConf() {
- return conf;
- }
-
- /**
- * Clear models.
- */
- public void clearModels() {
- ModelLoader.clearCache();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getModelPath(java.lang.String, java.lang.String)
- */
- public String getModelPath(String algorithm, String modelID) {
- return ModelLoader.getModelLocation(conf, algorithm, modelID).toString();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#testModel(org.apache.lens.api.LensSessionHandle, java.lang.String, java.lang.String,
- * java.lang.String)
- */
- @Override
- public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
- String outputTable) throws LensException {
- return null;
- }
-
- /**
- * Test a model in embedded mode.
- *
- * @param sessionHandle the session handle
- * @param table the table
- * @param algorithm the algorithm
- * @param modelID the model id
- * @param queryApiUrl the query api url
- * @return the ML test report
- * @throws LensException the lens exception
- */
- public MLTestReport testModelRemote(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
- String queryApiUrl, String outputTable) throws LensException {
- return testModel(sessionHandle, table, algorithm, modelID, new RemoteQueryRunner(sessionHandle, queryApiUrl),
- outputTable);
- }
-
- /**
- * Evaluate a model. Evaluation is done on data selected table from an input table. The model is run as a UDF and its
- * output is inserted into a table with a partition. Each evaluation is given a unique ID. The partition label is
- * associated with this unique ID.
- * <p/>
- * <p>
- * This call also required a query runner. Query runner is responsible for executing the evaluation query against Lens
- * server.
- * </p>
- *
- * @param sessionHandle the session handle
- * @param table the table
- * @param algorithm the algorithm
- * @param modelID the model id
- * @param queryRunner the query runner
- * @param outputTable table where test output will be written
- * @return the ML test report
- * @throws LensException the lens exception
- */
- public MLTestReport testModel(final LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
- QueryRunner queryRunner, String outputTable) throws LensException {
- if (sessionHandle == null) {
- throw new NullPointerException("Null session not allowed");
- }
- // check if algorithm exists
- if (!getAlgorithms().contains(algorithm)) {
- throw new LensException("No such algorithm " + algorithm);
- }
-
- MLModel<?> model;
- try {
- model = ModelLoader.loadModel(conf, algorithm, modelID);
- } catch (IOException e) {
- throw new LensException(e);
- }
-
- if (model == null) {
- throw new LensException("Model not found: " + modelID + " algorithm=" + algorithm);
- }
-
- String database = null;
-
- if (SessionState.get() != null) {
- database = SessionState.get().getCurrentDatabase();
- }
-
- String testID = UUID.randomUUID().toString().replace("-", "_");
- final String testTable = outputTable;
- final String testResultColumn = "prediction_result";
-
- // TODO support error metric UDAFs
- TableTestingSpec spec = TableTestingSpec.newBuilder().hiveConf(conf)
- .database(database == null ? "default" : database).inputTable(table).featureColumns(model.getFeatureColumns())
- .outputColumn(testResultColumn).lableColumn(model.getLabelColumn()).algorithm(algorithm).modelID(modelID)
- .outputTable(testTable).testID(testID).build();
-
- String testQuery = spec.getTestQuery();
- if (testQuery == null) {
- throw new LensException("Invalid test spec. " + "table=" + table + " algorithm=" + algorithm + " modelID="
- + modelID);
- }
-
- if (!spec.isOutputTableExists()) {
- LOG.info("Output table '" + testTable + "' does not exist for test algorithm = " + algorithm + " modelid="
- + modelID + ", Creating table using query: " + spec.getCreateOutputTableQuery());
- // create the output table
- String createOutputTableQuery = spec.getCreateOutputTableQuery();
- queryRunner.runQuery(createOutputTableQuery);
- LOG.info("Table created " + testTable);
- }
-
- // Check if ML UDF is registered in this session
- registerPredictUdf(sessionHandle, queryRunner);
-
- LOG.info("Running evaluation query " + testQuery);
- queryRunner.setQueryName("model_test_" + modelID);
- QueryHandle testQueryHandle = queryRunner.runQuery(testQuery);
-
- MLTestReport testReport = new MLTestReport();
- testReport.setReportID(testID);
- testReport.setAlgorithm(algorithm);
- testReport.setFeatureColumns(model.getFeatureColumns());
- testReport.setLabelColumn(model.getLabelColumn());
- testReport.setModelID(model.getId());
- testReport.setOutputColumn(testResultColumn);
- testReport.setOutputTable(testTable);
- testReport.setTestTable(table);
- testReport.setQueryID(testQueryHandle.toString());
-
- // Save test report
- persistTestReport(testReport);
- LOG.info("Saved test report " + testReport.getReportID());
- return testReport;
- }
-
- /**
- * Persist test report.
- *
- * @param testReport the test report
- * @throws LensException the lens exception
- */
- private void persistTestReport(MLTestReport testReport) throws LensException {
- LOG.info("saving test report " + testReport.getReportID());
- try {
- ModelLoader.saveTestReport(conf, testReport);
- LOG.info("Saved report " + testReport.getReportID());
- } catch (IOException e) {
- LOG.error("Error saving report " + testReport.getReportID() + " reason: " + e.getMessage());
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getTestReports(java.lang.String)
- */
- public List<String> getTestReports(String algorithm) throws LensException {
- Path reportBaseDir = new Path(conf.get(ModelLoader.TEST_REPORT_BASE_DIR, ModelLoader.TEST_REPORT_BASE_DIR_DEFAULT));
- FileSystem fs = null;
-
- try {
- fs = reportBaseDir.getFileSystem(conf);
- if (!fs.exists(reportBaseDir)) {
- return null;
- }
-
- Path algoDir = new Path(reportBaseDir, algorithm);
- if (!fs.exists(algoDir)) {
- return null;
- }
-
- List<String> reports = new ArrayList<String>();
- for (FileStatus stat : fs.listStatus(algoDir)) {
- reports.add(stat.getPath().getName());
- }
- return reports;
- } catch (IOException e) {
- LOG.error("Error reading report list for " + algorithm, e);
- return null;
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getTestReport(java.lang.String, java.lang.String)
- */
- public MLTestReport getTestReport(String algorithm, String reportID) throws LensException {
- try {
- return ModelLoader.loadReport(conf, algorithm, reportID);
- } catch (IOException e) {
- throw new LensException(e);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#predict(java.lang.String, java.lang.String, java.lang.Object[])
- */
- public Object predict(String algorithm, String modelID, Object[] features) throws LensException {
- // Load the model instance
- MLModel<?> model = getModel(algorithm, modelID);
- return model.predict(features);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#deleteModel(java.lang.String, java.lang.String)
- */
- public void deleteModel(String algorithm, String modelID) throws LensException {
- try {
- ModelLoader.deleteModel(conf, algorithm, modelID);
- LOG.info("DELETED model " + modelID + " algorithm=" + algorithm);
- } catch (IOException e) {
- LOG.error(
- "Error deleting model file. algorithm=" + algorithm + " model=" + modelID + " reason: " + e.getMessage(), e);
- throw new LensException("Unable to delete model " + modelID + " for algorithm " + algorithm, e);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#deleteTestReport(java.lang.String, java.lang.String)
- */
- public void deleteTestReport(String algorithm, String reportID) throws LensException {
- try {
- ModelLoader.deleteTestReport(conf, algorithm, reportID);
- LOG.info("DELETED report=" + reportID + " algorithm=" + algorithm);
- } catch (IOException e) {
- LOG.error("Error deleting report " + reportID + " algorithm=" + algorithm + " reason: " + e.getMessage(), e);
- throw new LensException("Unable to delete report " + reportID + " for algorithm " + algorithm, e);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getAlgoParamDescription(java.lang.String)
- */
- public Map<String, String> getAlgoParamDescription(String algorithm) {
- MLAlgo algo = null;
- try {
- algo = getAlgoForName(algorithm);
- } catch (LensException e) {
- LOG.error("Error getting algo description : " + algorithm, e);
- return null;
- }
- if (algo instanceof BaseSparkAlgo) {
- return ((BaseSparkAlgo) algo).getArgUsage();
- }
- return null;
- }
-
- /**
- * Submit model test query to a remote Lens server.
- */
- class RemoteQueryRunner extends QueryRunner {
-
- /** The query api url. */
- final String queryApiUrl;
-
- /**
- * Instantiates a new remote query runner.
- *
- * @param sessionHandle the session handle
- * @param queryApiUrl the query api url
- */
- public RemoteQueryRunner(LensSessionHandle sessionHandle, String queryApiUrl) {
- super(sessionHandle);
- this.queryApiUrl = queryApiUrl;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.TestQueryRunner#runQuery(java.lang.String)
- */
- @Override
- public QueryHandle runQuery(String query) throws LensException {
- // Create jersey client for query endpoint
- Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).build();
- WebTarget target = client.target(queryApiUrl);
- final FormDataMultiPart mp = new FormDataMultiPart();
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), sessionHandle,
- MediaType.APPLICATION_XML_TYPE));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), query));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
-
- LensConf lensConf = new LensConf();
- lensConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, false + "");
- lensConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false + "");
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), lensConf,
- MediaType.APPLICATION_XML_TYPE));
-
- final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- QueryHandle.class);
-
- LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", sessionHandle).request()
- .get(LensQuery.class);
-
- QueryStatus stat = ctx.getStatus();
- while (!stat.isFinished()) {
- ctx = target.path(handle.toString()).queryParam("sessionid", sessionHandle).request().get(LensQuery.class);
- stat = ctx.getStatus();
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- throw new LensException(e);
- }
- }
-
- if (stat.getStatus() != QueryStatus.Status.SUCCESSFUL) {
- throw new LensException("Query failed " + ctx.getQueryHandle().getHandleId() + " reason:"
- + stat.getErrorMessage());
- }
-
- return ctx.getQueryHandle();
- }
- }
-
- /**
- * To lens conf.
- *
- * @param conf the conf
- * @return the lens conf
- */
- private LensConf toLensConf(HiveConf conf) {
- LensConf lensConf = new LensConf();
- lensConf.getProperties().putAll(conf.getValByRegex(".*"));
- return lensConf;
- }
-
- protected void registerPredictUdf(LensSessionHandle sessionHandle, QueryRunner queryRunner) throws LensException {
- if (isUdfRegisterd(sessionHandle)) {
- // Already registered, nothing to do
- return;
- }
-
- LOG.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
- // We have to add UDF jars to the session
- try {
- SessionService sessionService = (SessionService) MLUtils.getServiceProvider().getService(SessionService.NAME);
- String[] udfJars = conf.getStrings("lens.server.ml.predict.udf.jars");
- if (udfJars != null) {
- for (String jar : udfJars) {
- sessionService.addResource(sessionHandle, "jar", jar);
- LOG.info(jar + " added UDF session " + sessionHandle.getPublicId().toString());
- }
- }
- } catch (Exception e) {
- throw new LensException(e);
- }
-
- String regUdfQuery = "CREATE TEMPORARY FUNCTION " + HiveMLUDF.UDF_NAME + " AS '" + HiveMLUDF.class
- .getCanonicalName() + "'";
- queryRunner.setQueryName("register_predict_udf_" + sessionHandle.getPublicId().toString());
- QueryHandle udfQuery = queryRunner.runQuery(regUdfQuery);
- predictUdfStatus.put(sessionHandle, true);
- LOG.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
- }
-
- protected boolean isUdfRegisterd(LensSessionHandle sessionHandle) {
- return predictUdfStatus.containsKey(sessionHandle);
- }
-
- /**
- * Periodically check if sessions have been closed, and clear UDF registered status.
- */
- private class UDFStatusExpiryRunnable implements Runnable {
- public void run() {
- try {
- SessionService sessionService = (SessionService) MLUtils.getServiceProvider().getService(SessionService.NAME);
- // Clear status of sessions which are closed.
- List<LensSessionHandle> sessions = new ArrayList<LensSessionHandle>(predictUdfStatus.keySet());
- for (LensSessionHandle sessionHandle : sessions) {
- if (!sessionService.isOpen(sessionHandle)) {
- LOG.info("Session closed, removing UDF status: " + sessionHandle);
- predictUdfStatus.remove(sessionHandle);
- }
- }
- } catch (Exception exc) {
- LOG.warn("Error clearing UDF statuses", exc);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/MLAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLAlgo.java
deleted file mode 100644
index 7dccf2c..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLAlgo.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensException;
-
-/**
- * The Interface MLAlgo.
- */
-public interface MLAlgo {
- String getName();
-
- String getDescription();
-
- /**
- * Configure.
- *
- * @param configuration the configuration
- */
- void configure(LensConf configuration);
-
- LensConf getConf();
-
- /**
- * Train.
- *
- * @param conf the conf
- * @param db the db
- * @param table the table
- * @param modelId the model id
- * @param params the params
- * @return the ML model
- * @throws LensException the lens exception
- */
- MLModel train(LensConf conf, String db, String table, String modelId, String... params) throws LensException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/MLDriver.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLDriver.java
deleted file mode 100644
index 567e717..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLDriver.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.util.List;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensException;
-
-/**
- * The Interface MLDriver.
- */
-public interface MLDriver {
-
- /**
- * Checks if is algo supported.
- *
- * @param algo the algo
- * @return true, if is algo supported
- */
- boolean isAlgoSupported(String algo);
-
- /**
- * Gets the algo instance.
- *
- * @param algo the algo
- * @return the algo instance
- * @throws LensException the lens exception
- */
- MLAlgo getAlgoInstance(String algo) throws LensException;
-
- /**
- * Inits the.
- *
- * @param conf the conf
- * @throws LensException the lens exception
- */
- void init(LensConf conf) throws LensException;
-
- /**
- * Start.
- *
- * @throws LensException the lens exception
- */
- void start() throws LensException;
-
- /**
- * Stop.
- *
- * @throws LensException the lens exception
- */
- void stop() throws LensException;
-
- List<String> getAlgoNames();
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
deleted file mode 100644
index c177757..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.io.Serializable;
-import java.util.Date;
-import java.util.List;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.ToString;
-
-/**
- * Instantiates a new ML model.
- */
-@NoArgsConstructor
-@ToString
-public abstract class MLModel<PREDICTION> implements Serializable {
-
- /** The id. */
- @Getter
- @Setter
- private String id;
-
- /** The created at. */
- @Getter
- @Setter
- private Date createdAt;
-
- /** The algo name. */
- @Getter
- @Setter
- private String algoName;
-
- /** The table. */
- @Getter
- @Setter
- private String table;
-
- /** The params. */
- @Getter
- @Setter
- private List<String> params;
-
- /** The label column. */
- @Getter
- @Setter
- private String labelColumn;
-
- /** The feature columns. */
- @Getter
- @Setter
- private List<String> featureColumns;
-
- /**
- * Predict.
- *
- * @param args the args
- * @return the prediction
- */
- public abstract PREDICTION predict(Object... args);
-}