You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:49:58 UTC
[24/50] [abbrv] incubator-lens git commit: Lens-465 : Refactor ml
packages. (sharad)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/LogitRegressionClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/LogitRegressionClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/LogitRegressionClassificationModel.java
deleted file mode 100644
index 1c5152b..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/LogitRegressionClassificationModel.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.spark.models;
-
-import org.apache.spark.mllib.classification.LogisticRegressionModel;
-
-/**
- * The Class LogitRegressionClassificationModel.
- */
-public class LogitRegressionClassificationModel extends BaseSparkClassificationModel<LogisticRegressionModel> {
-
- /**
- * Instantiates a new logit regression classification model.
- *
- * @param modelId the model id
- * @param model the model
- */
- public LogitRegressionClassificationModel(String modelId, LogisticRegressionModel model) {
- super(modelId, model);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/NaiveBayesClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/NaiveBayesClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/NaiveBayesClassificationModel.java
deleted file mode 100644
index 8f4552c..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/NaiveBayesClassificationModel.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.spark.models;
-
-import org.apache.spark.mllib.classification.NaiveBayesModel;
-
-/**
- * The Class NaiveBayesClassificationModel.
- */
-public class NaiveBayesClassificationModel extends BaseSparkClassificationModel<NaiveBayesModel> {
-
- /**
- * Instantiates a new naive bayes classification model.
- *
- * @param modelId the model id
- * @param model the model
- */
- public NaiveBayesClassificationModel(String modelId, NaiveBayesModel model) {
- super(modelId, model);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SVMClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SVMClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SVMClassificationModel.java
deleted file mode 100644
index 4e504fb..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SVMClassificationModel.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.spark.models;
-
-import org.apache.spark.mllib.classification.SVMModel;
-
-/**
- * The Class SVMClassificationModel.
- */
-public class SVMClassificationModel extends BaseSparkClassificationModel<SVMModel> {
-
- /**
- * Instantiates a new SVM classification model.
- *
- * @param modelId the model id
- * @param model the model
- */
- public SVMClassificationModel(String modelId, SVMModel model) {
- super(modelId, model);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SparkDecisionTreeModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SparkDecisionTreeModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SparkDecisionTreeModel.java
deleted file mode 100644
index 657070b..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/models/SparkDecisionTreeModel.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.spark.models;
-
-import org.apache.lens.ml.spark.DoubleValueMapper;
-
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.mllib.classification.ClassificationModel;
-import org.apache.spark.mllib.linalg.Vector;
-import org.apache.spark.mllib.tree.model.DecisionTreeModel;
-import org.apache.spark.rdd.RDD;
-
-/**
- * This class is created because the Spark decision tree model doesn't extend ClassificationModel.
- */
-public class SparkDecisionTreeModel implements ClassificationModel {
-
- /** The model. */
- private final DecisionTreeModel model;
-
- /**
- * Instantiates a new spark decision tree model.
- *
- * @param model the model
- */
- public SparkDecisionTreeModel(DecisionTreeModel model) {
- this.model = model;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.spark.mllib.classification.ClassificationModel#predict(org.apache.spark.rdd.RDD)
- */
- @Override
- public RDD<Object> predict(RDD<Vector> testData) {
- return model.predict(testData);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.spark.mllib.classification.ClassificationModel#predict(org.apache.spark.mllib.linalg.Vector)
- */
- @Override
- public double predict(Vector testData) {
- return model.predict(testData);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.spark.mllib.classification.ClassificationModel#predict(org.apache.spark.api.java.JavaRDD)
- */
- @Override
- public JavaRDD<Double> predict(JavaRDD<Vector> testData) {
- return model.predict(testData.rdd()).toJavaRDD().map(new DoubleValueMapper());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
deleted file mode 100644
index e4bb329..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.task;
-
-import java.util.*;
-
-import org.apache.lens.client.LensMLClient;
-import org.apache.lens.ml.LensML;
-import org.apache.lens.ml.MLTestReport;
-import org.apache.lens.ml.MLUtils;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import lombok.Getter;
-import lombok.ToString;
-
-/**
- * Run a complete cycle of train and test (evaluation) for an ML algorithm
- */
-@ToString
-public class MLTask implements Runnable {
- private static final Log LOG = LogFactory.getLog(MLTask.class);
-
- public enum State {
- RUNNING, SUCCESSFUL, FAILED
- }
-
- @Getter
- private State taskState;
-
- /**
- * Name of the algo/algorithm.
- */
- @Getter
- private String algorithm;
-
- /**
- * Name of the table containing training data.
- */
- @Getter
- private String trainingTable;
-
- /**
- * Name of the table containing test data. Optional, if not provided trainingTable itself is
- * used for testing
- */
- @Getter
- private String testTable;
-
- /**
- * Training table partition spec
- */
- @Getter
- private String partitionSpec;
-
- /**
- * Name of the column which is a label for supervised algorithms.
- */
- @Getter
- private String labelColumn;
-
- /**
- * Names of columns which are features in the training data.
- */
- @Getter
- private List<String> featureColumns;
-
- /**
- * Configuration for the example.
- */
- @Getter
- private HiveConf configuration;
-
- private LensML ml;
- private String taskID;
-
- /**
- * ml client
- */
- @Getter
- private LensMLClient mlClient;
-
- /**
- * Output table name
- */
- @Getter
- private String outputTable;
-
- /**
- * Extra params passed to the training algorithm
- */
- @Getter
- private Map<String, String> extraParams;
-
- @Getter
- private String modelID;
-
- @Getter
- private String reportID;
-
- /**
- * Use ExampleTask.Builder to create an instance
- */
- private MLTask() {
- // Use builder to construct the example
- extraParams = new HashMap<String, String>();
- taskID = UUID.randomUUID().toString();
- }
-
- /**
- * Builder to create an example task
- */
- public static class Builder {
- private MLTask task;
-
- public Builder() {
- task = new MLTask();
- }
-
- public Builder trainingTable(String trainingTable) {
- task.trainingTable = trainingTable;
- return this;
- }
-
- public Builder testTable(String testTable) {
- task.testTable = testTable;
- return this;
- }
-
- public Builder algorithm(String algorithm) {
- task.algorithm = algorithm;
- return this;
- }
-
- public Builder labelColumn(String labelColumn) {
- task.labelColumn = labelColumn;
- return this;
- }
-
- public Builder client(LensMLClient client) {
- task.mlClient = client;
- return this;
- }
-
- public Builder addFeatureColumn(String featureColumn) {
- if (task.featureColumns == null) {
- task.featureColumns = new ArrayList<String>();
- }
- task.featureColumns.add(featureColumn);
- return this;
- }
-
- public Builder hiveConf(HiveConf hiveConf) {
- task.configuration = hiveConf;
- return this;
- }
-
-
-
- public Builder extraParam(String param, String value) {
- task.extraParams.put(param, value);
- return this;
- }
-
- public Builder partitionSpec(String partitionSpec) {
- task.partitionSpec = partitionSpec;
- return this;
- }
-
- public Builder outputTable(String outputTable) {
- task.outputTable = outputTable;
- return this;
- }
-
- public MLTask build() {
- MLTask builtTask = task;
- task = null;
- return builtTask;
- }
-
- }
-
- @Override
- public void run() {
- taskState = State.RUNNING;
- LOG.info("Starting " + taskID);
- try {
- runTask();
- taskState = State.SUCCESSFUL;
- LOG.info("Complete " + taskID);
- } catch (Exception e) {
- taskState = State.FAILED;
- LOG.info("Error running task " + taskID, e);
- }
- }
-
- /**
- * Train an ML model, with specified algorithm and input data. Do model evaluation using the evaluation data and print
- * evaluation result
- *
- * @throws Exception
- */
- private void runTask() throws Exception {
- if (mlClient != null) {
- // Connect to a remote Lens server
- ml = mlClient;
- LOG.info("Working in client mode. Lens session handle " + mlClient.getSessionHandle().getPublicId());
- } else {
- // In server mode session handle has to be passed by the user as a request parameter
- ml = MLUtils.getMLService();
- LOG.info("Working in Lens server");
- }
-
- String[] algoArgs = buildTrainingArgs();
- LOG.info("Starting task " + taskID + " algo args: " + Arrays.toString(algoArgs));
-
- modelID = ml.train(trainingTable, algorithm, algoArgs);
- printModelMetadata(taskID, modelID);
-
- LOG.info("Starting test " + taskID);
- testTable = (testTable != null) ? testTable : trainingTable;
- MLTestReport testReport = ml.testModel(mlClient.getSessionHandle(), testTable, algorithm, modelID, outputTable);
- reportID = testReport.getReportID();
- printTestReport(taskID, testReport);
- saveTask();
- }
-
- // Save task metadata to DB
- private void saveTask() {
- LOG.info("Saving task details to DB");
- }
-
- private void printTestReport(String exampleID, MLTestReport testReport) {
- StringBuilder builder = new StringBuilder("Example: ").append(exampleID);
- builder.append("\n\t");
- builder.append("EvaluationReport: ").append(testReport.toString());
- System.out.println(builder.toString());
- }
-
- private String[] buildTrainingArgs() {
- List<String> argList = new ArrayList<String>();
- argList.add("label");
- argList.add(labelColumn);
-
- // Add all the features
- for (String featureCol : featureColumns) {
- argList.add("feature");
- argList.add(featureCol);
- }
-
- // Add extra params
- for (String param : extraParams.keySet()) {
- argList.add(param);
- argList.add(extraParams.get(param));
- }
-
- return argList.toArray(new String[argList.size()]);
- }
-
- // Get the model instance and print its metadat to stdout
- private void printModelMetadata(String exampleID, String modelID) throws Exception {
- StringBuilder builder = new StringBuilder("Example: ").append(exampleID);
- builder.append("\n\t");
- builder.append("Model: ");
- builder.append(ml.getModel(algorithm, modelID).toString());
- System.out.println(builder.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
index 2c2d28b..ac89eee 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
@@ -29,7 +29,7 @@ import org.apache.lens.api.LensException;
import org.apache.lens.api.query.*;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientResultSet;
-import org.apache.lens.ml.spark.HiveTableRDD;
+import org.apache.lens.ml.algo.spark.HiveTableRDD;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
deleted file mode 100644
index 75d4f03..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.ml;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import javax.ws.rs.ApplicationPath;
-import javax.ws.rs.core.Application;
-
-import org.glassfish.jersey.filter.LoggingFilter;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
-
-@ApplicationPath("/ml")
-public class MLApp extends Application {
-
- private final Set<Class<?>> classes;
-
- /**
- * Pass additional classes when running in test mode
- *
- * @param additionalClasses
- */
- public MLApp(Class<?>... additionalClasses) {
- classes = new HashSet<Class<?>>();
-
- // register root resource
- classes.add(MLServiceResource.class);
- classes.add(MultiPartFeature.class);
- classes.add(LoggingFilter.class);
- for (Class<?> cls : additionalClasses) {
- classes.add(cls);
- }
-
- }
-
- /**
- * Get classes for this resource
- */
- @Override
- public Set<Class<?>> getClasses() {
- return classes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLService.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLService.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLService.java
deleted file mode 100644
index 0dac605..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLService.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.ml;
-
-import org.apache.lens.ml.LensML;
-
-/**
- * The Interface MLService.
- */
-public interface MLService extends LensML {
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
deleted file mode 100644
index 0e8e9aa..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.ml;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.query.LensQuery;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.ml.*;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.ServiceProvider;
-import org.apache.lens.server.api.ServiceProviderFactory;
-import org.apache.lens.server.api.query.QueryExecutionService;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hive.service.CompositeService;
-
-/**
- * The Class MLServiceImpl.
- */
-public class MLServiceImpl extends CompositeService implements MLService {
-
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensMLImpl.class);
-
- /** The ml. */
- private LensMLImpl ml;
-
- /** The service provider. */
- private ServiceProvider serviceProvider;
-
- /** The service provider factory. */
- private ServiceProviderFactory serviceProviderFactory;
-
- /**
- * Instantiates a new ML service impl.
- */
- public MLServiceImpl() {
- this(NAME);
- }
-
- /**
- * Instantiates a new ML service impl.
- *
- * @param name the name
- */
- public MLServiceImpl(String name) {
- super(name);
- }
-
- @Override
- public List<String> getAlgorithms() {
- return ml.getAlgorithms();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getAlgoForName(java.lang.String)
- */
- @Override
- public MLAlgo getAlgoForName(String algorithm) throws LensException {
- return ml.getAlgoForName(algorithm);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#train(java.lang.String, java.lang.String, java.lang.String[])
- */
- @Override
- public String train(String table, String algorithm, String[] args) throws LensException {
- return ml.train(table, algorithm, args);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getModels(java.lang.String)
- */
- @Override
- public List<String> getModels(String algorithm) throws LensException {
- return ml.getModels(algorithm);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getModel(java.lang.String, java.lang.String)
- */
- @Override
- public MLModel getModel(String algorithm, String modelId) throws LensException {
- return ml.getModel(algorithm, modelId);
- }
-
- private ServiceProvider getServiceProvider() {
- if (serviceProvider == null) {
- serviceProvider = serviceProviderFactory.getServiceProvider();
- }
- return serviceProvider;
- }
-
- /**
- * Gets the service provider factory.
- *
- * @param conf the conf
- * @return the service provider factory
- */
- private ServiceProviderFactory getServiceProviderFactory(HiveConf conf) {
- Class<?> spfClass = conf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY, ServiceProviderFactory.class);
- try {
- return (ServiceProviderFactory) spfClass.newInstance();
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hive.service.CompositeService#init(org.apache.hadoop.hive.conf.HiveConf)
- */
- @Override
- public synchronized void init(HiveConf hiveConf) {
- ml = new LensMLImpl(hiveConf);
- ml.init(hiveConf);
- super.init(hiveConf);
- serviceProviderFactory = getServiceProviderFactory(hiveConf);
- LOG.info("Inited ML service");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hive.service.CompositeService#start()
- */
- @Override
- public synchronized void start() {
- ml.start();
- super.start();
- LOG.info("Started ML service");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hive.service.CompositeService#stop()
- */
- @Override
- public synchronized void stop() {
- ml.stop();
- super.stop();
- LOG.info("Stopped ML service");
- }
-
- /**
- * Clear models.
- */
- public void clearModels() {
- ModelLoader.clearCache();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getModelPath(java.lang.String, java.lang.String)
- */
- @Override
- public String getModelPath(String algorithm, String modelID) {
- return ml.getModelPath(algorithm, modelID);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#testModel(org.apache.lens.api.LensSessionHandle, java.lang.String, java.lang.String,
- * java.lang.String)
- */
- @Override
- public MLTestReport testModel(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
- String outputTable) throws LensException {
- return ml.testModel(sessionHandle, table, algorithm, modelID, new DirectQueryRunner(sessionHandle), outputTable);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getTestReports(java.lang.String)
- */
- @Override
- public List<String> getTestReports(String algorithm) throws LensException {
- return ml.getTestReports(algorithm);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getTestReport(java.lang.String, java.lang.String)
- */
- @Override
- public MLTestReport getTestReport(String algorithm, String reportID) throws LensException {
- return ml.getTestReport(algorithm, reportID);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#predict(java.lang.String, java.lang.String, java.lang.Object[])
- */
- @Override
- public Object predict(String algorithm, String modelID, Object[] features) throws LensException {
- return ml.predict(algorithm, modelID, features);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#deleteModel(java.lang.String, java.lang.String)
- */
- @Override
- public void deleteModel(String algorithm, String modelID) throws LensException {
- ml.deleteModel(algorithm, modelID);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#deleteTestReport(java.lang.String, java.lang.String)
- */
- @Override
- public void deleteTestReport(String algorithm, String reportID) throws LensException {
- ml.deleteTestReport(algorithm, reportID);
- }
-
- /**
- * Run the test model query directly in the current lens server process.
- */
- private class DirectQueryRunner extends QueryRunner {
-
- /**
- * Instantiates a new direct query runner.
- *
- * @param sessionHandle the session handle
- */
- public DirectQueryRunner(LensSessionHandle sessionHandle) {
- super(sessionHandle);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.TestQueryRunner#runQuery(java.lang.String)
- */
- @Override
- public QueryHandle runQuery(String testQuery) throws LensException {
- FunctionRegistry.registerTemporaryFunction("predict", HiveMLUDF.class);
- LOG.info("Registered predict UDF");
- // Run the query in query executions service
- QueryExecutionService queryService = (QueryExecutionService) getServiceProvider().getService("query");
-
- LensConf queryConf = new LensConf();
- queryConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, false + "");
- queryConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false + "");
-
- QueryHandle testQueryHandle = queryService.executeAsync(sessionHandle, testQuery, queryConf, queryName);
-
- // Wait for test query to complete
- LensQuery query = queryService.getQuery(sessionHandle, testQueryHandle);
- LOG.info("Submitted query " + testQueryHandle.getHandleId());
- while (!query.getStatus().isFinished()) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- throw new LensException(e);
- }
-
- query = queryService.getQuery(sessionHandle, testQueryHandle);
- }
-
- if (query.getStatus().getStatus() != QueryStatus.Status.SUCCESSFUL) {
- throw new LensException("Failed to run test query: " + testQueryHandle.getHandleId() + " reason= "
- + query.getStatus().getErrorMessage());
- }
-
- return testQueryHandle;
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.LensML#getAlgoParamDescription(java.lang.String)
- */
- @Override
- public Map<String, String> getAlgoParamDescription(String algorithm) {
- return ml.getAlgoParamDescription(algorithm);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
deleted file mode 100644
index c0b32d3..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
+++ /dev/null
@@ -1,415 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.ml;
-
-import static org.apache.commons.lang.StringUtils.isBlank;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.*;
-
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.StringList;
-import org.apache.lens.api.ml.ModelMetadata;
-import org.apache.lens.api.ml.TestReport;
-import org.apache.lens.ml.MLModel;
-import org.apache.lens.ml.MLTestReport;
-import org.apache.lens.ml.ModelLoader;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.ServiceProvider;
-import org.apache.lens.server.api.ServiceProviderFactory;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import org.glassfish.jersey.media.multipart.FormDataParam;
-
-/**
- * Machine Learning service.
- */
-@Path("/ml")
-@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
-public class MLServiceResource {
-
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(MLServiceResource.class);
-
- /** The ml service. */
- MLService mlService;
-
- /** The service provider. */
- ServiceProvider serviceProvider;
-
- /** The service provider factory. */
- ServiceProviderFactory serviceProviderFactory;
-
- private static final HiveConf HIVE_CONF;
-
- /**
- * Message indicating if ML service is up
- */
- public static final String ML_UP_MESSAGE = "ML service is up";
-
- static {
- HIVE_CONF = new HiveConf();
- // Add default config so that we know the service provider implementation
- HIVE_CONF.addResource("lensserver-default.xml");
- HIVE_CONF.addResource("lens-site.xml");
- }
-
- /**
- * Instantiates a new ML service resource.
- */
- public MLServiceResource() {
- serviceProviderFactory = getServiceProviderFactory(HIVE_CONF);
- }
-
- private ServiceProvider getServiceProvider() {
- if (serviceProvider == null) {
- serviceProvider = serviceProviderFactory.getServiceProvider();
- }
- return serviceProvider;
- }
-
- /**
- * Gets the service provider factory.
- *
- * @param conf the conf
- * @return the service provider factory
- */
- private ServiceProviderFactory getServiceProviderFactory(HiveConf conf) {
- Class<?> spfClass = conf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY, ServiceProviderFactory.class);
- try {
- return (ServiceProviderFactory) spfClass.newInstance();
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- private MLService getMlService() {
- if (mlService == null) {
- mlService = (MLService) getServiceProvider().getService(MLService.NAME);
- }
- return mlService;
- }
-
- /**
- * Indicates if ML resource is up
- *
- * @return
- */
- @GET
- public String mlResourceUp() {
- return ML_UP_MESSAGE;
- }
-
- /**
- * Get a list of algos available
- *
- * @return
- */
- @GET
- @Path("algos")
- public StringList getAlgoNames() {
- List<String> algos = getMlService().getAlgorithms();
- StringList result = new StringList(algos);
- return result;
- }
-
- /**
- * Gets the human readable param description of an algorithm
- *
- * @param algorithm the algorithm
- * @return the param description
- */
- @GET
- @Path("algos/{algorithm}")
- public StringList getParamDescription(@PathParam("algorithm") String algorithm) {
- Map<String, String> paramDesc = getMlService().getAlgoParamDescription(algorithm);
- if (paramDesc == null) {
- throw new NotFoundException("Param description not found for " + algorithm);
- }
-
- List<String> descriptions = new ArrayList<String>();
- for (String key : paramDesc.keySet()) {
- descriptions.add(key + " : " + paramDesc.get(key));
- }
- return new StringList(descriptions);
- }
-
- /**
- * Get model ID list for a given algorithm.
- *
- * @param algorithm algorithm name
- * @return the models for algo
- * @throws LensException the lens exception
- */
- @GET
- @Path("models/{algorithm}")
- public StringList getModelsForAlgo(@PathParam("algorithm") String algorithm) throws LensException {
- List<String> models = getMlService().getModels(algorithm);
- if (models == null || models.isEmpty()) {
- throw new NotFoundException("No models found for algorithm " + algorithm);
- }
- return new StringList(models);
- }
-
- /**
- * Get metadata of the model given algorithm and model ID.
- *
- * @param algorithm algorithm name
- * @param modelID model ID
- * @return model metadata
- * @throws LensException the lens exception
- */
- @GET
- @Path("models/{algorithm}/{modelID}")
- public ModelMetadata getModelMetadata(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID)
- throws LensException {
- MLModel model = getMlService().getModel(algorithm, modelID);
- if (model == null) {
- throw new NotFoundException("Model not found " + modelID + ", algo=" + algorithm);
- }
-
- ModelMetadata meta = new ModelMetadata(model.getId(), model.getTable(), model.getAlgoName(), StringUtils.join(
- model.getParams(), ' '), model.getCreatedAt().toString(), getMlService().getModelPath(algorithm, modelID),
- model.getLabelColumn(), StringUtils.join(model.getFeatureColumns(), ","));
- return meta;
- }
-
- /**
- * Delete a model given model ID and algorithm name.
- *
- * @param algorithm the algorithm
- * @param modelID the model id
- * @return confirmation text
- * @throws LensException the lens exception
- */
- @DELETE
- @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
- @Path("models/{algorithm}/{modelID}")
- public String deleteModel(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID)
- throws LensException {
- getMlService().deleteModel(algorithm, modelID);
- return "DELETED model=" + modelID + " algorithm=" + algorithm;
- }
-
- /**
- * Train a model given an algorithm name and algorithm parameters
- * <p>
- * Following parameters are mandatory and must be passed as part of the form
- * <p/>
- * <ol>
- * <li>table - input Hive table to load training data from</li>
- * <li>label - name of the labelled column</li>
- * <li>feature - one entry per feature column. At least one feature column is required</li>
- * </ol>
- * <p/>
- * </p>
- *
- * @param algorithm algorithm name
- * @param form form data
- * @return if model is successfully trained, the model ID will be returned
- * @throws LensException the lens exception
- */
- @POST
- @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
- @Path("{algorithm}/train")
- public String train(@PathParam("algorithm") String algorithm, MultivaluedMap<String, String> form)
- throws LensException {
-
- // Check if algo is valid
- if (getMlService().getAlgoForName(algorithm) == null) {
- throw new NotFoundException("Algo for algo: " + algorithm + " not found");
- }
-
- if (isBlank(form.getFirst("table"))) {
- throw new BadRequestException("table parameter is rquired");
- }
-
- String table = form.getFirst("table");
-
- if (isBlank(form.getFirst("label"))) {
- throw new BadRequestException("label parameter is required");
- }
-
- // Check features
- List<String> featureNames = form.get("feature");
- if (featureNames.size() < 1) {
- throw new BadRequestException("At least one feature is required");
- }
-
- List<String> algoArgs = new ArrayList<String>();
- Set<Map.Entry<String, List<String>>> paramSet = form.entrySet();
-
- for (Map.Entry<String, List<String>> e : paramSet) {
- String p = e.getKey();
- List<String> values = e.getValue();
- if ("algorithm".equals(p) || "table".equals(p)) {
- continue;
- } else if ("feature".equals(p)) {
- for (String feature : values) {
- algoArgs.add("feature");
- algoArgs.add(feature);
- }
- } else if ("label".equals(p)) {
- algoArgs.add("label");
- algoArgs.add(values.get(0));
- } else {
- algoArgs.add(p);
- algoArgs.add(values.get(0));
- }
- }
- LOG.info("Training table " + table + " with algo " + algorithm + " params=" + algoArgs.toString());
- String modelId = getMlService().train(table, algorithm, algoArgs.toArray(new String[]{}));
- LOG.info("Done training " + table + " modelid = " + modelId);
- return modelId;
- }
-
- /**
- * Clear model cache (for admin use).
- *
- * @return OK if the cache was cleared
- */
- @DELETE
- @Path("clearModelCache")
- @Produces(MediaType.TEXT_PLAIN)
- public Response clearModelCache() {
- ModelLoader.clearCache();
- LOG.info("Cleared model cache");
- return Response.ok("Cleared cache", MediaType.TEXT_PLAIN_TYPE).build();
- }
-
- /**
- * Run a test on a model for an algorithm.
- *
- * @param algorithm algorithm name
- * @param modelID model ID
- * @param table Hive table to run test on
- * @param session Lens session ID. This session ID will be used to run the test query
- * @return Test report ID
- * @throws LensException the lens exception
- */
- @POST
- @Path("test/{table}/{algorithm}/{modelID}")
- @Consumes(MediaType.MULTIPART_FORM_DATA)
- public String test(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID,
- @PathParam("table") String table, @FormDataParam("sessionid") LensSessionHandle session,
- @FormDataParam("outputTable") String outputTable) throws LensException {
- MLTestReport testReport = getMlService().testModel(session, table, algorithm, modelID, outputTable);
- return testReport.getReportID();
- }
-
- /**
- * Get list of reports for a given algorithm.
- *
- * @param algoritm the algoritm
- * @return the reports for algorithm
- * @throws LensException the lens exception
- */
- @GET
- @Path("reports/{algorithm}")
- public StringList getReportsForAlgorithm(@PathParam("algorithm") String algoritm) throws LensException {
- List<String> reports = getMlService().getTestReports(algoritm);
- if (reports == null || reports.isEmpty()) {
- throw new NotFoundException("No test reports found for " + algoritm);
- }
- return new StringList(reports);
- }
-
- /**
- * Get a single test report given the algorithm name and report id.
- *
- * @param algorithm the algorithm
- * @param reportID the report id
- * @return the test report
- * @throws LensException the lens exception
- */
- @GET
- @Path("reports/{algorithm}/{reportID}")
- public TestReport getTestReport(@PathParam("algorithm") String algorithm, @PathParam("reportID") String reportID)
- throws LensException {
- MLTestReport report = getMlService().getTestReport(algorithm, reportID);
-
- if (report == null) {
- throw new NotFoundException("Test report: " + reportID + " not found for algorithm " + algorithm);
- }
-
- TestReport result = new TestReport(report.getTestTable(), report.getOutputTable(), report.getOutputColumn(),
- report.getLabelColumn(), StringUtils.join(report.getFeatureColumns(), ","), report.getAlgorithm(),
- report.getModelID(), report.getReportID(), report.getLensQueryID());
- return result;
- }
-
- /**
- * DELETE a report given the algorithm name and report ID.
- *
- * @param algorithm the algorithm
- * @param reportID the report id
- * @return the string
- * @throws LensException the lens exception
- */
- @DELETE
- @Path("reports/{algorithm}/{reportID}")
- @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
- public String deleteTestReport(@PathParam("algorithm") String algorithm, @PathParam("reportID") String reportID)
- throws LensException {
- getMlService().deleteTestReport(algorithm, reportID);
- return "DELETED report=" + reportID + " algorithm=" + algorithm;
- }
-
- /**
- * Predict.
- *
- * @param algorithm the algorithm
- * @param modelID the model id
- * @param uriInfo the uri info
- * @return the string
- * @throws LensException the lens exception
- */
- @GET
- @Path("/predict/{algorithm}/{modelID}")
- @Produces({MediaType.APPLICATION_ATOM_XML, MediaType.APPLICATION_JSON})
- public String predict(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID,
- @Context UriInfo uriInfo) throws LensException {
- // Load the model instance
- MLModel<?> model = getMlService().getModel(algorithm, modelID);
-
- // Get input feature names
- MultivaluedMap<String, String> params = uriInfo.getQueryParameters();
- String[] features = new String[model.getFeatureColumns().size()];
- // Assuming that feature name parameters are same
- int i = 0;
- for (String feature : model.getFeatureColumns()) {
- features[i++] = params.getFirst(feature);
- }
-
- // TODO needs a 'prediction formatter'
- return getMlService().predict(algorithm, modelID, features).toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
new file mode 100644
index 0000000..9fe1ea0
--- /dev/null
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+/**
+ * The Class ExampleUtils.
+ */
+public final class ExampleUtils {
+ private ExampleUtils() {
+ }
+
+ private static final Log LOG = LogFactory.getLog(ExampleUtils.class);
+
+ /**
+ * Creates the example table.
+ *
+ * @param conf the conf
+ * @param database the database
+ * @param tableName the table name
+ * @param sampleDataFile the sample data file
+ * @param labelColumn the label column
+ * @param features the features
+ * @throws HiveException the hive exception
+ */
+ public static void createTable(HiveConf conf, String database, String tableName, String sampleDataFile,
+ String labelColumn, Map<String, String> tableParams, String... features) throws HiveException {
+
+ Path dataFilePath = new Path(sampleDataFile);
+ Path partDir = dataFilePath.getParent();
+
+ // Create table
+ List<FieldSchema> columns = new ArrayList<FieldSchema>();
+
+ // Label is optional. Not used for unsupervised models.
+ // If present, label will be the first column, followed by features
+ if (labelColumn != null) {
+ columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
+ }
+
+ for (String feature : features) {
+ columns.add(new FieldSchema(feature, "double", "Feature " + feature));
+ }
+
+ Table tbl = Hive.get(conf).newTable(database + "." + tableName);
+ tbl.setTableType(TableType.MANAGED_TABLE);
+ tbl.getTTable().getSd().setCols(columns);
+ tbl.getTTable().getParameters().putAll(tableParams);
+ tbl.setInputFormatClass(TextInputFormat.class);
+ tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
+ tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");
+
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
+ partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
+ tbl.setPartCols(partCols);
+
+ Hive.get(conf).createTable(tbl, false);
+ LOG.info("Created table " + tableName);
+
+ // Add partition for the data file
+ AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName, false);
+ Map<String, String> partSpec = new HashMap<String, String>();
+ partSpec.put("dummy_partition_col", "dummy_val");
+ partitionDesc.addPartition(partSpec, partDir.toUri().toString());
+ Hive.get(conf).createPartitions(partitionDesc);
+ LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index f712481..8b7e3f3 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -33,15 +33,16 @@ import javax.ws.rs.core.UriBuilder;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensMLClient;
-import org.apache.lens.ml.spark.algos.DecisionTreeAlgo;
-import org.apache.lens.ml.spark.algos.LogisticRegressionAlgo;
-import org.apache.lens.ml.spark.algos.NaiveBayesAlgo;
-import org.apache.lens.ml.spark.algos.SVMAlgo;
-import org.apache.lens.ml.task.MLTask;
+import org.apache.lens.ml.algo.spark.dt.DecisionTreeAlgo;
+import org.apache.lens.ml.algo.spark.lr.LogisticRegressionAlgo;
+import org.apache.lens.ml.algo.spark.nb.NaiveBayesAlgo;
+import org.apache.lens.ml.algo.spark.svm.SVMAlgo;
+import org.apache.lens.ml.impl.MLTask;
+import org.apache.lens.ml.impl.MLUtils;
+import org.apache.lens.ml.server.MLApp;
+import org.apache.lens.ml.server.MLServiceResource;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.ml.MLApp;
-import org.apache.lens.server.ml.MLServiceResource;
import org.apache.lens.server.query.QueryServiceResource;
import org.apache.lens.server.session.SessionResource;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
index d7f2f8f..655b55e 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
@@ -26,23 +26,24 @@ import javax.ws.rs.core.UriBuilder;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensMLClient;
-import org.apache.lens.ml.task.MLTask;
+import org.apache.lens.ml.impl.MLRunner;
+import org.apache.lens.ml.impl.MLTask;
+import org.apache.lens.ml.server.MLApp;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.metastore.MetastoreResource;
-import org.apache.lens.server.ml.MLApp;
import org.apache.lens.server.query.QueryServiceResource;
import org.apache.lens.server.session.SessionResource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
+
import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/lens-ml-lib/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml
index 9ce4703..2e1ddab 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -103,7 +103,7 @@
<property>
<name>lens.server.ml.ws.resource.impl</name>
- <value>org.apache.lens.server.ml.MLServiceResource</value>
+ <value>org.apache.lens.ml.server.MLServiceResource</value>
<description>Implementation class for ML Service Resource</description>
</property>
@@ -138,13 +138,13 @@
<property>
<name>lens.server.ml.service.impl</name>
- <value>org.apache.lens.server.ml.MLServiceImpl</value>
+ <value>org.apache.lens.ml.server.MLServiceImpl</value>
<description>Implementation class for ML service</description>
</property>
<property>
<name>lens.ml.drivers</name>
- <value>org.apache.lens.ml.spark.SparkMLDriver</value>
+ <value>org.apache.lens.ml.algo.spark.SparkMLDriver</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f5ea4c7/tools/conf-pseudo-distr/server/lens-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/lens-site.xml b/tools/conf-pseudo-distr/server/lens-site.xml
index f11c6d6..ce3e753 100644
--- a/tools/conf-pseudo-distr/server/lens-site.xml
+++ b/tools/conf-pseudo-distr/server/lens-site.xml
@@ -59,19 +59,19 @@
<property>
<name>lens.server.ml.ws.resource.impl</name>
- <value>org.apache.lens.server.ml.MLServiceResource</value>
+ <value>org.apache.lens.ml.server.MLServiceResource</value>
<description>Implementation class for ML Service Resource</description>
</property>
<property>
<name>lens.server.ml.service.impl</name>
- <value>org.apache.lens.server.ml.MLServiceImpl</value>
+ <value>org.apache.lens.ml.server.MLServiceImpl</value>
<description>Implementation class for ML service</description>
</property>
<property>
<name>lens.ml.drivers</name>
- <value>org.apache.lens.ml.spark.SparkMLDriver</value>
+ <value>org.apache.lens.ml.algo.spark.SparkMLDriver</value>
</property>
<property>