You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/04/16 07:28:44 UTC
incubator-lens git commit: LENS-436 : Fix dependencies in ML
distribution
Repository: incubator-lens
Updated Branches:
refs/heads/master 120bf6cd7 -> 001970de5
LENS-436 : Fix dependencies in ML distribution
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/001970de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/001970de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/001970de
Branch: refs/heads/master
Commit: 001970de5ff73bb6effba630c474550b286b2a3b
Parents: 120bf6c
Author: Sharad Agarwal <sh...@apache.org>
Authored: Thu Apr 16 10:58:18 2015 +0530
Committer: jdhok <jd...@apache.org>
Committed: Thu Apr 16 10:58:18 2015 +0530
----------------------------------------------------------------------
lens-dist/src/main/assembly/bin-dist.xml | 1 +
lens-docker/lens-test/Dockerfile | 7 ++++
lens-docker/lens-test/lens-bootstrap.sh | 10 ++++-
lens-ml-dist/pom.xml | 40 ++++++++++++++++++++
lens-ml-dist/src/main/assembly/ml-dist.xml | 2 +-
.../org/apache/lens/ml/impl/LensMLImpl.java | 14 +------
.../apache/lens/ml/server/MLServiceImpl.java | 4 --
lens-ml-lib/src/test/resources/lens-site.xml | 4 --
pom.xml | 2 +-
.../server/hivedriver-site.xml | 4 ++
10 files changed, 63 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-dist/src/main/assembly/bin-dist.xml
----------------------------------------------------------------------
diff --git a/lens-dist/src/main/assembly/bin-dist.xml b/lens-dist/src/main/assembly/bin-dist.xml
index b7b74d5..9e2ce8f 100644
--- a/lens-dist/src/main/assembly/bin-dist.xml
+++ b/lens-dist/src/main/assembly/bin-dist.xml
@@ -118,6 +118,7 @@
<include>lens-config.sh</include>
<include>lens-cli.sh</include>
<include>lens-cli</include>
+ <include>lens-run-class.sh</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-docker/lens-test/Dockerfile
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/Dockerfile b/lens-docker/lens-test/Dockerfile
index c4886c2..a405b0f 100644
--- a/lens-docker/lens-test/Dockerfile
+++ b/lens-docker/lens-test/Dockerfile
@@ -17,6 +17,13 @@
FROM inmobi/docker-hive
+RUN wget http://apache.mirrors.lucidnetworks.net/spark/spark-1.3.0/spark-1.3.0-bin-hadoop2.4.tgz
+RUN gunzip spark-1.3.0-bin-hadoop2.4.tgz
+RUN tar -xvf spark-1.3.0-bin-hadoop2.4.tar
+RUN mv spark-1.3.0-bin-hadoop2.4 /usr/local
+ENV SPARK_HOME /usr/local/spark-1.3.0-bin-hadoop2.4
+RUN rm spark-1.3.0-bin-hadoop2.4.tar
+
ENV LENS_VERSION 2.1.0-beta-incubating-SNAPSHOT
ENV BASEDIR /opt/lens
ENV LENS_HOME $BASEDIR/lens-dist/target/apache-lens-${LENS_VERSION}-bin/apache-lens-${LENS_VERSION}-bin/server
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 19685bb..d4935be 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -22,11 +22,17 @@ echo "LENS_SERVER_CONF " $LENS_SERVER_CONF
echo "LENS_CLIENT " $LENS_CLIENT
echo "LENS_CLIENT_CONF " $LENS_CLIENT_CONF
echo "LENS_ML " $LENS_ML
+echo "SPARK_HOME " $SPARK_HOME
#set ml classpath into LENS_EXT_CLASSPATH
LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
export LENS_EXT_CLASSPATH
+HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.1.0-beta-incubating-SNAPSHOT.jar,$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar
+export HIVE_AUX_JARS_PATH
+
+echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH
+
#start hive bootstrap script
/etc/hive-bootstrap.sh
@@ -37,8 +43,8 @@ sleep 20
echo "Starting Lens server..."
$LENS_HOME/bin/lens-ctl start --conf $LENS_SERVER_CONF
-echo "Waiting for 40 secs for Lens Server to start ..."
-sleep 40
+echo "Waiting for 60 secs for Lens Server to start ..."
+sleep 60
#Setting up client
$LENS_CLIENT/bin/run-examples.sh sample-metastore --conf $LENS_CLIENT_CONF
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index 9e4fd1d..7b6f760 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -37,6 +37,46 @@
<groupId>org.apache.lens</groupId>
<artifactId>lens-ml-lib</artifactId>
<version>${project.version}</version>
+ <scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-server-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-multipart</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.test-framework.providers</groupId>
+ <artifactId>jersey-test-framework-provider-grizzly2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.test-framework</groupId>
+ <artifactId>jersey-test-framework-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
+
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-dist/src/main/assembly/ml-dist.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/src/main/assembly/ml-dist.xml b/lens-ml-dist/src/main/assembly/ml-dist.xml
index 30b6f23..50c7375 100644
--- a/lens-ml-dist/src/main/assembly/ml-dist.xml
+++ b/lens-ml-dist/src/main/assembly/ml-dist.xml
@@ -36,7 +36,7 @@
<scope>runtime</scope>
<outputDirectory>lib</outputDirectory>
<includes>
- <include>org.apache.lens:lens-ml</include>
+ <include>org.apache.lens:lens-ml-lib</include>
</includes>
<useTransitiveFiltering>true</useTransitiveFiltering>
<useTransitiveDependencies>true</useTransitiveDependencies>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
index 39617df..aff7c84 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
@@ -695,24 +695,12 @@ public class LensMLImpl implements LensML {
}
LOG.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
- // We have to add UDF jars to the session
- try {
- SessionService sessionService = (SessionService) MLUtils.getServiceProvider().getService(SessionService.NAME);
- String[] udfJars = conf.getStrings("lens.server.ml.predict.udf.jars");
- if (udfJars != null) {
- for (String jar : udfJars) {
- sessionService.addResource(sessionHandle, "jar", jar);
- LOG.info(jar + " added UDF session " + sessionHandle.getPublicId().toString());
- }
- }
- } catch (Exception e) {
- throw new LensException(e);
- }
String regUdfQuery = "CREATE TEMPORARY FUNCTION " + HiveMLUDF.UDF_NAME + " AS '" + HiveMLUDF.class
.getCanonicalName() + "'";
queryRunner.setQueryName("register_predict_udf_" + sessionHandle.getPublicId().toString());
QueryHandle udfQuery = queryRunner.runQuery(regUdfQuery);
+ LOG.info("udf query handle is " + udfQuery);
predictUdfStatus.put(sessionHandle, true);
LOG.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
index dc29aef..264ca51 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
@@ -29,7 +29,6 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.ml.algo.api.*;
import org.apache.lens.ml.api.MLTestReport;
-import org.apache.lens.ml.impl.HiveMLUDF;
import org.apache.lens.ml.impl.LensMLImpl;
import org.apache.lens.ml.impl.ModelLoader;
import org.apache.lens.ml.impl.QueryRunner;
@@ -41,7 +40,6 @@ import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hive.service.CompositeService;
/**
@@ -284,8 +282,6 @@ public class MLServiceImpl extends CompositeService implements MLService {
*/
@Override
public QueryHandle runQuery(String testQuery) throws LensException {
- FunctionRegistry.registerTemporaryFunction("predict", HiveMLUDF.class);
- LOG.info("Registered predict UDF");
// Run the query in query executions service
QueryExecutionService queryService = (QueryExecutionService) getServiceProvider().getService("query");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-lib/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml
index 2e1ddab..3d5dbef 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -159,8 +159,4 @@
<description>Implementation class for session service</description>
</property>
- <property>
- <name>lens.server.ml.predict.udf.jars</name>
- <value>testjars/serde.jar,testjars/test.jar</value>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d05e55a..66f7824 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
<!-- hadoop stack -->
<hadoop.version>2.4.0</hadoop.version>
- <spark.version>1.1.0</spark.version>
+ <spark.version>1.3.0</spark.version>
<hive.version>0.13.3-inm</hive.version>
<!-- common library/framework versions -->
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/tools/conf-pseudo-distr/server/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/hivedriver-site.xml b/tools/conf-pseudo-distr/server/hivedriver-site.xml
index 0d14777..ec5954c 100644
--- a/tools/conf-pseudo-distr/server/hivedriver-site.xml
+++ b/tools/conf-pseudo-distr/server/hivedriver-site.xml
@@ -50,4 +50,8 @@
<value>local,cluster</value>
<final>true</final>
</property>
+<property>
+ <name>hive.aux.jars.path</name>
+ <value>file:///opt/lens/lens-ml-dist/target/apache-lens-2.1.0-beta-incubating-SNAPSHOT-ml/lib/lens-ml-lib-2.1.0-beta-incubating-SNAPSHOT.jar,file:///usr/local/spark-1.3.0-bin-hadoop2.4/lib/spark-assembly-1.3.0-hadoop2.4.0.jar</value>
+</property>
</configuration>