You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/03/24 09:57:44 UTC
incubator-lens git commit: LENS-153 : Add examples execution to ML
(Sharad Agarwal via jdhok)
Repository: incubator-lens
Updated Branches:
refs/heads/master fc26d9144 -> 8d185913e
LENS-153 : Add examples execution to ML (Sharad Agarwal via jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/8d185913
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/8d185913
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/8d185913
Branch: refs/heads/master
Commit: 8d185913e76d632027028b3bbba15ceb8e158a9f
Parents: fc26d91
Author: Sharad Agarwal <sh...@apache.org>
Authored: Tue Mar 24 14:27:04 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Tue Mar 24 14:27:26 2015 +0530
----------------------------------------------------------------------
lens-docker/lens-test/Dockerfile | 8 +-
lens-docker/lens-test/lens-bootstrap.sh | 5 +
lens-ml-dist/LICENSE.txt | 201 +++++++++++++++++++
lens-ml-dist/pom.xml | 77 +++++++
lens-ml-dist/src/main/assembly/ml-dist.xml | 65 ++++++
lens-ml-dist/src/site/apt/index.apt | 20 ++
lens-ml-lib/data/naive_bayes/ml.properties | 7 +
.../data/naive_bayes/naive_bayes_train.data | 6 -
lens-ml-lib/data/naive_bayes/test.data | 6 +
lens-ml-lib/data/naive_bayes/train.data | 6 +
.../org/apache/lens/client/LensMLClient.java | 33 +--
.../main/java/org/apache/lens/ml/MLRunner.java | 173 ++++++++++++++++
.../java/org/apache/lens/ml/task/MLTask.java | 73 +++----
.../java/org/apache/lens/ml/TestMLResource.java | 130 ++++++------
.../java/org/apache/lens/ml/TestMLRunner.java | 138 +++++++++++++
lens-ml-lib/tools/scripts/lens-ml-classpath.sh | 52 +++++
pom.xml | 2 +
.../client/lens-client-site.xml | 5 +
tools/conf-pseudo-distr/server/lens-site.xml | 35 ++++
tools/scripts/lens-config.sh | 4 +
tools/scripts/lens-run-class.sh | 170 ++++++++++++++++
21 files changed, 1079 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-docker/lens-test/Dockerfile
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/Dockerfile b/lens-docker/lens-test/Dockerfile
index d26facd..c4886c2 100644
--- a/lens-docker/lens-test/Dockerfile
+++ b/lens-docker/lens-test/Dockerfile
@@ -17,14 +17,16 @@
FROM inmobi/docker-hive
-ENV LENS_VERSION apache-lens-2.1.0-beta-incubating-SNAPSHOT
+ENV LENS_VERSION 2.1.0-beta-incubating-SNAPSHOT
ENV BASEDIR /opt/lens
-ENV LENS_HOME $BASEDIR/lens-dist/target/${LENS_VERSION}-bin/${LENS_VERSION}-bin/server
-ENV LENS_CLIENT $BASEDIR/lens-dist/target/${LENS_VERSION}-bin/${LENS_VERSION}-bin/client
+ENV LENS_HOME $BASEDIR/lens-dist/target/apache-lens-${LENS_VERSION}-bin/apache-lens-${LENS_VERSION}-bin/server
+ENV LENS_CLIENT $BASEDIR/lens-dist/target/apache-lens-${LENS_VERSION}-bin/apache-lens-${LENS_VERSION}-bin/client
ENV LENS_SERVER_CONF $LENS_HOME/conf-pseudo-distr/
ENV LENS_CLIENT_CONF $LENS_CLIENT/conf-pseudo-distr/
+ENV LENS_ML $BASEDIR/lens-ml-dist/target/apache-lens-${LENS_VERSION}-ml/
+
# set permissions for lens bootstrap file
ADD lens-bootstrap.sh /etc/lens-bootstrap.sh
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 1c61603..19685bb 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -21,6 +21,11 @@ echo "LENS_HOME " $LENS_HOME
echo "LENS_SERVER_CONF " $LENS_SERVER_CONF
echo "LENS_CLIENT " $LENS_CLIENT
echo "LENS_CLIENT_CONF " $LENS_CLIENT_CONF
+echo "LENS_ML " $LENS_ML
+
+#set ml classpath into LENS_EXT_CLASSPATH
+LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
+export LENS_EXT_CLASSPATH
#start hive bootstrap script
/etc/hive-bootstrap.sh
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-dist/LICENSE.txt
----------------------------------------------------------------------
diff --git a/lens-ml-dist/LICENSE.txt b/lens-ml-dist/LICENSE.txt
new file mode 100644
index 0000000..261eeb9
--- /dev/null
+++ b/lens-ml-dist/LICENSE.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
new file mode 100644
index 0000000..9e4fd1d
--- /dev/null
+++ b/lens-ml-dist/pom.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <name>Lens ML Ext Distribution</name>
+
+ <parent>
+ <artifactId>apache-lens</artifactId>
+ <groupId>org.apache.lens</groupId>
+ <version>2.1.0-beta-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>lens-ml-dist</artifactId>
+ <description> Packaging and distribution for lens ml </description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-ml-lib</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <finalName>apache-lens-${project.version}</finalName>
+ <ignoreDirFormatExtensions>true</ignoreDirFormatExtensions>
+ <descriptors>
+ <descriptor>src/main/assembly/ml-dist.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assemble</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <!--Plugin to build deb-->
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.7</version>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-dist/src/main/assembly/ml-dist.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/src/main/assembly/ml-dist.xml b/lens-ml-dist/src/main/assembly/ml-dist.xml
new file mode 100644
index 0000000..30b6f23
--- /dev/null
+++ b/lens-ml-dist/src/main/assembly/ml-dist.xml
@@ -0,0 +1,65 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>ml</id>
+
+ <formats>
+ <format>dir</format>
+ </formats>
+
+ <baseDirectory>apache-lens-${project.version}-ml</baseDirectory>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <unpack>false</unpack>
+ <scope>runtime</scope>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>org.apache.lens:lens-ml</include>
+ </includes>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <fileMode>755</fileMode>
+ <directory>../lens-ml-lib/tools/scripts</directory>
+ <outputDirectory>/bin/</outputDirectory>
+ <includes>
+ <include>**</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>../lens-ml-lib/data</directory>
+ <outputDirectory>/data/</outputDirectory>
+ <includes>
+ <include>**</include>
+ </includes>
+ </fileSet>
+
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-dist/src/site/apt/index.apt
----------------------------------------------------------------------
diff --git a/lens-ml-dist/src/site/apt/index.apt b/lens-ml-dist/src/site/apt/index.apt
new file mode 100644
index 0000000..b45f4f8
--- /dev/null
+++ b/lens-ml-dist/src/site/apt/index.apt
@@ -0,0 +1,20 @@
+~~
+~~ Licensed to the Apache Software Foundation (ASF) under one
+~~ or more contributor license agreements. See the NOTICE file
+~~ distributed with this work for additional information
+~~ regarding copyright ownership. The ASF licenses this file
+~~ to you under the Apache License, Version 2.0 (the
+~~ "License"); you may not use this file except in compliance
+~~ with the License. You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing,
+~~ software distributed under the License is distributed on an
+~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+~~ KIND, either express or implied. See the License for the
+~~ specific language governing permissions and limitations
+~~ under the License.
+~~
+
+Lens Distribution Documentation
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/data/naive_bayes/ml.properties
----------------------------------------------------------------------
diff --git a/lens-ml-lib/data/naive_bayes/ml.properties b/lens-ml-lib/data/naive_bayes/ml.properties
new file mode 100644
index 0000000..61b156e
--- /dev/null
+++ b/lens-ml-lib/data/naive_bayes/ml.properties
@@ -0,0 +1,7 @@
+algo=spark_naive_bayes
+database=default
+traintable=naive_bayes_train
+testtable=naive_bayes_test
+outputtable=naive_bayes_output
+features=feature1,feature2,feature3
+labelcolumn=label
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/data/naive_bayes/naive_bayes_train.data
----------------------------------------------------------------------
diff --git a/lens-ml-lib/data/naive_bayes/naive_bayes_train.data b/lens-ml-lib/data/naive_bayes/naive_bayes_train.data
deleted file mode 100644
index dfb9ccd..0000000
--- a/lens-ml-lib/data/naive_bayes/naive_bayes_train.data
+++ /dev/null
@@ -1,6 +0,0 @@
-0 1 0 0
-0 2 0 0
-1 0 1 0
-1 0 2 0
-2 0 0 1
-2 0 0 2
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/data/naive_bayes/test.data
----------------------------------------------------------------------
diff --git a/lens-ml-lib/data/naive_bayes/test.data b/lens-ml-lib/data/naive_bayes/test.data
new file mode 100644
index 0000000..0677fe0
--- /dev/null
+++ b/lens-ml-lib/data/naive_bayes/test.data
@@ -0,0 +1,6 @@
+1 0 0
+2 0 0
+0 1 0
+0 2 0
+0 0 1
+0 0 2
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/data/naive_bayes/train.data
----------------------------------------------------------------------
diff --git a/lens-ml-lib/data/naive_bayes/train.data b/lens-ml-lib/data/naive_bayes/train.data
new file mode 100644
index 0000000..dfb9ccd
--- /dev/null
+++ b/lens-ml-lib/data/naive_bayes/train.data
@@ -0,0 +1,6 @@
+0 1 0 0
+0 2 0 0
+1 0 1 0
+1 0 2 0
+2 0 0 1
+2 0 0 2
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
index 9f7fa26..d9ec314 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
@@ -54,24 +54,25 @@ public class LensMLClient implements LensML, Closeable {
/** The client. */
private LensMLJerseyClient client;
- /**
- * Instantiates a new lens ml client.
- *
- * @param clientConf the client conf
- */
- public LensMLClient(LensConnectionParams clientConf, String password) {
- client = new LensMLJerseyClient(new LensConnection(clientConf), password);
- LOG.info("Client created with new session");
+ public LensMLClient(String password) {
+ this(new LensClientConfig(), password);
}
- /**
- * Instantiates a new lens ml client.
- *
- * @param clientConf the client conf
- */
- public LensMLClient(LensConnectionParams clientConf, LensSessionHandle sessionHandle) {
- client = new LensMLJerseyClient(new LensConnection(clientConf, sessionHandle), sessionHandle);
- LOG.info("Client created with existing session");
+ public LensMLClient(LensClientConfig conf, String password) {
+ this(conf, conf.getUser(), password);
+ }
+
+ public LensMLClient(String username, String password) {
+ this(new LensClientConfig(), username, password);
+ }
+
+ public LensMLClient(LensClientConfig conf, String username, String password) {
+ this(new LensClient(conf, username, password));
+ }
+
+ public LensMLClient(LensClient lensClient) {
+ client = new LensMLJerseyClient(lensClient.getConnection(), lensClient
+ .getConnection().getSessionHandle());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/src/main/java/org/apache/lens/ml/MLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLRunner.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLRunner.java
new file mode 100644
index 0000000..bd50cba
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLRunner.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.lens.client.LensClient;
+import org.apache.lens.client.LensClientConfig;
+import org.apache.lens.client.LensMLClient;
+import org.apache.lens.ml.task.MLTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+public class MLRunner {
+
+ private static final Log LOG = LogFactory.getLog(MLRunner.class);
+
+ private LensMLClient mlClient;
+ private String algoName;
+ private String database;
+ private String trainTable;
+ private String trainFile;
+ private String testTable;
+ private String testFile;
+ private String outputTable;
+ private String[] features;
+ private String labelColumn;
+ private HiveConf conf;
+
+ public void init(LensMLClient mlClient, String confDir) throws Exception {
+ File dir = new File(confDir);
+ File propFile = new File(dir, "ml.properties");
+ Properties props = new Properties();
+ props.load(new FileInputStream(propFile));
+ String feat = props.getProperty("features");
+ String trainFile = confDir + File.separator + "train.data";
+ String testFile = confDir + File.separator + "test.data";
+ init(mlClient, props.getProperty("algo"), props.getProperty("database"),
+ props.getProperty("traintable"), trainFile,
+ props.getProperty("testtable"), testFile,
+ props.getProperty("outputtable"), feat.split(","),
+ props.getProperty("labelcolumn"));
+ }
+
+ public void init(LensMLClient mlClient, String algoName,
+ String database, String trainTable, String trainFile,
+ String testTable, String testFile, String outputTable, String[] features,
+ String labelColumn) {
+ this.mlClient = mlClient;
+ this.algoName = algoName;
+ this.database = database;
+ this.trainTable = trainTable;
+ this.trainFile = trainFile;
+ this.testTable = testTable;
+ this.testFile = testFile;
+ this.outputTable = outputTable;
+ this.features = features;
+ this.labelColumn = labelColumn;
+ //hive metastore settings are loaded via lens-site.xml, so loading LensClientConfig
+ //is required
+ this.conf = new HiveConf(new LensClientConfig(), MLRunner.class);
+ }
+
+ public MLTask train() throws Exception {
+ LOG.info("Starting train & eval");
+
+ createTable(trainTable, trainFile);
+ createTable(testTable, testFile);
+ MLTask.Builder taskBuilder = new MLTask.Builder();
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
+ .client(mlClient).trainingTable(trainTable).testTable(testTable);
+
+ // Add features
+ for (String feature : features) {
+ taskBuilder.addFeatureColumn(feature);
+ }
+ MLTask task = taskBuilder.build();
+ LOG.info("Created task " + task.toString());
+ task.run();
+ return task;
+ }
+
+ public void createTable(String tableName, String dataFile) throws HiveException {
+
+ File filedataFile = new File(dataFile);
+ Path dataFilePath = new Path(filedataFile.toURI());
+ Path partDir = dataFilePath.getParent();
+
+ // Create table
+ List<FieldSchema> columns = new ArrayList<FieldSchema>();
+
+ // Label is optional. Not used for unsupervised models.
+ // If present, label will be the first column, followed by features
+ if (labelColumn != null) {
+ columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
+ }
+
+ for (String feature : features) {
+ columns.add(new FieldSchema(feature, "double", "Feature " + feature));
+ }
+
+ Table tbl = Hive.get(conf).newTable(database + "." + tableName);
+ tbl.setTableType(TableType.MANAGED_TABLE);
+ tbl.getTTable().getSd().setCols(columns);
+ // tbl.getTTable().getParameters().putAll(new HashMap<String, String>());
+ tbl.setInputFormatClass(TextInputFormat.class);
+ tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
+ tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");
+
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
+ partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
+ tbl.setPartCols(partCols);
+
+ Hive.get(conf).dropTable(database, tableName, false, true);
+ Hive.get(conf).createTable(tbl, true);
+ LOG.info("Created table " + tableName);
+
+ // Add partition for the data file
+ AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName,
+ false);
+ Map<String, String> partSpec = new HashMap<String, String>();
+ partSpec.put("dummy_partition_col", "dummy_val");
+ partitionDesc.addPartition(partSpec, partDir.toUri().toString());
+ Hive.get(conf).createPartitions(partitionDesc);
+ LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1) {
+ System.out.println("Usage: org.apache.lens.ml.MLRunner <ml-conf-dir>");
+ System.exit(-1);
+ }
+ String confDir = args[0];
+ LensMLClient client = new LensMLClient(new LensClient());
+ MLRunner runner = new MLRunner();
+ runner.init(client, confDir);
+ runner.train();
+ System.out.println("Created the Model successfully. Output Table: " + runner.outputTable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
index aa59100..e4bb329 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
@@ -20,8 +20,6 @@ package org.apache.lens.ml.task;
import java.util.*;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.client.LensConnectionParams;
import org.apache.lens.client.LensMLClient;
import org.apache.lens.ml.LensML;
import org.apache.lens.ml.MLTestReport;
@@ -61,6 +59,13 @@ public class MLTask implements Runnable {
private String trainingTable;
/**
+ * Name of the table containing test data. Optional, if not provided trainingTable itself is
+ * used for testing
+ */
+ @Getter
+ private String testTable;
+
+ /**
* Training table partition spec
*/
@Getter
@@ -84,26 +89,20 @@ public class MLTask implements Runnable {
@Getter
private HiveConf configuration;
- /**
- * Lens Server base URL, when running example as a client.
- */
- @Getter
- private String serverLocation;
-
private LensML ml;
private String taskID;
/**
- * Output table name
+ * ml client
*/
@Getter
- private String outputTable;
+ private LensMLClient mlClient;
/**
- * Session handle
+ * Output table name
*/
@Getter
- private LensSessionHandle sessionHandle;
+ private String outputTable;
/**
* Extra params passed to the training algorithm
@@ -111,18 +110,6 @@ public class MLTask implements Runnable {
@Getter
private Map<String, String> extraParams;
- /**
- * User name to connect to Lens server
- */
- @Getter
- private String userName;
-
- /**
- * Password to connect to Lens server
- */
- @Getter
- private String password;
-
@Getter
private String modelID;
@@ -153,6 +140,11 @@ public class MLTask implements Runnable {
return this;
}
+ public Builder testTable(String testTable) {
+ task.testTable = testTable;
+ return this;
+ }
+
public Builder algorithm(String algorithm) {
task.algorithm = algorithm;
return this;
@@ -163,6 +155,11 @@ public class MLTask implements Runnable {
return this;
}
+ public Builder client(LensMLClient client) {
+ task.mlClient = client;
+ return this;
+ }
+
public Builder addFeatureColumn(String featureColumn) {
if (task.featureColumns == null) {
task.featureColumns = new ArrayList<String>();
@@ -176,15 +173,7 @@ public class MLTask implements Runnable {
return this;
}
- public Builder serverLocation(String serverLocation) {
- task.serverLocation = serverLocation;
- return this;
- }
- public Builder sessionHandle(LensSessionHandle sessionHandle) {
- task.sessionHandle = sessionHandle;
- return this;
- }
public Builder extraParam(String param, String value) {
task.extraParams.put(param, value);
@@ -207,15 +196,6 @@ public class MLTask implements Runnable {
return builtTask;
}
- public Builder userName(String userName) {
- task.userName = userName;
- return this;
- }
-
- public Builder password(String password) {
- task.password = password;
- return this;
- }
}
@Override
@@ -239,14 +219,10 @@ public class MLTask implements Runnable {
* @throws Exception
*/
private void runTask() throws Exception {
- if (serverLocation != null) {
+ if (mlClient != null) {
// Connect to a remote Lens server
- LensConnectionParams connectionParams = new LensConnectionParams();
- connectionParams.setBaseUrl(serverLocation);
- connectionParams.getConf().setUser(userName);
- LensMLClient mlClient = new LensMLClient(connectionParams, sessionHandle);
ml = mlClient;
- LOG.info("Working in client mode. Lens session handle " + sessionHandle.getPublicId());
+ LOG.info("Working in client mode. Lens session handle " + mlClient.getSessionHandle().getPublicId());
} else {
// In server mode session handle has to be passed by the user as a request parameter
ml = MLUtils.getMLService();
@@ -260,7 +236,8 @@ public class MLTask implements Runnable {
printModelMetadata(taskID, modelID);
LOG.info("Starting test " + taskID);
- MLTestReport testReport = ml.testModel(sessionHandle, trainingTable, algorithm, modelID, outputTable);
+ testTable = (testTable != null) ? testTable : trainingTable;
+ MLTestReport testReport = ml.testModel(mlClient.getSessionHandle(), testTable, algorithm, modelID, outputTable);
reportID = testReport.getReportID();
printTestReport(taskID, testReport);
saveTask();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index 1d40b76..f712481 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -20,13 +20,18 @@ package org.apache.lens.ml;
import java.io.File;
import java.net.URI;
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Application;
+import javax.ws.rs.core.UriBuilder;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.client.LensConnectionParams;
+import org.apache.lens.client.LensClient;
+import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensMLClient;
import org.apache.lens.ml.spark.algos.DecisionTreeAlgo;
import org.apache.lens.ml.spark.algos.LogisticRegressionAlgo;
@@ -34,29 +39,21 @@ import org.apache.lens.ml.spark.algos.NaiveBayesAlgo;
import org.apache.lens.ml.spark.algos.SVMAlgo;
import org.apache.lens.ml.task.MLTask;
import org.apache.lens.server.LensJerseyTest;
-import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.ServiceProvider;
-import org.apache.lens.server.api.ServiceProviderFactory;
-import org.apache.lens.server.api.session.SessionService;
import org.apache.lens.server.ml.MLApp;
-import org.apache.lens.server.ml.MLService;
-import org.apache.lens.server.ml.MLServiceImpl;
import org.apache.lens.server.ml.MLServiceResource;
import org.apache.lens.server.query.QueryServiceResource;
-import org.apache.lens.server.session.HiveSessionService;
import org.apache.lens.server.session.SessionResource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hive.service.Service;
-
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.testng.Assert;
@@ -65,33 +62,19 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+
+
@Test
public class TestMLResource extends LensJerseyTest {
private static final Log LOG = LogFactory.getLog(TestMLResource.class);
- private static final String TEST_CONN_URL = "http://localhost:8089/lens-server";
- private static final LensConnectionParams LENS_CONNECTION_PARAMS = new LensConnectionParams();
-
- static {
- LENS_CONNECTION_PARAMS.setBaseUrl(TEST_CONN_URL);
- LENS_CONNECTION_PARAMS.getConf().setUser("foo@localhost");
- }
+ private static final String TEST_DB = "default";
private WebTarget mlTarget;
private LensMLClient mlClient;
- private ServiceProvider serviceProvider;
- private LensSessionHandle sessionHandle;
-
- public void setServiceProvider() throws Exception {
- HiveConf conf = LensServerConf.get();
- Class<? extends ServiceProviderFactory> spfClass = conf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY, null,
- ServiceProviderFactory.class);
- ServiceProviderFactory spf = spfClass.newInstance();
- this.serviceProvider = spf.getServiceProvider();
- }
@Override
protected int getTestPort() {
- return 8089;
+ return 10002;
}
@Override
@@ -104,18 +87,37 @@ public class TestMLResource extends LensJerseyTest {
config.register(MultiPartFeature.class);
}
+ @Override
+ protected URI getBaseUri() {
+ return UriBuilder.fromUri("http://localhost/").port(getTestPort()).path("/lensapi").build();
+ }
+
@BeforeTest
public void setUp() throws Exception {
super.setUp();
- setServiceProvider();
- HiveSessionService sessionService = serviceProvider.getService(SessionService.NAME);
- this.sessionHandle = sessionService.openSession("foo@localhost", "bar", new HashMap<String, String>());
- mlClient = new LensMLClient(LENS_CONNECTION_PARAMS, sessionHandle);
+ Hive hive = Hive.get(new HiveConf());
+ Database db = new Database();
+ db.setName(TEST_DB);
+ hive.createDatabase(db, true);
+ LensClientConfig lensClientConfig = new LensClientConfig();
+ lensClientConfig.setLensDatabase(TEST_DB);
+ lensClientConfig.set(LensConfConstants.SERVER_BASE_URL,
+ "http://localhost:" + getTestPort() + "/lensapi");
+ LensClient client = new LensClient(lensClientConfig);
+ mlClient = new LensMLClient(client);
}
@AfterTest
public void tearDown() throws Exception {
super.tearDown();
+ Hive hive = Hive.get(new HiveConf());
+
+ try {
+ hive.dropDatabase(TEST_DB);
+ } catch (Exception exc) {
+ // Ignore drop db exception
+ exc.printStackTrace();
+ }
mlClient.close();
}
@@ -125,13 +127,6 @@ public class TestMLResource extends LensJerseyTest {
}
@Test
- public void testStartMLServiceStarted() throws Exception {
- LOG.info("## testStartMLServiceStarted");
- MLServiceImpl svcImpl = serviceProvider.getService(MLService.NAME);
- Assert.assertEquals(svcImpl.getServiceState(), Service.STATE.STARTED);
- }
-
- @Test
public void testMLResourceUp() throws Exception {
String mlUpMsg = mlTarget.request().get(String.class);
Assert.assertEquals(mlUpMsg, MLServiceResource.ML_UP_MESSAGE);
@@ -142,22 +137,26 @@ public class TestMLResource extends LensJerseyTest {
List<String> algoNames = mlClient.getAlgorithms();
Assert.assertNotNull(algoNames);
- Assert.assertTrue(algoNames.contains(MLUtils.getAlgoName(NaiveBayesAlgo.class)),
- MLUtils.getAlgoName(NaiveBayesAlgo.class));
+ Assert.assertTrue(
+ algoNames.contains(MLUtils.getAlgoName(NaiveBayesAlgo.class)),
+ MLUtils.getAlgoName(NaiveBayesAlgo.class));
Assert.assertTrue(algoNames.contains(MLUtils.getAlgoName(SVMAlgo.class)),
- MLUtils.getAlgoName(SVMAlgo.class));
+ MLUtils.getAlgoName(SVMAlgo.class));
- Assert.assertTrue(algoNames.contains(MLUtils.getAlgoName(LogisticRegressionAlgo.class)),
- MLUtils.getAlgoName(LogisticRegressionAlgo.class));
+ Assert.assertTrue(
+ algoNames.contains(MLUtils.getAlgoName(LogisticRegressionAlgo.class)),
+ MLUtils.getAlgoName(LogisticRegressionAlgo.class));
- Assert.assertTrue(algoNames.contains(MLUtils.getAlgoName(DecisionTreeAlgo.class)),
- MLUtils.getAlgoName(DecisionTreeAlgo.class));
+ Assert.assertTrue(
+ algoNames.contains(MLUtils.getAlgoName(DecisionTreeAlgo.class)),
+ MLUtils.getAlgoName(DecisionTreeAlgo.class));
}
@Test
public void testGetAlgoParams() throws Exception {
- Map<String, String> params = mlClient.getAlgoParamDescription(MLUtils.getAlgoName(DecisionTreeAlgo.class));
+ Map<String, String> params = mlClient.getAlgoParamDescription(MLUtils
+ .getAlgoName(DecisionTreeAlgo.class));
Assert.assertNotNull(params);
Assert.assertFalse(params.isEmpty());
@@ -171,7 +170,6 @@ public class TestMLResource extends LensJerseyTest {
LOG.info("Starting train & eval");
final String algoName = MLUtils.getAlgoName(NaiveBayesAlgo.class);
HiveConf conf = new HiveConf();
- String database = "default";
String tableName = "naivebayes_training_table";
String sampleDataFilePath = "data/naive_bayes/naive_bayes_train.data";
@@ -179,26 +177,27 @@ public class TestMLResource extends LensJerseyTest {
URI sampleDataFileURI = sampleDataFile.toURI();
String labelColumn = "label";
- String[] features = {"feature_1", "feature_2", "feature_3"};
+ String[] features = { "feature_1", "feature_2", "feature_3" };
String outputTable = "naivebayes_eval_table";
- LOG.info("Creating training table from file " + sampleDataFileURI.toString());
+ LOG.info("Creating training table from file "
+ + sampleDataFileURI.toString());
Map<String, String> tableParams = new HashMap<String, String>();
try {
- ExampleUtils.createTable(conf, database, tableName, sampleDataFileURI.toString(), labelColumn, tableParams,
- features);
+ ExampleUtils.createTable(conf, TEST_DB, tableName,
+ sampleDataFileURI.toString(), labelColumn, tableParams, features);
} catch (HiveException exc) {
exc.printStackTrace();
}
MLTask.Builder taskBuilder = new MLTask.Builder();
- taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
- .serverLocation(getBaseUri().toString()).sessionHandle(mlClient.getSessionHandle()).trainingTable(tableName)
- .userName("foo@localhost").password("bar");
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn)
+ .outputTable(outputTable).client(mlClient).trainingTable(tableName);
// Add features
- taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2").addFeatureColumn("feature_3");
+ taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2")
+ .addFeatureColumn("feature_3");
MLTask task = taskBuilder.build();
@@ -212,10 +211,11 @@ public class TestMLResource extends LensJerseyTest {
Assert.assertNotNull(firstModelID);
taskBuilder = new MLTask.Builder();
- taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
- .serverLocation(getBaseUri().toString()).sessionHandle(mlClient.getSessionHandle()).trainingTable(tableName)
- .userName("foo@localhost").password("bar");
- taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2").addFeatureColumn("feature_3");
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn)
+ .outputTable(outputTable).client(mlClient).trainingTable(tableName);
+
+ taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2")
+ .addFeatureColumn("feature_3");
MLTask anotherTask = taskBuilder.build();
@@ -241,8 +241,10 @@ public class TestMLResource extends LensJerseyTest {
}
// Verify partitions created for each run
- Assert.assertTrue(partReports.contains(firstReportID), firstReportID + " first partition not there");
- Assert.assertTrue(partReports.contains(secondReportID), secondReportID + " second partition not there");
+ Assert.assertTrue(partReports.contains(firstReportID), firstReportID
+ + " first partition not there");
+ Assert.assertTrue(partReports.contains(secondReportID), secondReportID
+ + " second partition not there");
LOG.info("Completed task run");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
new file mode 100644
index 0000000..d928ff1
--- /dev/null
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml;
+
+import java.net.URI;
+
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.UriBuilder;
+
+import org.apache.lens.client.LensClient;
+import org.apache.lens.client.LensClientConfig;
+import org.apache.lens.client.LensMLClient;
+import org.apache.lens.ml.task.MLTask;
+import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.metastore.MetastoreResource;
+import org.apache.lens.server.ml.MLApp;
+import org.apache.lens.server.query.QueryServiceResource;
+import org.apache.lens.server.session.SessionResource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.testng.Assert;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+
+@Test
+public class TestMLRunner extends LensJerseyTest {
+ private static final Log LOG = LogFactory.getLog(TestMLRunner.class);
+ private static final String TEST_DB = TestMLRunner.class.getSimpleName();
+
+ private LensMLClient mlClient;
+
+ @Override
+ protected int getTestPort() {
+ return 10000;
+ }
+
+ @Override
+ protected Application configure() {
+ return new MLApp(SessionResource.class, QueryServiceResource.class, MetastoreResource.class);
+ }
+
+ @Override
+ protected URI getBaseUri() {
+ return UriBuilder.fromUri("http://localhost/").port(getTestPort()).path("/lensapi").build();
+ }
+
+ @Override
+ protected void configureClient(ClientConfig config) {
+ config.register(MultiPartFeature.class);
+ }
+
+ @BeforeTest
+ public void setUp() throws Exception {
+ super.setUp();
+ Hive hive = Hive.get(new HiveConf());
+ Database db = new Database();
+ db.setName(TEST_DB);
+ hive.createDatabase(db, true);
+ LensClientConfig lensClientConfig = new LensClientConfig();
+ lensClientConfig.setLensDatabase(TEST_DB);
+ lensClientConfig.set(LensConfConstants.SERVER_BASE_URL,
+ "http://localhost:" + getTestPort() + "/lensapi");
+ LensClient client = new LensClient(lensClientConfig);
+ mlClient = new LensMLClient(client);
+ }
+
+ @AfterTest
+ public void tearDown() throws Exception {
+ super.tearDown();
+ Hive hive = Hive.get(new HiveConf());
+ hive.dropDatabase(TEST_DB);
+ mlClient.close();
+ }
+
+ @Test
+ public void trainAndEval() throws Exception {
+ LOG.info("Starting train & eval");
+ String algoName = "spark_naive_bayes";
+ String database = "default";
+ String trainTable = "naivebayes_training_table";
+ String trainFile = "data/naive_bayes/train.data";
+ String testTable = "naivebayes_test_table";
+ String testFile = "data/naive_bayes/test.data";
+ String outputTable = "naivebayes_eval_table";
+ String[] features = { "feature_1", "feature_2", "feature_3" };
+ String labelColumn = "label";
+
+ MLRunner runner = new MLRunner();
+ runner.init(mlClient, algoName, database, trainTable, trainFile,
+ testTable, testFile, outputTable, features, labelColumn);
+ MLTask task = runner.train();
+ Assert.assertEquals(task.getTaskState(), MLTask.State.SUCCESSFUL);
+ String modelID = task.getModelID();
+ String reportID = task.getReportID();
+ Assert.assertNotNull(modelID);
+ Assert.assertNotNull(reportID);
+ }
+
+ @Test
+ public void trainAndEvalFromDir() throws Exception {
+ LOG.info("Starting train & eval from Dir");
+ MLRunner runner = new MLRunner();
+ runner.init(mlClient, "data/naive_bayes");
+ MLTask task = runner.train();
+ Assert.assertEquals(task.getTaskState(), MLTask.State.SUCCESSFUL);
+ String modelID = task.getModelID();
+ String reportID = task.getReportID();
+ Assert.assertNotNull(modelID);
+ Assert.assertNotNull(reportID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
----------------------------------------------------------------------
diff --git a/lens-ml-lib/tools/scripts/lens-ml-classpath.sh b/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
new file mode 100644
index 0000000..7254b78
--- /dev/null
+++ b/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+LENS_ML_CLASSPATH=`ls ${BASEDIR}/lib/* 2>/dev/null | tr "\n" ':' 2>/dev/null`
+echo $LENS_ML_CLASSPATH
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 16f5aad..a5dd80f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -496,6 +496,7 @@
<!-- Used on ui -->
<exclude>**/codemirror.min.*</exclude>
<exclude>**/*.js</exclude>
+ <exclude>**/*.properties</exclude>
</excludes>
</configuration>
<executions>
@@ -1134,6 +1135,7 @@
<module>lens-examples</module>
<module>lens-dist</module>
<module>lens-ml-lib</module>
+ <module>lens-ml-dist</module>
<module>lens-regression</module>
</modules>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/tools/conf-pseudo-distr/client/lens-client-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/client/lens-client-site.xml b/tools/conf-pseudo-distr/client/lens-client-site.xml
index 706e356..120bf67 100644
--- a/tools/conf-pseudo-distr/client/lens-client-site.xml
+++ b/tools/conf-pseudo-distr/client/lens-client-site.xml
@@ -23,5 +23,10 @@
<configuration>
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://localhost:9083</value>
+ <description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/tools/conf-pseudo-distr/server/lens-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/lens-site.xml b/tools/conf-pseudo-distr/server/lens-site.xml
index 6dc7eff..f11c6d6 100644
--- a/tools/conf-pseudo-distr/server/lens-site.xml
+++ b/tools/conf-pseudo-distr/server/lens-site.xml
@@ -43,4 +43,39 @@
<name>lens.query.enable.persistent.resultset.indriver</name>
<value>false</value>
</property>
+
+<property>
+ <name>lens.server.drivers</name>
+ <value>org.apache.lens.driver.hive.HiveDriver,org.apache.lens.driver.jdbc.JDBCDriver</value>
+ <name>lens.server.servicenames</name>
+ <value>session,query,metastore,scheduler,quota,ml</value>
+</property>
+
+<property>
+ <name>lens.server.ws.resourcenames</name>
+ <value>session,metastore,query,quota,scheduler,ml</value>
+ <description>These JAXRS resources would be started in the specified order when lensserver starts up</description>
+</property>
+
+<property>
+ <name>lens.server.ml.ws.resource.impl</name>
+ <value>org.apache.lens.server.ml.MLServiceResource</value>
+ <description>Implementation class for ML Service Resource</description>
+</property>
+
+<property>
+ <name>lens.server.ml.service.impl</name>
+ <value>org.apache.lens.server.ml.MLServiceImpl</value>
+ <description>Implementation class for ML service</description>
+</property>
+
+<property>
+ <name>lens.ml.drivers</name>
+ <value>org.apache.lens.ml.spark.SparkMLDriver</value>
+</property>
+
+<property>
+ <name>lens.ml.sparkdriver.spark.master</name>
+ <value>local</value>
+</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/tools/scripts/lens-config.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-config.sh b/tools/scripts/lens-config.sh
index 8bd9013..0e810cb 100644
--- a/tools/scripts/lens-config.sh
+++ b/tools/scripts/lens-config.sh
@@ -125,5 +125,9 @@ case $type in
exit 1
;;
esac
+
+# add LENS_EXT_CLASSPATH
+LENSCPPATH=$LENS_EXT_CLASSPATH:$LENSCPPATH
+
export LENSCPPATH
export LENS_OPTS
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8d185913/tools/scripts/lens-run-class.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-run-class.sh b/tools/scripts/lens-run-class.sh
new file mode 100644
index 0000000..ef64b03
--- /dev/null
+++ b/tools/scripts/lens-run-class.sh
@@ -0,0 +1,170 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+. ${BASEDIR}/bin/lens-config.sh 'client'
+
+JAVA_PROPERTIES="$LENS_OPTS $LENS_PROPERTIES -Dlens.log.dir=$LENS_LOG_DIR -Dlens.home=${LENS_HOME_DIR} -Dconfig.location=$LENS_CONF"
+
+
+################################
+# functions
+################################
+
+info() {
+ local msg=$1
+
+ echo "Info: $msg" >&2
+}
+
+warn() {
+ local msg=$1
+
+ echo "Warning: $msg" >&2
+}
+
+error() {
+ local msg=$1
+ local exit_code=$2
+
+ echo "Error: $msg" >&2
+
+ if [ -n "$exit_code" ] ; then
+ exit $exit_code
+ fi
+}
+
+setenv() {
+
+# HADOOP_HOME env variable overrides hadoop in the path
+ HADOOP_HOME=${HADOOP_HOME:-${HADOOP_PREFIX}}
+ if [ "$HADOOP_HOME" == "" ]; then
+ echo "Cannot find hadoop installation: \$HADOOP_HOME or \$HADOOP_PREFIX must be set or hadoop must be in the path";
+ exit 4;
+ else
+ echo "Adding hadoop libs in classpath from $HADOOP_HOME"
+
+ #ASSUMPTION: hadoop jars would be present in HADOOP_HOME if installed through deb or would be present
+ # in HADOOP_HOME/share/hadoop if installed through tarball. They can not coexist.
+
+ CORE_JARS=`ls $HADOOP_HOME/hadoop-core-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+ LIB_JARS=`ls $HADOOP_HOME/lib/guava-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ LIB_JARS=$LIB_JARS:`ls ${HADOOP_HOME}/lib/commons-configuration-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ LIB_JARS=$LIB_JARS:`ls ${HADOOP_HOME}/lib/protobuf-java-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ LIB_JARS=$LIB_JARS:`ls ${HADOOP_HOME}/share/hadoop/common/lib/commons-configuration-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ LIB_JARS=$LIB_JARS:`ls ${HADOOP_HOME}/share/hadoop/hdfs/lib/protobuf-java-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+
+ COMMON_JARS=`ls ${HADOOP_HOME}/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ COMMON_JARS=$COMMON_JARS:`ls ${HADOOP_HOME}/share/hadoop/common/hadoop-common-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ COMMON_JARS=$COMMON_JARS:`ls ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+ HDFS_JARS=`ls ${HADOOP_HOME}/../hadoop-hdfs/hadoop-hdfs-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ HDFS_JARS=$HDFS_JARS:`ls ${HADOOP_HOME}/share/hadoop/hdfs/hadoop-hdfs-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+ MAPRED_JARS=`ls ${HADOOP_HOME}/../hadoop-mapreduce/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+ HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS
+ LENSCPPATH=${LENSCPPATH}:$HADOOP_JARPATH
+ fi
+
+ if [ "$HIVE_HOME" != "" ]; then
+ echo "HIVE_HOME is set, adding ${HIVE_HOME}/lib/* into lens classpath"
+ LENSCPPATH=${LENSCPPATH}:`ls ${HIVE_HOME}/lib/* 2>/dev/null | tr "\n" ':' 2>/dev/null`
+ else
+ echo "HIVE_HOME is not set. Set HIVE_HOME and try again"
+ exit 1
+ fi
+
+ # Add HIVE_HOME to HADOOP_CLASS_PATH
+ HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HIVE_HOME}/lib/*"
+ export HADOOP_CLASSPATH
+
+}
+################################
+# main
+################################
+
+opt_conf=""
+opt_classname=""
+while [ -n "$*" ] ; do
+ arg=$1
+ shift
+ case "$arg" in
+ --conf|-c)
+ [ -n "$1" ] || error "Option --conf requires an argument" 1
+ opt_conf=$1
+ shift
+ ;;
+ --classpath|-C)
+ [ -n "$1" ] || error "Option --classpath requires an argument" 1
+ LENSCPPATH="LENSCPPATH:$1"
+ shift
+ ;;
+ -D*)
+ JAVA_PROPERTIES="${JAVA_PROPERTIES} $arg"
+ ;;
+ *)
+ if [ "$opt_classname" == "" ]; then
+ opt_classname=$arg
+ echo "opt_classname is " $opt_classname
+ else
+ args="$args $arg"
+ fi
+ ;;
+ esac
+done
+echo "args are " $args
+
+
+# prepend conf dir to classpath
+if [ -n "$opt_conf" ]; then
+ LENSCPPATH="$opt_conf:$LENSCPPATH"
+fi
+
+# finally, invoke the appropriate command
+if [ "$opt_classname" == "" ]; then
+ echo "Usage : $0 <classname>"
+ exit 1
+fi
+echo "Executing class " $opt_classname
+
+setenv
+
+exec ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${LENSCPPATH} \
+ "$opt_classname" $args
+
+exit 0