You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mi...@apache.org on 2016/06/24 03:58:19 UTC
zeppelin git commit: [ZEPPELIN-1046] bin/install-interpreter.sh for
netinst package
Repository: zeppelin
Updated Branches:
refs/heads/master df7dd5c37 -> 4efb39f45
[ZEPPELIN-1046] bin/install-interpreter.sh for netinst package
### What is this PR for?
Implementation of bin/install-interpreter.sh for netinst package which suggested in the [discussion](http://apache-zeppelin-users-incubating-mailing-list.75479.x6.nabble.com/Ask-opinion-regarding-0-6-0-release-package-tp3298p3314.html).
Some usages will be
```
# download all interpreters provided by Apache Zeppelin project
bin/install-interpreter.sh --all
# download an interpreter with name (for example markdown interpreter)
bin/install-interpreter.sh --name md
# download an (3rd party) interpreter with specific maven artifact name
bin/install-interpreter.sh --name md -t org.apache.zeppelin:zeppelin-markdown:0.6.0-SNAPSHOT
```
If it looks fine, i'll continue the work (refactor code, and add test)
### What type of PR is it?
Feature
### Todos
* [x] - working implementation
* [x] - refactor
* [x] - add test
### What is the Jira issue?
* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?
Author: Lee moon soo <mo...@apache.org>
Author: AhyoungRyu <fb...@hanmail.net>
Closes #1042 from Leemoonsoo/netinst and squashes the following commits:
f81d16e [Lee moon soo] address mina's comment
049bc89 [Lee moon soo] Update docs
7307c67 [Lee moon soo] Merge remote-tracking branch 'AhyoungRyu/netinst-docs' into netinst
7e749ad [Lee moon soo] Address mina's comment
0eedd2a [AhyoungRyu] Address @minahlee feedback
13f2d04 [Lee moon soo] generate netinst package
03c664e [AhyoungRyu] Add a new line
5d0a971 [AhyoungRyu] Revert install.md to latest version
13899fb [AhyoungRyu] Reorganize interpreter installation docs
4c1f029 [Lee moon soo] Proxy support
9079580 [Lee moon soo] fix artifact name
1077296 [Lee moon soo] update test
aebca17 [Lee moon soo] Add docs
d547551 [Lee moon soo] Remove test entries
6ee06b8 [Lee moon soo] Make DependencyResolver in zeppelin-interpreter module not aware of ZEPPELIN_HOME
7b1b36a [Lee moon soo] update usage
49f0568 [Lee moon soo] Add conf/interpreter-list
1b558fd [Lee moon soo] update some text
ec7d152 [Lee moon soo] add tip
2c81a3f [Lee moon soo] update
78a7c52 [Lee moon soo] Refactor and add test
47f5706 [Lee moon soo] Install multiple interpreters at once
38e2556 [Lee moon soo] Initial implementation of install-interpreter.sh
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4efb39f4
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4efb39f4
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4efb39f4
Branch: refs/heads/master
Commit: 4efb39f45079b018d9a7907870c8e435924c63f8
Parents: df7dd5c
Author: Lee moon soo <mo...@apache.org>
Authored: Thu Jun 23 15:04:43 2016 -0700
Committer: Mina Lee <mi...@apache.org>
Committed: Thu Jun 23 20:58:10 2016 -0700
----------------------------------------------------------------------
bin/install-interpreter.sh | 45 +++
conf/interpreter-list | 35 +++
dev/create_release.sh | 1 +
docs/_includes/themes/zeppelin/_navigation.html | 6 +-
docs/install/install.md | 5 +-
docs/manual/interpreterinstallation.md | 164 ++++++++++
.../dep/AbstractDependencyResolver.java | 12 +
.../apache/zeppelin/dep/DependencyResolver.java | 25 +-
.../zeppelin/dep/DependencyResolverTest.java | 31 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 4 +
.../interpreter/InterpreterFactory.java | 6 +-
.../interpreter/install/InstallInterpreter.java | 301 +++++++++++++++++++
.../install/InstallInterpreterTest.java | 86 ++++++
13 files changed, 680 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/bin/install-interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/install-interpreter.sh b/bin/install-interpreter.sh
new file mode 100755
index 0000000..06be75c
--- /dev/null
+++ b/bin/install-interpreter.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Run Zeppelin
+#
+
+bin=$(dirname "${BASH_SOURCE-$0}")
+bin=$(cd "${bin}">/dev/null; pwd)
+
+. "${bin}/common.sh"
+
+
+ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter
+ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/install-interpreter.log"
+JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
+fi
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
+fi
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
+
+addJarInDir "${ZEPPELIN_HOME}/lib"
+
+CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_INSTALL_INTERPRETER_MAIN ${@}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/conf/interpreter-list
----------------------------------------------------------------------
diff --git a/conf/interpreter-list b/conf/interpreter-list
new file mode 100644
index 0000000..a2f6426
--- /dev/null
+++ b/conf/interpreter-list
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# [name] [maven artifact] [description]
+
+alluxio org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter
+angular org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering
+cassandra org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter
+elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter
+file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter
+flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter
+hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter
+ignite org.apache.zeppelin:zeppelin-ignite:0.6.0 Ignite interpreter
+jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.0 Jdbc interpreter
+kylin org.apache.zeppelin:zeppelin-kylin:0.6.0 Kylin interpreter
+lens org.apache.zeppelin:zeppelin-lens:0.6.0 Lens interpreter
+livy org.apache.zeppelin:zeppelin-livy:0.6.0 Livy interpreter
+md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown support
+postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter
+python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter
+shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/dev/create_release.sh
----------------------------------------------------------------------
diff --git a/dev/create_release.sh b/dev/create_release.sh
index f1bba0d..2363d90 100755
--- a/dev/create_release.sh
+++ b/dev/create_release.sh
@@ -103,6 +103,7 @@ function make_binary_release() {
git_clone
make_source_package
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
+make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell"
# remove non release files and dirs
rm -rf "${WORKING_DIR}/zeppelin"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/docs/_includes/themes/zeppelin/_navigation.html
----------------------------------------------------------------------
diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html
index 40b9fb6..498a2d3 100644
--- a/docs/_includes/themes/zeppelin/_navigation.html
+++ b/docs/_includes/themes/zeppelin/_navigation.html
@@ -21,8 +21,8 @@
<li><a href="{{BASE_PATH}}/index.html">What is Apache Zeppelin ?</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Getting Started</b><span></li>
- <li><a href="{{BASE_PATH}}/install/install.html">Quick Start</a></li>
- <li><a href="{{BASE_PATH}}/install/install.html#zeppelin-configuration">Configuration</a></li>
+ <li><a href="{{BASE_PATH}}/install/install.html">Install</a></li>
+ <li><a href="{{BASE_PATH}}/install/install.html#apache-zeppelin-configuration">Configuration</a></li>
<li><a href="{{BASE_PATH}}/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li>
<li><a href="{{BASE_PATH}}/quickstart/tutorial.html">Tutorial</a></li>
<li role="separator" class="divider"></li>
@@ -42,6 +42,7 @@
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Usage</b><span></li>
+ <li><a href="{{BASE_PATH}}/manual/interpreterinstallation.html">Interpreter Installation</a></li>
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
<li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
<li role="separator" class="divider"></li>
@@ -110,3 +111,4 @@
</nav><!--/.navbar-collapse -->
</div>
</div>
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/docs/install/install.md
----------------------------------------------------------------------
diff --git a/docs/install/install.md b/docs/install/install.md
index 93d9feb..55741dc 100644
--- a/docs/install/install.md
+++ b/docs/install/install.md
@@ -60,6 +60,9 @@ Although it can be unstable somehow since it is on development status, you can e
### Downloading Binary Package
If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html).
+
+If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`.
+
After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section.
### Building from Source
@@ -388,4 +391,4 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<td>1024000</td>
<td>Size in characters of the maximum text message to be received by websocket.</td>
</tr>
-</table>
+</table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/docs/manual/interpreterinstallation.md
----------------------------------------------------------------------
diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md
new file mode 100644
index 0000000..d522e62
--- /dev/null
+++ b/docs/manual/interpreterinstallation.md
@@ -0,0 +1,164 @@
+---
+layout: page
+title: "Interpreter Installation"
+description: ""
+group: manual
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# Interpreter Installation
+
+Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters.
+
+## Community managed interpreters
+Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters).
+If you downloaded `netinst` binary package, you need to install by using below commands.
+
+#### Install all community managed interpreters
+
+```
+./bin/install-interpreter.sh --all
+```
+
+#### Install specific interpreters
+
+```
+./bin/install-interpreter.sh --name md,shell,jdbc,python
+```
+
+You can get full list of community managed interpreters by running
+
+```
+./bin/install-interpreter.sh --list
+```
+
+Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
+
+
+## 3rd party interpreters
+
+You can also install 3rd party interpreters located in the maven repository by using below commands.
+
+#### Install 3rd party interpreters
+
+```
+./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1
+```
+
+The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory.
+
+Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration).
+And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
+
+
+#### Install multiple 3rd party interpreters at once
+
+```
+./bin/install-interpreter.sh --name interpreter1,interpreter2 --artifact groupId1:artifact1:version1,groupId2:artifact2:version2
+```
+
+`--name` and `--artifact` arguments will recieve comma separated list.
+
+## Available community managed interpreters
+
+You can also find the below community managed interpreter list in `conf/interpreter-list` file.
+<table class="table-configuration">
+ <tr>
+ <th>Name</th>
+ <th>Maven Artifact</th>
+ <th>Description</th>
+ </tr>
+ <tr>
+ <td>alluxio</td>
+ <td>org.apache.zeppelin:zeppelin-alluxio:0.6.0</td>
+ <td>Alluxio interpreter</td>
+ </tr>
+ <tr>
+ <td>angular</td>
+ <td>org.apache.zeppelin:zeppelin-angular:0.6.0</td>
+ <td>HTML and AngularJS view rendering</td>
+ </tr>
+ <tr>
+ <td>cassandra</td>
+ <td>org.apache.zeppelin:zeppelin-cassandra:0.6.0</td>
+ <td>Cassandra interpreter</td>
+ </tr>
+ <tr>
+ <td>elasticsearch</td>
+ <td>org.apache.zeppelin:zeppelin-elasticsearch:0.6.0</td>
+ <td>Elasticsearch interpreter</td>
+ </tr>
+ <tr>
+ <td>file</td>
+ <td>org.apache.zeppelin:zeppelin-file:0.6.0</td>
+ <td>HDFS file interpreter</td>
+ </tr>
+ <tr>
+ <td>flink</td>
+ <td>org.apache.zeppelin:zeppelin-flink:0.6.0</td>
+ <td>Flink interpreter</td>
+ </tr>
+ <tr>
+ <td>hbase</td>
+ <td>org.apache.zeppelin:zeppelin-hbase:0.6.0</td>
+ <td>Hbase interpreter</td>
+ </tr>
+ <tr>
+ <td>ignite</td>
+ <td>org.apache.zeppelin:zeppelin-ignite:0.6.0</td>
+ <td>Ignite interpreter</td>
+ </tr>
+ <tr>
+ <td>jdbc</td>
+ <td>org.apache.zeppelin:zeppelin-jdbc:0.6.0</td>
+ <td>Jdbc interpreter</td>
+ </tr>
+ <tr>
+ <td>kylin</td>
+ <td>org.apache.zeppelin:zeppelin-kylin:0.6.0</td>
+ <td>Kylin interpreter</td>
+ </tr>
+ <tr>
+ <td>lens</td>
+ <td>org.apache.zeppelin:zeppelin-lens:0.6.0</td>
+ <td>Lens interpreter</td>
+ </tr>
+ <tr>
+ <td>livy</td>
+ <td>org.apache.zeppelin:zeppelin-livy:0.6.0</td>
+ <td>Livy interpreter</td>
+ </tr>
+ <tr>
+ <td>md</td>
+ <td>org.apache.zeppelin:zeppelin-markdown:0.6.0</td>
+ <td>Markdown support</td>
+ </tr>
+ <tr>
+ <td>postgresql</td>
+ <td>org.apache.zeppelin:zeppelin-postgresql:0.6.0</td>
+ <td>Postgresql interpreter</td>
+ </tr>
+ <tr>
+ <td>python</td>
+ <td>org.apache.zeppelin:zeppelin-python:0.6.0</td>
+ <td>Python interpreter</td>
+ </tr>
+ <tr>
+ <td>shell</td>
+ <td>org.apache.zeppelin:zeppelin-shell:0.6.0</td>
+ <td>Shell command</td>
+ </tr>
+</table>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
index b22941e..1e0844e 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
@@ -17,6 +17,7 @@
package org.apache.zeppelin.dep;
+import java.net.URL;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
@@ -25,6 +26,7 @@ import java.util.List;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.Proxy;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.RepositoryPolicy;
import org.sonatype.aether.resolution.ArtifactResult;
@@ -44,6 +46,16 @@ public abstract class AbstractDependencyResolver {
repos.add(Booter.newLocalRepository());
}
+ public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) {
+ Authentication auth = new Authentication(proxyUser, proxyPassword);
+ Proxy proxy = new Proxy(proxyUrl.getProtocol(), proxyUrl.getHost(), proxyUrl.getPort(), auth);
+ synchronized (repos) {
+ for (RemoteRepository repo : repos) {
+ repo.setProxy(proxy);
+ }
+ }
+ }
+
public List<RemoteRepository> getRepos() {
return this.repos;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
index 60ef1f9..214175a 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
@@ -19,6 +19,7 @@ package org.apache.zeppelin.dep;
import java.io.File;
import java.io.IOException;
+import java.net.URL;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
@@ -63,11 +64,6 @@ public class DependencyResolver extends AbstractDependencyResolver {
return load(artifact, new LinkedList<String>());
}
- public List<File> load(String artifact, String destPath)
- throws RepositoryException, IOException {
- return load(artifact, new LinkedList<String>(), destPath);
- }
-
public synchronized List<File> load(String artifact, Collection<String> excludes)
throws RepositoryException, IOException {
if (StringUtils.isBlank(artifact)) {
@@ -85,25 +81,20 @@ public class DependencyResolver extends AbstractDependencyResolver {
return libs;
}
}
-
- public List<File> load(String artifact, Collection<String> excludes, String destPath)
+
+ public List<File> load(String artifact, File destPath) throws IOException, RepositoryException {
+ return load(artifact, new LinkedList<String>(), destPath);
+ }
+
+ public List<File> load(String artifact, Collection<String> excludes, File destPath)
throws RepositoryException, IOException {
List<File> libs = new LinkedList<File>();
if (StringUtils.isNotBlank(artifact)) {
libs = load(artifact, excludes);
- // find home dir
- String home = System.getenv("ZEPPELIN_HOME");
- if (home == null) {
- home = System.getProperty("zeppelin.home");
- }
- if (home == null) {
- home = "..";
- }
-
for (File srcFile : libs) {
- File destFile = new File(home + "/" + destPath, srcFile.getName());
+ File destFile = new File(destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
index a8664ef..af2c7ff 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
@@ -33,27 +33,20 @@ import org.sonatype.aether.RepositoryException;
public class DependencyResolverTest {
private static DependencyResolver resolver;
private static String testPath;
- private static String testCopyPath;
- private static String home;
-
+ private static File testCopyPath;
+ private static File tmpDir;
+
@BeforeClass
public static void setUp() throws Exception {
- testPath = "test-repo";
- testCopyPath = "test-copy-repo";
+ tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
+ testPath = tmpDir.getAbsolutePath() + "/test-repo";
+ testCopyPath = new File(tmpDir, "test-copy-repo");
resolver = new DependencyResolver(testPath);
- home = System.getenv("ZEPPELIN_HOME");
- if (home == null) {
- home = System.getProperty("zeppelin.home");
- }
- if (home == null) {
- home = "..";
- }
}
@AfterClass
public static void tearDown() throws Exception {
- FileUtils.deleteDirectory(new File(home + "/" + testPath));
- FileUtils.deleteDirectory(new File(home + "/" + testCopyPath));
+ FileUtils.deleteDirectory(tmpDir);
}
@Rule
@@ -78,19 +71,19 @@ public class DependencyResolverTest {
public void testLoad() throws Exception {
// basic load
resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath);
- assertEquals(new File(home + "/" + testCopyPath).list().length, 4);
- FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
+ assertEquals(testCopyPath.list().length, 4);
+ FileUtils.cleanDirectory(testCopyPath);
// load with exclusions parameter
resolver.load("com.databricks:spark-csv_2.10:1.3.0",
Collections.singletonList("org.scala-lang:scala-library"), testCopyPath);
- assertEquals(new File(home + "/" + testCopyPath).list().length, 3);
- FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
+ assertEquals(testCopyPath.list().length, 3);
+ FileUtils.cleanDirectory(testCopyPath);
// load from added repository
resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false);
resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
- assertEquals(new File(home + "/" + testCopyPath).list().length, 8);
+ assertEquals(testCopyPath.list().length, 8);
// load invalid artifact
resolver.delRepo("sonatype");
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 45fbba4..0a7b8c0 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -346,6 +346,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
}
+ public String getInterpreterListPath() {
+ return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
+ }
+
public String getInterpreterDir() {
return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index bad18c0..aeb7818 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -350,15 +350,17 @@ public class InterpreterFactory implements InterpreterGroupFactory {
List<Dependency> deps = intSetting.getDependencies();
if (deps != null) {
for (Dependency d: deps) {
+ File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
if (d.getExclusions() != null) {
depResolver.load(
d.getGroupArtifactVersion(),
d.getExclusions(),
- conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id());
+ new File(destDir, intSetting.id()));
} else {
depResolver.load(
d.getGroupArtifactVersion(),
- conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id());
+ new File(destDir, intSetting.id()));
}
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
new file mode 100644
index 0000000..da67d9f
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.install;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Logger;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.util.Util;
+import org.sonatype.aether.RepositoryException;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Commandline utility to install interpreter from maven repository
+ */
+public class InstallInterpreter {
+ private final File interpreterListFile;
+ private final File interpreterBaseDir;
+ private final List<AvailableInterpreterInfo> availableInterpreters;
+ private final String localRepoDir;
+ private URL proxyUrl;
+ private String proxyUser;
+ private String proxyPassword;
+
+ /**
+ *
+ * @param interpreterListFile
+ * @param interpreterBaseDir interpreter directory for installing binaries
+ * @throws IOException
+ */
+ public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir)
+ throws IOException {
+ this.interpreterListFile = interpreterListFile;
+ this.interpreterBaseDir = interpreterBaseDir;
+ this.localRepoDir = localRepoDir;
+ availableInterpreters = new LinkedList<AvailableInterpreterInfo>();
+ readAvailableInterpreters();
+ }
+
+
+ /**
+ * Information for available informations
+ */
+ private static class AvailableInterpreterInfo {
+ public final String name;
+ public final String artifact;
+ public final String description;
+
+ public AvailableInterpreterInfo(String name, String artifact, String description) {
+ this.name = name;
+ this.artifact = artifact;
+ this.description = description;
+ }
+ }
+
+ private void readAvailableInterpreters() throws IOException {
+ if (!interpreterListFile.isFile()) {
+ System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath());
+ return;
+ }
+ String text = FileUtils.readFileToString(interpreterListFile);
+ String[] lines = text.split("\n");
+
+ Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)");
+
+ int lineNo = 0;
+ for (String line : lines) {
+ lineNo++;
+ if (line == null || line.length() == 0 || line.startsWith("#")) {
+ continue;
+ }
+
+ Matcher match = pattern.matcher(line);
+ if (match.groupCount() != 3) {
+ System.err.println("Error on line " + lineNo + ", " + line);
+ continue;
+ }
+
+ match.find();
+
+ String name = match.group(1);
+ String artifact = match.group(2);
+ String description = match.group(3);
+
+ availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description));
+ }
+ }
+
+ public List<AvailableInterpreterInfo> list() {
+ for (AvailableInterpreterInfo info : availableInterpreters) {
+ System.out.println(info.name + "\t\t\t" + info.description);
+ }
+
+ return availableInterpreters;
+ }
+
+ public void installAll() {
+ for (AvailableInterpreterInfo info : availableInterpreters) {
+ install(info.name, info.artifact);
+ }
+ }
+
+ public void install(String [] names) {
+ for (String name : names) {
+ install(name);
+ }
+ }
+
+ public void install(String name) {
+ // find artifact name
+ for (AvailableInterpreterInfo info : availableInterpreters) {
+ if (name.equals(info.name)) {
+ install(name, info.artifact);
+ return;
+ }
+ }
+
+ throw new RuntimeException("Can't find interpreter '" + name + "'");
+ }
+
+ public void install(String [] names, String [] artifacts) {
+ if (names.length != artifacts.length) {
+ throw new RuntimeException("Length of given names and artifacts are different");
+ }
+
+ for (int i = 0; i < names.length; i++) {
+ install(names[i], artifacts[i]);
+ }
+ }
+
+ public void install(String name, String artifact) {
+ DependencyResolver depResolver = new DependencyResolver(localRepoDir);
+ if (proxyUrl != null) {
+ depResolver.setProxy(proxyUrl, proxyUser, proxyPassword);
+ }
+
+ File installDir = new File(interpreterBaseDir, name);
+ if (installDir.exists()) {
+ System.err.println("Directory " + installDir.getAbsolutePath() + " already exists. Skipping");
+ return;
+ }
+
+ System.out.println("Install " + name + "(" + artifact + ") to "
+ + installDir.getAbsolutePath() + " ... ");
+
+ try {
+ depResolver.load(artifact, installDir);
+ System.out.println("Interpreter " + name + " installed under " +
+ installDir.getAbsolutePath() + ".");
+ } catch (RepositoryException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) {
+ this.proxyUrl = proxyUrl;
+ this.proxyUser = proxyUser;
+ this.proxyPassword = proxyPassword;
+ }
+
+ public static void usage() {
+ System.out.println("Options");
+ System.out.println(" -l, --list List available interpreters");
+ System.out.println(" -a, --all Install all available interpreters");
+ System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " +
+ "list)" +
+ "e.g. md,shell,jdbc,python,angular");
+ System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" +
+ ". " +
+ "(comma separated list correspond to --name) " +
+ "e.g. customGroup:customArtifact:customVersion");
+ System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port");
+ System.out.println(" --proxy-user [user] (Optional) proxy user");
+ System.out.println(" --proxy-password [password] (Optional) proxy password");
+ }
+
+ public static void main(String [] args) throws IOException {
+ if (args.length == 0) {
+ usage();
+ return;
+ }
+
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ InstallInterpreter installer = new InstallInterpreter(
+ new File(conf.getInterpreterListPath()),
+ new File(conf.getInterpreterDir()),
+ conf.getInterpreterLocalRepoPath());
+
+ String names = null;
+ String artifacts = null;
+ URL proxyUrl = null;
+ String proxyUser = null;
+ String proxyPassword = null;
+ boolean all = false;
+
+ for (int i = 0; i < args.length; i++) {
+ String arg = args[i].toLowerCase(Locale.US);
+ switch (arg) {
+ case "--list":
+ case "-l":
+ installer.list();
+ System.exit(0);
+ break;
+ case "--all":
+ case "-a":
+ all = true;
+ break;
+ case "--name":
+ case "-n":
+ names = args[++i];
+ break;
+ case "--artifact":
+ case "-t":
+ artifacts = args[++i];
+ break;
+ case "--version":
+ case "-v":
+ Util.getVersion();
+ break;
+ case "--proxy-url":
+ proxyUrl = new URL(args[++i]);
+ break;
+ case "--proxy-user":
+ proxyUser = args[++i];
+ break;
+ case "--proxy-password":
+ proxyPassword = args[++i];
+ break;
+ case "--help":
+ case "-h":
+ usage();
+ System.exit(0);
+ break;
+ default:
+ System.out.println("Unknown option " + arg);
+ }
+ }
+
+ if (proxyUrl != null) {
+ installer.setProxy(proxyUrl, proxyUser, proxyPassword);
+ }
+
+ if (all) {
+ installer.installAll();
+ System.exit(0);
+ }
+
+ if (names != null) {
+ if (artifacts != null) {
+ installer.install(names.split(","), artifacts.split(","));
+ startTip();
+ configurationTip();
+ interpreterSettingTip();
+ } else {
+ installer.install(names.split(","));
+ startTip();
+ interpreterSettingTip();
+ }
+ }
+ }
+
+ private static void startTip() {
+ System.out.println("");
+ }
+
+ private static void configurationTip() {
+ System.out.println("Add interpreter class name to '"
+ + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName() + "' property "
+ + "in your conf/zeppelin-site.xml file");
+ }
+
+ private static void interpreterSettingTip() {
+ System.out.println("Create interpreter setting in 'Interpreter' menu on GUI."
+ + " And then you can bind interpreter on your notebook");
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
new file mode 100644
index 0000000..e934f1a
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
@@ -0,0 +1,86 @@
+package org.apache.zeppelin.interpreter.install;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class InstallInterpreterTest {
+ private File tmpDir;
+ private InstallInterpreter installer;
+ private File interpreterBaseDir;
+
+ @Before
+ public void setUp() throws IOException {
+ tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
+ new File(tmpDir, "conf").mkdirs();
+ interpreterBaseDir = new File(tmpDir, "interpreter");
+ File localRepoDir = new File(tmpDir, "local-repo");
+ interpreterBaseDir.mkdir();
+ localRepoDir.mkdir();
+
+ File interpreterListFile = new File(tmpDir, "conf/interpreter-list");
+
+
+ // create interpreter list file
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
+
+ String interpreterList = "";
+ interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n";
+ interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n";
+
+ FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList);
+
+ installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir
+ .getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.deleteDirectory(tmpDir);
+ }
+
+
+ @Test
+ public void testList() {
+ assertEquals(2, installer.list().size());
+ }
+
+ @Test
+ public void install() {
+ assertEquals(0, interpreterBaseDir.listFiles().length);
+
+ installer.install("intp1");
+ assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
+ }
+
+ @Test
+ public void installAll() {
+ installer.installAll();
+ assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
+ assertTrue(new File(interpreterBaseDir, "intp2").isDirectory());
+ }
+}