You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2017/05/28 00:36:01 UTC
zeppelin git commit: [ZEPPELIN-2152] Added proxy config support in
Helium bundle factory
Repository: zeppelin
Updated Branches:
refs/heads/master ecb688f45 -> 87480056a
[ZEPPELIN-2152] Added proxy config support in Helium bundle factory
### What is this PR for?
Proxy configuration implementation in Helium bundle factory. Used by Node, NPM and YarnPkg installers.
### What type of PR is it?
[Improvement]
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2152
### How should this be tested?
Trigger Helium plugins behind proxy
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Nelson Costa <ne...@gmail.com>
Closes #2363 from necosta/zeppelin2152 and squashes the following commits:
f681e73 [Nelson Costa] [ZEPPELIN-2152] Added documentation
5e29cca [Nelson Costa] [ZEPPELIN-2152] Fixed format issues
f55e6e2 [Nelson Costa] [ZEPPELIN-2152] Fixed bug in secure/insecure proxy. Added proxy for HeliumOnlineRegistry
d9a086a [Nelson Costa] [ZEPPELIN-2152] Added proxy config support in Helium bundle factory
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/87480056
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/87480056
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/87480056
Branch: refs/heads/master
Commit: 87480056a0b10effaf5556d000f8d0c5186848d0
Parents: ecb688f
Author: Nelson Costa <ne...@gmail.com>
Authored: Wed May 24 17:03:53 2017 +0100
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun May 28 09:35:53 2017 +0900
----------------------------------------------------------------------
docs/_includes/themes/zeppelin/_navigation.html | 1 +
docs/index.md | 1 +
docs/security/helium_authorization.md | 28 ++++++++
.../zeppelin/helium/HeliumBundleFactory.java | 71 +++++++++++++++++---
.../zeppelin/helium/HeliumOnlineRegistry.java | 32 ++++++++-
.../helium/HeliumBundleFactoryTest.java | 2 +-
6 files changed, 124 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/docs/_includes/themes/zeppelin/_navigation.html
----------------------------------------------------------------------
diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html
index caa1627..296be25 100644
--- a/docs/_includes/themes/zeppelin/_navigation.html
+++ b/docs/_includes/themes/zeppelin/_navigation.html
@@ -117,6 +117,7 @@
<li><a href="{{BASE_PATH}}/security/shiroauthentication.html">Shiro Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
+ <li><a href="{{BASE_PATH}}/security/helium_authorization.html">Helium Authorization</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Helium Framework (Experimental)</b></span></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/docs/index.md
----------------------------------------------------------------------
diff --git a/docs/index.md b/docs/index.md
index 7477070..bb2222b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -173,6 +173,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* [Shiro Authentication](./security/shiroauthentication.html)
* [Notebook Authorization](./security/notebook_authorization.html)
* [Data Source Authorization](./security/datasource_authorization.html)
+ * [Helium Authorization](./security/helium_authorization.html)
* Helium Framework (Experimental)
* [Writing Zeppelin Application](./development/writingzeppelinapplication.html)
* [Writing Zeppelin Spell](./development/writingzeppelinspell.html)
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/docs/security/helium_authorization.md
----------------------------------------------------------------------
diff --git a/docs/security/helium_authorization.md b/docs/security/helium_authorization.md
new file mode 100644
index 0000000..fdfb435
--- /dev/null
+++ b/docs/security/helium_authorization.md
@@ -0,0 +1,28 @@
+---
+layout: page
+title: "Helium Authorization in Apache Zeppelin"
+description: "Apache Zeppelin supports Helium plugins which fetch required installer packages from remote registry/repositories"
+group: security
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# Helium Authorization in Apache Zeppelin
+
+<div id="toc"></div>
+
+## How to configure proxies?
+
+Set **http_proxy** and **https_proxy** env variables to allow connection to npm registry behind a corporate firewall.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
index ef032bd..c585567 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
@@ -29,6 +29,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Appender;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
@@ -38,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
+import java.net.URI;
import java.util.*;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -50,6 +52,7 @@ public class HeliumBundleFactory {
private final String NODE_VERSION = "v6.9.1";
private final String NPM_VERSION = "3.10.8";
private final String YARN_VERSION = "v0.21.3";
+ private static final String NPM_PACKAGE_NAME = "npm";
public static final String HELIUM_LOCAL_REPO = "helium-bundle";
public static final String HELIUM_BUNDLES_DIR = "bundles";
public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
@@ -74,7 +77,9 @@ public class HeliumBundleFactory {
private File tabledataModulePath;
private File visualizationModulePath;
private File spellModulePath;
+ private String defaultNodeRegistryUrl;
private String defaultNpmRegistryUrl;
+ private String defaultYarnRegistryUrl;
private Gson gson;
private boolean nodeAndNpmInstalled = false;
@@ -102,7 +107,10 @@ public class HeliumBundleFactory {
this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
this.yarnCacheDir = new File(heliumLocalRepoDirectory, YARN_CACHE_DIR);
this.conf = conf;
+ // To be done in ZEPPELIN-2214: Soft-code installer urls
+ this.defaultNodeRegistryUrl = "https://nodejs.org/dist/";
this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
+ this.defaultYarnRegistryUrl = "https://github.com/yarnpkg/yarn/releases/download/";
nodeInstallationDirectory = (nodeInstallationDir == null) ?
heliumLocalRepoDirectory : nodeInstallationDir;
@@ -118,16 +126,22 @@ public class HeliumBundleFactory {
return;
}
try {
- NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
+ NodeInstaller nodeInstaller = frontEndPluginFactory
+ .getNodeInstaller(getProxyConfig(isSecure(defaultNodeRegistryUrl)));
nodeInstaller.setNodeVersion(NODE_VERSION);
+ nodeInstaller.setNodeDownloadRoot(defaultNodeRegistryUrl);
nodeInstaller.install();
- NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig());
+ NPMInstaller npmInstaller = frontEndPluginFactory
+ .getNPMInstaller(getProxyConfig(isSecure(defaultNpmRegistryUrl)));
npmInstaller.setNpmVersion(NPM_VERSION);
+ npmInstaller.setNpmDownloadRoot(defaultNpmRegistryUrl + "/" + NPM_PACKAGE_NAME + "/-/");
npmInstaller.install();
- YarnInstaller yarnInstaller = frontEndPluginFactory.getYarnInstaller(getProxyConfig());
+ YarnInstaller yarnInstaller = frontEndPluginFactory
+ .getYarnInstaller(getProxyConfig(isSecure(defaultYarnRegistryUrl)));
yarnInstaller.setYarnVersion(YARN_VERSION);
+ yarnInstaller.setYarnDownloadRoot(defaultYarnRegistryUrl);
yarnInstaller.install();
yarnCacheDir.mkdirs();
String yarnCacheDirPath = yarnCacheDir.getAbsolutePath();
@@ -140,9 +154,48 @@ public class HeliumBundleFactory {
}
}
- private ProxyConfig getProxyConfig() {
- List<ProxyConfig.Proxy> proxy = new LinkedList<>();
- return new ProxyConfig(proxy);
+ private ProxyConfig getProxyConfig(boolean isSecure) {
+ List<ProxyConfig.Proxy> proxies = new LinkedList<>();
+
+ String httpProxy = StringUtils.isBlank(System.getenv("http_proxy")) ?
+ System.getenv("HTTP_PROXY") : System.getenv("http_proxy");
+
+ String httpsProxy = StringUtils.isBlank(System.getenv("https_proxy")) ?
+ System.getenv("HTTPS_PROXY") : System.getenv("https_proxy");
+
+ try {
+ if (isSecure)
+ proxies.add(generateProxy("secure", new URI(httpsProxy)));
+ else proxies.add(generateProxy("insecure", new URI(httpProxy)));
+ } catch (Exception ex) {
+ logger.error(ex.getMessage(), ex);
+ }
+ return new ProxyConfig(proxies);
+ }
+
+ private ProxyConfig.Proxy generateProxy(String proxyId, URI uri) {
+
+ String protocol = uri.getScheme();
+ String host = uri.getHost();
+ int port = uri.getPort() <= 0 ? 80 : uri.getPort();
+
+ String username = null, password = null;
+ if (uri.getUserInfo() != null) {
+ String[] authority = uri.getUserInfo().split(":");
+ if (authority.length == 2) {
+ username = authority[0];
+ password = authority[1];
+ } else if (authority.length == 1) {
+ username = authority[0];
+ }
+ }
+ String nonProxyHosts = StringUtils.isBlank(System.getenv("no_proxy")) ?
+ System.getenv("NO_PROXY") : System.getenv("no_proxy");
+ return new ProxyConfig.Proxy(proxyId, protocol, host, port, username, password, nonProxyHosts);
+ }
+
+ private boolean isSecure(String url) {
+ return url.toLowerCase().startsWith("https");
}
public void buildAllPackages(List<HeliumPackage> pkgs) throws IOException {
@@ -609,7 +662,8 @@ public class HeliumBundleFactory {
}
private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
- NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl);
+ NpmRunner npm = frontEndPluginFactory.getNpmRunner(
+ getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl);
npm.execute(args, env);
}
@@ -623,7 +677,8 @@ public class HeliumBundleFactory {
private void yarnCommand(FrontendPluginFactory fpf,
String args, Map<String, String> env) throws TaskRunnerException {
- YarnRunner yarn = fpf.getYarnRunner(getProxyConfig(), defaultNpmRegistryUrl);
+ YarnRunner yarn = fpf.getYarnRunner(
+ getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl);
yarn.execute(args, env);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java
index b1c3a83..165d70a 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java
@@ -19,6 +19,8 @@ package org.apache.zeppelin.helium;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
@@ -28,7 +30,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
-import java.net.URL;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -67,6 +70,7 @@ public class HeliumOnlineRegistry extends HeliumRegistry {
public synchronized List<HeliumPackage> getAll() throws IOException {
HttpClient client = HttpClientBuilder.create()
.setUserAgent("ApacheZeppelin/" + Util.getVersion())
+ .setProxy(getProxy(uri()))
.build();
HttpGet get = new HttpGet(uri());
HttpResponse response;
@@ -78,7 +82,6 @@ public class HeliumOnlineRegistry extends HeliumRegistry {
return readFromCache();
}
-
if (response.getStatusLine().getStatusCode() != 200) {
// try read from cache
logger.error(uri() + " returned " + response.getStatusLine().toString());
@@ -107,6 +110,31 @@ public class HeliumOnlineRegistry extends HeliumRegistry {
}
}
+ private HttpHost getProxy(String uri) {
+ String httpProxy = StringUtils.isBlank(System.getenv("http_proxy")) ?
+ System.getenv("HTTP_PROXY") : System.getenv("http_proxy");
+
+ String httpsProxy = StringUtils.isBlank(System.getenv("https_proxy")) ?
+ System.getenv("HTTPS_PROXY") : System.getenv("https_proxy");
+
+ try {
+ String scheme = new URI(uri).getScheme();
+ if (scheme.toLowerCase().startsWith("https")) {
+ URI httpsProxyUri = new URI(httpsProxy);
+ return new HttpHost(httpsProxyUri.getHost(),
+ httpsProxyUri.getPort(), httpsProxyUri.getScheme());
+ }
+ else {
+ URI httpProxyUri = new URI(httpProxy);
+ return new HttpHost(httpProxyUri.getHost(),
+ httpProxyUri.getPort(), httpProxyUri.getScheme());
+ }
+ } catch (URISyntaxException ex) {
+ logger.error(ex.getMessage(), ex);
+ return null;
+ }
+ }
+
private List<HeliumPackage> readFromCache() {
synchronized (registryCacheFile) {
if (registryCacheFile.isFile()) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
index 5feac1d..aaabec5 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
@@ -39,7 +39,7 @@ public class HeliumBundleFactoryTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private HeliumBundleFactory hbf;
- static File nodeInstallationDir = new File(
+ private static File nodeInstallationDir = new File(
System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_nodeCache");
@BeforeClass