You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2017/03/22 23:22:08 UTC
zeppelin git commit: [ZEPPELIN-2278] Env variable to configure Npm
registry
Repository: zeppelin
Updated Branches:
refs/heads/master 405cb44cf -> 621c5be2d
[ZEPPELIN-2278] Env variable to configure Npm registry
### What is this PR for?
When deploying Zeppelin in private Cloud or within closed networks, it happens that Npm Registry isn't directly reachable, so we want to configure our own proxy/registry otherwise we couldn't fetch dependencies. It is anyhow worth to let user configure npm registry instead of hardcoding the URL.
### What type of PR is it?
Improvement
### What is the Jira issue?
[ZEPPELIN-2278]
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?
Author: Andrea Peruffo <an...@gmail.com>
Closes #2150 from andreaTP/npmConfig and squashes the following commits:
56fab25 [Andrea Peruffo] [ZEPPELIN-2278] Env variable to configure Npm registry
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/621c5be2
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/621c5be2
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/621c5be2
Branch: refs/heads/master
Commit: 621c5be2d138a1771f53d0a21553880c48351ee0
Parents: 405cb44
Author: Andrea Peruffo <an...@gmail.com>
Authored: Fri Mar 17 10:59:44 2017 +0100
Committer: Lee moon soo <mo...@apache.org>
Committed: Wed Mar 22 16:22:05 2017 -0700
----------------------------------------------------------------------
conf/zeppelin-env.cmd.template | 1 +
conf/zeppelin-env.sh.template | 1 +
conf/zeppelin-site.xml.template | 6 ++++++
docs/install/configuration.md | 8 +++++++-
.../org/apache/zeppelin/server/ZeppelinServer.java | 3 ++-
.../apache/zeppelin/conf/ZeppelinConfiguration.java | 5 +++++
.../apache/zeppelin/helium/HeliumBundleFactory.java | 16 ++++++++++++----
.../zeppelin/helium/HeliumBundleFactoryTest.java | 8 +++++++-
8 files changed, 41 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/conf/zeppelin-env.cmd.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template
index 548a798..cd1541e 100644
--- a/conf/zeppelin-env.cmd.template
+++ b/conf/zeppelin-env.cmd.template
@@ -39,6 +39,7 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading
+REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth?
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index 5e32b15..754c8ef 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -44,6 +44,7 @@
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading
+# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index d172942..6ce764b 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -252,6 +252,12 @@
</property>
<property>
+ <name>zeppelin.helium.npm.registry</name>
+ <value>http://registry.npmjs.org/</value>
+ <description>Remote Npm registry for Helium dependency loader</description>
+</property>
+
+<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,or
g.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/docs/install/configuration.md
----------------------------------------------------------------------
diff --git a/docs/install/configuration.md b/docs/install/configuration.md
index ac41e6f..a35eb91 100644
--- a/docs/install/configuration.md
+++ b/docs/install/configuration.md
@@ -44,7 +44,7 @@ If both are defined, then the **environment variables** will take priority.
<td><h6 class="properties">zeppelin.server.port</h6></td>
<td>8080</td>
<td>Zeppelin server port </br>
- <span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with
+ <span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with
<a href="https://zeppelin.apache.org/contribution/webapplication.html#dev-mode" target="_blank">Zeppelin web application development port</a> (default: 9000).</span></td>
</tr>
<tr>
@@ -282,6 +282,12 @@ If both are defined, then the **environment variables** will take priority.
<td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td>
</tr>
<tr>
+ <td><h6 class="properties">ZEPPELIN_HELIUM_NPM_REGISTRY</h6></td>
+ <td><h6 class="properties">zeppelin.helium.npm.registry</h6></td>
+ <td>http://registry.npmjs.org/</td>
+ <td>Remote Npm registry for Helium dependency loader</td>
+ </tr>
+ <tr>
<td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td>
<td><h6 class="properties">zeppelin.websocket.max.text.message.size</h6></td>
<td>1024000</td>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 768a154..ce38e46 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -114,12 +114,14 @@ public class ZeppelinServer extends Application {
* packaged into binary package.
*/
heliumBundleFactory = new HeliumBundleFactory(
+ conf,
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-spell")));
} else {
heliumBundleFactory = new HeliumBundleFactory(
+ conf,
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")),
new File(conf.getRelativeDir("zeppelin-web/src/app/visualization")),
@@ -410,4 +412,3 @@ public class ZeppelinServer extends Application {
return !new File(conf.getRelativeDir("zeppelin-web")).isDirectory();
}
}
-
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index facf89a..55f502c 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -424,6 +424,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
}
+ public String getHeliumNpmRegistry() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_NPM_REGISTRY);
+ }
+
public String getNotebookAuthorizationPath() {
return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir()));
}
@@ -633,6 +637,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
+ ZEPPELIN_HELIUM_NPM_REGISTRY("zeppelin.helium.npm.registry", "http://registry.npmjs.org/"),
// Allows a way to specify a ',' separated list of allowed origins for rest and websockets
// i.e. http://localhost:8080
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
index e04f9ac..c776506 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
@@ -33,6 +33,8 @@ import java.io.*;
import java.net.URL;
import java.util.*;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+
/**
* Load helium visualization & spell
*/
@@ -40,7 +42,6 @@ public class HeliumBundleFactory {
Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);
private final String NODE_VERSION = "v6.9.1";
private final String NPM_VERSION = "3.10.8";
- private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/";
public static final String HELIUM_LOCAL_REPO = "helium-bundle";
public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
public static final String HELIUM_BUNDLE = "helium.bundle.js";
@@ -52,9 +53,11 @@ public class HeliumBundleFactory {
private final FrontendPluginFactory frontEndPluginFactory;
private final File workingDirectory;
+ private ZeppelinConfiguration conf;
private File tabledataModulePath;
private File visualizationModulePath;
private File spellModulePath;
+ private String defaultNpmRegistryUrl;
private Gson gson;
private boolean nodeAndNpmInstalled = false;
@@ -64,18 +67,23 @@ public class HeliumBundleFactory {
ByteArrayOutputStream out = new ByteArrayOutputStream();
public HeliumBundleFactory(
+ ZeppelinConfiguration conf,
File moduleDownloadPath,
File tabledataModulePath,
File visualizationModulePath,
File spellModulePath) throws TaskRunnerException {
- this(moduleDownloadPath);
+ this(conf, moduleDownloadPath);
this.tabledataModulePath = tabledataModulePath;
this.visualizationModulePath = visualizationModulePath;
this.spellModulePath = spellModulePath;
}
- public HeliumBundleFactory(File moduleDownloadPath) throws TaskRunnerException {
+ public HeliumBundleFactory(
+ ZeppelinConfiguration conf,
+ File moduleDownloadPath) throws TaskRunnerException {
this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
+ this.conf = conf;
+ this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
File installDirectory = workingDirectory;
frontEndPluginFactory = new FrontendPluginFactory(
@@ -412,7 +420,7 @@ public class HeliumBundleFactory {
private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
installNodeAndNpm();
- NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL);
+ NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl);
npm.execute(args, env);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/621c5be2/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
index 28cba4c..7b816e6 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
@@ -30,10 +30,13 @@ import java.net.URL;
import java.util.LinkedList;
import java.util.List;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+
import static org.junit.Assert.*;
public class HeliumBundleFactoryTest {
private File tmpDir;
+ private ZeppelinConfiguration conf;
private HeliumBundleFactory hbf;
@Before
@@ -46,7 +49,10 @@ public class HeliumBundleFactoryTest {
String resDir = new File(res.getFile()).getParent();
File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/");
- hbf = new HeliumBundleFactory(tmpDir,
+ conf = new ZeppelinConfiguration();
+
+ hbf = new HeliumBundleFactory(conf,
+ tmpDir,
new File(moduleDir, "tabledata"),
new File(moduleDir, "visualization"),
new File(moduleDir, "spell"));