You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2017/04/08 22:48:36 UTC
zeppelin git commit: [ZEPPELIN-2088] Bundle helium packages one by one
Repository: zeppelin
Updated Branches:
refs/heads/master 495be1ede -> 241fd0344
[ZEPPELIN-2088] Bundle helium packages one by one
### What is this PR for?
Bundle helium packages one by one because
#### Summary
Let's say you have one helium package called `ultimate-line-chart`.
**1. Package Path**
- **(before)** `$ZEPPELIN_HOME/local-repo/helium-bundles/node_modules/ultimate-line-chart`
- **(after)** `$ZEPPELIN_HOME/local-repo/helium-bundles/bundles/ultimate-line-chart`
**2. Zeppelin Local Module Path**
- **(before)** `$ZEPPELIN_HOME/local-repo/helium-bundles/node_modules/zeppelin-tabledata`
- **(after)** `$ZEPPELIN_HOME/local-repo/helium-bundles/local_modules/zeppelin-tabledata`
**3. Bundle Cache Path**
- **(before)** `$ZEPPELIN_HOME/local-repo/helium-bundles/helium.bundle.cache.js` (mixed one)
- **(after)** `$ZEPPELIN_HOME/local-repo/helium-bundles/bundles/ultimate-line-chart/helium.bundle.cache.js` (for each helium package)
#### Details
- Bundling them in one file is not good idea because the whole bundling is broken when only 1 bundle fails
- Some node packages might not installed correctly. For example, `amchart/amcharts3` will be installed in `helium-bundles/node_modules/my-helium-vis/node_modules` but `amchart-export` will be placed in `helium-bundles/node_modules` so `my-helium-vis` can't import `amchart-export`. (This is real case)
- Additionally, I used yarn to install required node_modules. separated bundling requires install same dependencies multiple times. This takes more time than before and npm is too slow to install them all.
- Install Zeppelin framework modules using [local path](https://docs.npmjs.com/files/package.json#local-paths). It's more safe and correct way to install local modules.
### What type of PR is it?
[Improvement]
### Todos
NONE
### What is the Jira issue?
[ZEPPELIN-2088](https://issues.apache.org/jira/browse/ZEPPELIN-2088)
### How should this be tested?
1. Install any local helium package
2. Install any online helium package
3. Use them
4. Disable
5. Test `npm run dev:helium`
### Screenshots (if appropriate)
NONE
### Questions:
* Does the licenses files need update? - NO
* Is there breaking changes for older versions? - NO
* Does this needs documentation? - NO
Author: 1ambda <1a...@gmail.com>
Closes #2210 from 1ambda/ZEPPELIN-2088/evaluate-helium-bundle-one-by-one and squashes the following commits:
bea6c09 [1ambda] fix: Bundling errors
53145fa [1ambda] fix: Add error handling for failed bundle
212c737 [1ambda] feat: Apply #2178
736acee [1ambda] fix: Add nodeInstallationDir
11ef0ae [1ambda] fix: HeliumBundleFactoryTest
5a2a938 [1ambda] fix: Log single bundle error to browser console
a99f981 [1ambda] test: Fix APIs
743aba4 [1ambda] feat: Loading bundles
697c5e6 [1ambda] feat: enable, disable
002e66f [1ambda] feat: Build online packages
512508d [1ambda] feat: Build each bundle using yarn
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/241fd034
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/241fd034
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/241fd034
Branch: refs/heads/master
Commit: 241fd0344da3f6fbda7ba8d0976a85a51e3c33e9
Parents: 495be1e
Author: 1ambda <1a...@gmail.com>
Authored: Tue Apr 4 17:59:14 2017 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun Apr 9 07:48:31 2017 +0900
----------------------------------------------------------------------
.../org/apache/zeppelin/rest/HeliumRestApi.java | 50 +-
.../apache/zeppelin/server/ZeppelinServer.java | 4 +-
.../src/components/helium/helium.service.js | 100 ++--
zeppelin-zengine/pom.xml | 5 +
.../java/org/apache/zeppelin/helium/Helium.java | 36 +-
.../zeppelin/helium/HeliumBundleFactory.java | 479 +++++++++++++------
.../zeppelin/helium/HeliumLocalRegistry.java | 1 -
.../src/main/resources/helium/package.json | 8 +-
.../src/main/resources/helium/webpack.config.js | 55 +--
.../helium/HeliumBundleFactoryTest.java | 45 +-
10 files changed, 513 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
index 9234cc5..44b583c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
@@ -24,6 +24,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumPackage;
+import org.apache.zeppelin.helium.HeliumPackageSearchResult;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.Paragraph;
@@ -69,6 +70,16 @@ public class HeliumRestApi {
}
/**
+ * Get all enabled package infos
+ */
+ @GET
+ @Path("enabledPackage")
+ public Response getAllEnabledPackageInfo() {
+ return new JsonResponse(
+ Response.Status.OK, "", helium.getAllEnabledPackages()).build();
+ }
+
+ /**
* Get single package info
*/
@GET
@@ -130,26 +141,43 @@ public class HeliumRestApi {
}
@GET
- @Path("bundle/load")
+ @Path("bundle/load/{packageName}")
@Produces("text/javascript")
- public Response bundleLoad(@QueryParam("refresh") String refresh) {
+ public Response bundleLoad(@QueryParam("refresh") String refresh,
+ @PathParam("packageName") String packageName) {
+ if (StringUtils.isEmpty(packageName)) {
+ return new JsonResponse(
+ Response.Status.BAD_REQUEST,
+ "Can't get bundle due to empty package name").build();
+ }
+
+ HeliumPackageSearchResult psr = null;
+ List<HeliumPackageSearchResult> enabledPackages = helium.getAllEnabledPackages();
+ for (HeliumPackageSearchResult e : enabledPackages) {
+ if (e.getPkg().getName().equals(packageName)) {
+ psr = e;
+ break;
+ }
+ }
+
+ if (psr == null) {
+ // return empty to specify
+ return Response.ok().build();
+ }
+
try {
File bundle;
- if (refresh != null && refresh.equals("true")) {
- bundle = helium.recreateBundle();
- } else {
- bundle = helium.getBundleFactory().getCurrentCacheBundle();
- }
+ boolean rebuild = (refresh != null && refresh.equals("true"));
+ bundle = helium.getBundle(psr.getPkg(), rebuild);
if (bundle == null) {
return Response.ok().build();
} else {
- String stringifiedBundle = FileUtils.readFileToString(bundle);
- return Response.ok(stringifiedBundle).build();
+ String stringified = FileUtils.readFileToString(bundle);
+ return Response.ok(stringified).build();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
-
// returning error will prevent zeppelin front-end render any notebook.
// visualization load fail doesn't need to block notebook rendering work.
// so it's better return ok instead of any error.
@@ -172,7 +200,7 @@ public class HeliumRestApi {
@POST
@Path("disable/{packageName}")
- public Response enablePackage(@PathParam("packageName") String packageName) {
+ public Response disablePackage(@PathParam("packageName") String packageName) {
try {
helium.disable(packageName);
return new JsonResponse(Response.Status.OK).build();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index ce38e46..fe2823c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -115,6 +115,7 @@ public class ZeppelinServer extends Application {
*/
heliumBundleFactory = new HeliumBundleFactory(
conf,
+ null,
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")),
@@ -122,6 +123,7 @@ public class ZeppelinServer extends Application {
} else {
heliumBundleFactory = new HeliumBundleFactory(
conf,
+ null,
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")),
new File(conf.getRelativeDir("zeppelin-web/src/app/visualization")),
@@ -138,7 +140,7 @@ public class ZeppelinServer extends Application {
// create bundle
try {
- heliumBundleFactory.buildBundle(helium.getBundlePackagesToBundle());
+ heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-web/src/components/helium/helium.service.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/components/helium/helium.service.js b/zeppelin-web/src/components/helium/helium.service.js
index 8b3e6a3..34520e1 100644
--- a/zeppelin-web/src/components/helium/helium.service.js
+++ b/zeppelin-web/src/components/helium/helium.service.js
@@ -27,41 +27,14 @@ angular.module('zeppelinWebApp').service('heliumService', heliumService);
export default function heliumService($http, $sce, baseUrlSrv) {
'ngInject';
- var url = baseUrlSrv.getRestApiBase() + '/helium/bundle/load';
- if (process.env.HELIUM_BUNDLE_DEV) {
- url = url + '?refresh=true';
- }
-
let visualizationBundles = [];
- // name `heliumBundles` should be same as `HelumBundleFactory.HELIUM_BUNDLES_VAR`
+ // name `heliumBundles` should be same as `HeliumBundleFactory.HELIUM_BUNDLES_VAR`
let heliumBundles = [];
// map for `{ magic: interpreter }`
let spellPerMagic = {};
// map for `{ magic: package-name }`
let pkgNamePerMagic = {}
- // load should be promise
- this.load = $http.get(url).success(function(response) {
- if (response.substring(0, 'ERROR:'.length) !== 'ERROR:') {
- // evaluate bundles
- eval(response);
-
- // extract bundles by type
- heliumBundles.map(b => {
- if (b.type === HeliumType.SPELL) {
- const spell = new b.class(); // eslint-disable-line new-cap
- const pkgName = b.id;
- spellPerMagic[spell.getMagic()] = spell;
- pkgNamePerMagic[spell.getMagic()] = pkgName;
- } else if (b.type === HeliumType.VISUALIZATION) {
- visualizationBundles.push(b);
- }
- });
- } else {
- console.error(response);
- }
- });
-
/**
* @param magic {string} e.g `%flowchart`
* @returns {SpellBase} undefined if magic is not registered
@@ -160,6 +133,37 @@ export default function heliumService($http, $sce, baseUrlSrv) {
});
};
+ this.getAllEnabledPackages = function() {
+ return $http.get(`${baseUrlSrv.getRestApiBase()}/helium/enabledPackage`)
+ .then(function(response, status) {
+ return response.data.body;
+ })
+ .catch(function(error) {
+ console.error('Failed to get all enabled package infos', error);
+ });
+ };
+
+ this.getSingleBundle = function(pkgName) {
+ let url = `${baseUrlSrv.getRestApiBase()}/helium/bundle/load/${pkgName}`
+ if (process.env.HELIUM_BUNDLE_DEV) {
+ url = url + '?refresh=true';
+ }
+
+ return $http.get(url)
+ .then(function(response, status) {
+ const bundle = response.data
+ if (bundle.substring(0, 'ERROR:'.length) === 'ERROR:') {
+ console.error(`Failed to get bundle: ${pkgName}`, bundle);
+ return '' // empty bundle will be filtered later
+ }
+
+ return bundle
+ })
+ .catch(function(error) {
+ console.error(`Failed to get single bundle: ${pkgName}`, error);
+ });
+ }
+
this.getDefaultPackages = function() {
return this.getAllPackageInfo()
.then(pkgSearchResults => {
@@ -241,4 +245,44 @@ export default function heliumService($http, $sce, baseUrlSrv) {
return merged;
});
}
+
+ const p = this.getAllEnabledPackages()
+ .then(enabledPackageSearchResults => {
+ const promises = enabledPackageSearchResults.map(packageSearchResult => {
+ const pkgName = packageSearchResult.pkg.name
+ return this.getSingleBundle(pkgName)
+ })
+
+ return Promise.all(promises)
+ })
+ .then(bundles => {
+ return bundles.reduce((acc, b) => {
+ // filter out empty bundle
+ if (b === '') { return acc }
+ acc.push(b)
+ return acc;
+ }, [])
+ })
+
+ // load should be promise
+ this.load = p.then(availableBundles => {
+
+ // evaluate bundles
+ availableBundles.map(b => {
+ eval(b)
+ })
+
+ // extract bundles by type
+ heliumBundles.map(b => {
+ if (b.type === HeliumType.SPELL) {
+ const spell = new b.class() // eslint-disable-line new-cap
+ const pkgName = b.id
+ spellPerMagic[spell.getMagic()] = spell
+ pkgNamePerMagic[spell.getMagic()] = pkgName
+ } else if (b.type === HeliumType.VISUALIZATION) {
+ visualizationBundles.push(b)
+ }
+ })
+ })
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index d5fef16..6589e3b 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -290,6 +290,11 @@
<artifactId>mongo-java-driver</artifactId>
<version>3.4.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.5</version>
+ </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
index b4d5a79..f5f6695 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
@@ -242,6 +242,22 @@ public class Helium {
}
}
+ public List<HeliumPackageSearchResult> getAllEnabledPackages() {
+ Map<String, List<HeliumPackageSearchResult>> allPackages = getAllPackageInfoWithoutRefresh();
+ List<HeliumPackageSearchResult> enabledPackages = new ArrayList<>();
+
+ for (List<HeliumPackageSearchResult> versionedPackages : allPackages.values()) {
+ for (HeliumPackageSearchResult psr : versionedPackages) {
+ if (psr.isEnabled()) {
+ enabledPackages.add(psr);
+ break;
+ }
+ }
+ }
+
+ return enabledPackages;
+ }
+
public List<HeliumPackageSearchResult> getSinglePackageInfo(String packageName) {
Map<String, List<HeliumPackageSearchResult>> result = getAllPackageInfo(false, packageName);
@@ -281,8 +297,8 @@ public class Helium {
return null;
}
- public File recreateBundle() throws IOException {
- return bundleFactory.buildBundle(getBundlePackagesToBundle(), true);
+ public File getBundle(HeliumPackage pkg, boolean rebuild) throws IOException {
+ return bundleFactory.buildPackage(pkg, rebuild, true);
}
public void enable(String name, String artifact) throws IOException {
@@ -293,14 +309,13 @@ public class Helium {
return;
}
- // enable package
- heliumConf.enablePackage(name, artifact);
-
// if package is visualization, rebuild bundle
if (HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
- bundleFactory.buildBundle(getBundlePackagesToBundle());
+ bundleFactory.buildPackage(pkgInfo.getPkg(), true, true);
}
+ // update conf and save
+ heliumConf.enablePackage(name, artifact);
save();
}
@@ -311,13 +326,8 @@ public class Helium {
return;
}
+ // update conf and save
heliumConf.disablePackage(name);
-
- HeliumPackageSearchResult pkgInfo = getPackageInfo(name, artifact);
- if (pkgInfo == null || HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
- bundleFactory.buildBundle(getBundlePackagesToBundle());
- }
-
save();
}
@@ -445,7 +455,7 @@ public class Helium {
heliumConf.setBundleDisplayOrder(orderedPackageList);
// if package is visualization, rebuild buildBundle
- bundleFactory.buildBundle(getBundlePackagesToBundle());
+ bundleFactory.buildAllPackages(getBundlePackagesToBundle());
save();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
index 2dbefe0..d0e9b00 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
@@ -17,10 +17,18 @@
package org.apache.zeppelin.helium;
import com.github.eirslett.maven.plugins.frontend.lib.*;
+
import com.google.common.base.Charsets;
import com.google.common.io.Resources;
import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.google.gson.stream.JsonReader;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.log4j.Appender;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
@@ -30,7 +38,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
-import java.net.URL;
import java.util.*;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -42,17 +49,25 @@ public class HeliumBundleFactory {
Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);
private final String NODE_VERSION = "v6.9.1";
private final String NPM_VERSION = "3.10.8";
+ private final String YARN_VERSION = "v0.21.3";
public static final String HELIUM_LOCAL_REPO = "helium-bundle";
+ public static final String HELIUM_BUNDLES_DIR = "bundles";
+ public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
+ public static final String HELIUM_BUNDLES_SRC_DIR = "src";
+ public static final String HELIUM_BUNDLES_SRC = "load.js";
+ public static final String PACKAGE_JSON = "package.json";
public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
public static final String HELIUM_BUNDLE = "helium.bundle.js";
public static final String HELIUM_BUNDLES_VAR = "heliumBundles";
private final int FETCH_RETRY_COUNT = 2;
private final int FETCH_RETRY_FACTOR_COUNT = 1;
- // Milliseconds
- private final int FETCH_RETRY_MIN_TIMEOUT = 5000;
+ private final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds
private final FrontendPluginFactory frontEndPluginFactory;
- private final File workingDirectory;
+ private final File nodeInstallationDirectory;
+ private final File heliumLocalRepoDirectory;
+ private final File heliumBundleDirectory;
+ private final File heliumLocalModuleDirectory;
private ZeppelinConfiguration conf;
private File tabledataModulePath;
private File visualizationModulePath;
@@ -61,18 +76,16 @@ public class HeliumBundleFactory {
private Gson gson;
private boolean nodeAndNpmInstalled = false;
- String bundleCacheKey = "";
- File currentCacheBundle;
-
ByteArrayOutputStream out = new ByteArrayOutputStream();
public HeliumBundleFactory(
ZeppelinConfiguration conf,
+ File nodeInstallationDir,
File moduleDownloadPath,
File tabledataModulePath,
File visualizationModulePath,
File spellModulePath) throws TaskRunnerException {
- this(conf, moduleDownloadPath);
+ this(conf, nodeInstallationDir, moduleDownloadPath);
this.tabledataModulePath = tabledataModulePath;
this.visualizationModulePath = visualizationModulePath;
this.spellModulePath = spellModulePath;
@@ -80,16 +93,20 @@ public class HeliumBundleFactory {
public HeliumBundleFactory(
ZeppelinConfiguration conf,
+ File nodeInstallationDir,
File moduleDownloadPath) throws TaskRunnerException {
- this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
+ this.heliumLocalRepoDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
+ this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR);
+ this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
this.conf = conf;
this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
- File installDirectory = workingDirectory;
+
+ nodeInstallationDirectory = (nodeInstallationDir == null) ?
+ heliumLocalRepoDirectory : nodeInstallationDir;
frontEndPluginFactory = new FrontendPluginFactory(
- workingDirectory, installDirectory);
+ heliumLocalRepoDirectory, nodeInstallationDirectory);
- currentCacheBundle = new File(workingDirectory, HELIUM_BUNDLE_CACHE);
gson = new Gson();
}
@@ -98,13 +115,18 @@ public class HeliumBundleFactory {
return;
}
try {
+ NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
+ nodeInstaller.setNodeVersion(NODE_VERSION);
+ nodeInstaller.install();
+
NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig());
npmInstaller.setNpmVersion(NPM_VERSION);
npmInstaller.install();
- NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
- nodeInstaller.setNodeVersion(NODE_VERSION);
- nodeInstaller.install();
+ YarnInstaller yarnInstaller = frontEndPluginFactory.getYarnInstaller(getProxyConfig());
+ yarnInstaller.setYarnVersion(YARN_VERSION);
+ yarnInstaller.install();
+
configureLogger();
nodeAndNpmInstalled = true;
} catch (InstallationException e) {
@@ -117,33 +139,60 @@ public class HeliumBundleFactory {
return new ProxyConfig(proxy);
}
- public File buildBundle(List<HeliumPackage> pkgs) throws IOException {
- return buildBundle(pkgs, false);
+ public void buildAllPackages(List<HeliumPackage> pkgs) throws IOException {
+ buildAllPackages(pkgs, false);
}
- public synchronized File buildBundle(List<HeliumPackage> pkgs, boolean forceRefresh)
- throws IOException {
+ public File getHeliumPackageDirectory(String pkgName) {
+ return new File(heliumBundleDirectory, pkgName);
+ }
- if (pkgs == null || pkgs.size() == 0) {
- // when no package is selected, simply return an empty file instead of try bundle package
- synchronized (this) {
- currentCacheBundle.getParentFile().mkdirs();
- currentCacheBundle.delete();
- currentCacheBundle.createNewFile();
- bundleCacheKey = "";
- return currentCacheBundle;
+ public File getHeliumPackageSourceDirectory(String pkgName) {
+ return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLES_SRC_DIR);
+ }
+
+ public File getHeliumPackageBundleCache(String pkgName) {
+ return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLE_CACHE);
+ }
+
+ public static List<String> unTgz(File tarFile, File directory) throws IOException {
+ List<String> result = new ArrayList<String>();
+ InputStream is = new FileInputStream(tarFile);
+ GzipCompressorInputStream gcis = new GzipCompressorInputStream(is);
+ TarArchiveInputStream in = new TarArchiveInputStream(gcis);
+ TarArchiveEntry entry = in.getNextTarEntry();
+ while (entry != null) {
+ if (entry.isDirectory()) {
+ entry = in.getNextTarEntry();
+ continue;
}
+ File curfile = new File(directory, entry.getName());
+ File parent = curfile.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ OutputStream out = new FileOutputStream(curfile);
+ IOUtils.copy(in, out);
+ out.close();
+ result.add(entry.getName());
+ entry = in.getNextTarEntry();
}
+ in.close();
+ return result;
+ }
- installNodeAndNpm();
-
- // package.json
- URL pkgUrl = Resources.getResource("helium/package.json");
- String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8);
- StringBuilder dependencies = new StringBuilder();
- StringBuilder cacheKeyBuilder = new StringBuilder();
+ /**
+ * @return main file name of this helium package (relative path)
+ */
+ public String downloadPackage(HeliumPackage pkg, String[] nameAndVersion, File bundleDir,
+ String templateWebpackConfig, String templatePackageJson,
+ FrontendPluginFactory fpf) throws IOException, TaskRunnerException {
+ if (bundleDir.exists()) {
+ FileUtils.deleteQuietly(bundleDir);
+ }
+ FileUtils.forceMkdir(bundleDir);
- FileFilter npmPackageCopyFilter = new FileFilter() {
+ FileFilter copyFilter = new FileFilter() {
@Override
public boolean accept(File pathname) {
String fileName = pathname.getName();
@@ -155,73 +204,97 @@ public class HeliumBundleFactory {
}
};
- for (HeliumPackage pkg : pkgs) {
- String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
- if (moduleNameVersion == null) {
- logger.error("Can't get module name and version of package " + pkg.getName());
- continue;
- }
- if (dependencies.length() > 0) {
- dependencies.append(",\n");
- }
- dependencies.append("\"" + moduleNameVersion[0] + "\": \"" + moduleNameVersion[1] + "\"");
- cacheKeyBuilder.append(pkg.getName() + pkg.getArtifact());
+ if (isLocalPackage(pkg)) {
+ FileUtils.copyDirectory(
+ new File(pkg.getArtifact()),
+ bundleDir,
+ copyFilter);
+ } else {
+ // if online package
+ String version = nameAndVersion[1];
+ File tgz = new File(heliumLocalRepoDirectory, pkg.getName() + "-" + version + ".tgz");
+ tgz.delete();
+
+ // wget, extract and move dir to `bundles/${pkg.getName()}`, and remove tgz
+ npmCommand(fpf, "pack " + pkg.getArtifact());
+ File extracted = new File(heliumBundleDirectory, "package");
+ FileUtils.deleteDirectory(extracted);
+ unTgz(tgz, heliumBundleDirectory);
+ tgz.delete();
+ FileUtils.copyDirectory(extracted, bundleDir);
+ FileUtils.deleteDirectory(extracted);
+ }
+
+ // 1. setup package.json
+ File existingPackageJson = new File(bundleDir, "package.json");
+ JsonReader reader = new JsonReader(new FileReader(existingPackageJson));
+ Map<String, Object> packageJson = gson.fromJson(reader,
+ new TypeToken<Map<String, Object>>(){}.getType());
+ Map<String, String> existingDeps = (Map<String, String>) packageJson.get("dependencies");
+ String mainFileName = (String) packageJson.get("main");
- File pkgInstallDir = new File(workingDirectory, "node_modules/" + pkg.getName());
- if (pkgInstallDir.exists()) {
- FileUtils.deleteDirectory(pkgInstallDir);
+ StringBuilder dependencies = new StringBuilder();
+ int index = 0;
+ for (Map.Entry<String, String> e: existingDeps.entrySet()) {
+ dependencies.append(" \"").append(e.getKey()).append("\": ");
+ if (e.getKey().equals("zeppelin-vis") ||
+ e.getKey().equals("zeppelin-tabledata") ||
+ e.getKey().equals("zeppelin-spell")) {
+ dependencies.append("\"file:../../" + HELIUM_LOCAL_MODULE_DIR + "/")
+ .append(e.getKey()).append("\"");
+ } else {
+ dependencies.append("\"").append(e.getValue()).append("\"");
}
- if (isLocalPackage(pkg)) {
- FileUtils.copyDirectory(
- new File(pkg.getArtifact()),
- pkgInstallDir,
- npmPackageCopyFilter);
+ if (index < existingDeps.size() - 1) {
+ dependencies.append(",\n");
}
+ index = index + 1;
}
- pkgJson = pkgJson.replaceFirst("DEPENDENCIES", dependencies.toString());
- // check if we can use previous buildBundle or not
- if (cacheKeyBuilder.toString().equals(bundleCacheKey) &&
- currentCacheBundle.isFile() && !forceRefresh) {
- return currentCacheBundle;
- }
+ FileUtils.deleteQuietly(new File(bundleDir, PACKAGE_JSON));
+ templatePackageJson = templatePackageJson.replaceFirst("PACKAGE_NAME", pkg.getName());
+ templatePackageJson = templatePackageJson.replaceFirst("MAIN_FILE", mainFileName);
+ templatePackageJson = templatePackageJson.replaceFirst("DEPENDENCIES", dependencies.toString());
+ FileUtils.write(new File(bundleDir, PACKAGE_JSON), templatePackageJson);
- // webpack.config.js
- URL webpackConfigUrl = Resources.getResource("helium/webpack.config.js");
- String webpackConfig = Resources.toString(webpackConfigUrl, Charsets.UTF_8);
+ // 2. setup webpack.config
+ FileUtils.write(new File(bundleDir, "webpack.config.js"), templateWebpackConfig);
- // generate load.js
+ return mainFileName;
+ }
+
+ public void prepareSource(HeliumPackage pkg, String[] moduleNameVersion,
+ String mainFileName) throws IOException {
StringBuilder loadJsImport = new StringBuilder();
StringBuilder loadJsRegister = new StringBuilder();
+ String className = "bundles" + pkg.getName().replaceAll("[-_]", "");
- long idx = 0;
- for (HeliumPackage pkg : pkgs) {
- String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
- if (moduleNameVersion == null) {
- continue;
- }
-
- String className = "bundles" + idx++;
- loadJsImport.append(
- "import " + className + " from \"" + moduleNameVersion[0] + "\"\n");
-
- loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
- loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
- loadJsRegister.append("name: \"" + pkg.getName() + "\",\n");
- loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n");
- loadJsRegister.append("type: \"" + pkg.getType() + "\",\n");
- loadJsRegister.append("class: " + className + "\n");
- loadJsRegister.append("})\n");
+ // remove postfix `.js` for ES6 import
+ if (mainFileName.endsWith(".js")) {
+ mainFileName = mainFileName.substring(0, mainFileName.length() - 3);
}
- FileUtils.write(new File(workingDirectory, "package.json"), pkgJson);
- FileUtils.write(new File(workingDirectory, "webpack.config.js"), webpackConfig);
- FileUtils.write(new File(workingDirectory, "load.js"),
- loadJsImport.append(loadJsRegister).toString());
-
- copyFrameworkModuleToInstallPath(npmPackageCopyFilter);
+ loadJsImport
+ .append("import ")
+ .append(className)
+ .append(" from \"../" + mainFileName + "\"\n");
+
+ loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
+ loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
+ loadJsRegister.append("name: \"" + pkg.getName() + "\",\n");
+ loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n");
+ loadJsRegister.append("type: \"" + pkg.getType() + "\",\n");
+ loadJsRegister.append("class: " + className + "\n");
+ loadJsRegister.append("})\n");
+
+ File srcDir = getHeliumPackageSourceDirectory(pkg.getName());
+ FileUtils.forceMkdir(srcDir);
+ FileUtils.write(new File(srcDir, HELIUM_BUNDLES_SRC),
+ loadJsImport.append(loadJsRegister).toString());
+ }
+ public synchronized void installNodeModules(FrontendPluginFactory fpf) throws IOException {
try {
out.reset();
String commandForNpmInstall =
@@ -229,112 +302,213 @@ public class HeliumBundleFactory {
"--fetch-retry-mintimeout=%d",
FETCH_RETRY_COUNT, FETCH_RETRY_FACTOR_COUNT, FETCH_RETRY_MIN_TIMEOUT);
logger.info("Installing required node modules");
- npmCommand(commandForNpmInstall);
+ yarnCommand(fpf, commandForNpmInstall);
logger.info("Installed required node modules");
} catch (TaskRunnerException e) {
- // ignore `(empty)` warning
- String cause = new String(out.toByteArray());
- if (!cause.contains("(empty)")) {
- throw new IOException(cause);
- }
+ throw new IOException(e);
}
+ }
+ public synchronized File bundleHeliumPackage(FrontendPluginFactory fpf,
+ File bundleDir) throws IOException {
try {
out.reset();
logger.info("Bundling helium packages");
- npmCommand("run bundle");
+ yarnCommand(fpf, "run bundle");
logger.info("Bundled helium packages");
} catch (TaskRunnerException e) {
throw new IOException(new String(out.toByteArray()));
}
String bundleStdoutResult = new String(out.toByteArray());
-
- File heliumBundle = new File(workingDirectory, HELIUM_BUNDLE);
+ File heliumBundle = new File(bundleDir, HELIUM_BUNDLE);
if (!heliumBundle.isFile()) {
throw new IOException(
- "Can't create bundle: \n" + bundleStdoutResult);
+ "Can't create bundle: \n" + bundleStdoutResult);
}
WebpackResult result = getWebpackResultFromOutput(bundleStdoutResult);
if (result.errors.length > 0) {
- heliumBundle.delete();
+ FileUtils.deleteQuietly(heliumBundle);
throw new IOException(result.errors[0]);
}
- synchronized (this) {
- currentCacheBundle.delete();
- FileUtils.moveFile(heliumBundle, currentCacheBundle);
- bundleCacheKey = cacheKeyBuilder.toString();
+ return heliumBundle;
+ }
+
+ public synchronized File buildPackage(HeliumPackage pkg,
+ boolean rebuild,
+ boolean recopyLocalModule) throws IOException {
+ if (pkg == null) {
+ return null;
+ }
+
+ String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
+ if (moduleNameVersion == null) {
+ return null;
+ }
+
+ if (moduleNameVersion == null) {
+ logger.error("Can't get module name and version of package " + pkg.getName());
+ return null;
+ }
+
+ String pkgName = pkg.getName();
+ File bundleDir = getHeliumPackageDirectory(pkgName);
+ File bundleCache = getHeliumPackageBundleCache(pkgName);
+
+ if (!rebuild && bundleCache.exists() && !bundleCache.isDirectory()) {
+ return bundleCache;
+ }
+
+ // 0. install node, npm (should be called before `downloadPackage`
+ installNodeAndNpm();
+
+ // 1. prepare directories
+ if (!heliumLocalRepoDirectory.exists() || !heliumLocalRepoDirectory.isDirectory()) {
+ FileUtils.deleteQuietly(heliumLocalRepoDirectory);
+ FileUtils.forceMkdir(heliumLocalRepoDirectory);
+ }
+ FrontendPluginFactory fpf = new FrontendPluginFactory(
+ bundleDir, nodeInstallationDirectory);
+
+ // resources: webpack.js, package.json
+ String templateWebpackConfig = Resources.toString(
+ Resources.getResource("helium/webpack.config.js"), Charsets.UTF_8);
+ String templatePackageJson = Resources.toString(
+ Resources.getResource("helium/" + PACKAGE_JSON), Charsets.UTF_8);
+
+ // 2. download helium package using `npm pack`
+ String mainFileName = null;
+ try {
+ mainFileName = downloadPackage(pkg, moduleNameVersion, bundleDir,
+ templateWebpackConfig, templatePackageJson, fpf);
+ } catch (TaskRunnerException e) {
+ throw new IOException(e);
+ }
+
+ // 3. prepare bundle source
+ prepareSource(pkg, moduleNameVersion, mainFileName);
+
+ // 4. install node and local modules for a bundle
+ copyFrameworkModuleToInstallPath(recopyLocalModule); // should copy local modules first
+ installNodeModules(fpf);
+
+ // 5. let's bundle and update cache
+ File heliumBundle = bundleHeliumPackage(fpf, bundleDir);
+ bundleCache.delete();
+ FileUtils.moveFile(heliumBundle, bundleCache);
+
+ return bundleCache;
+ }
+
+ public synchronized void buildAllPackages(List<HeliumPackage> pkgs, boolean rebuild)
+ throws IOException {
+
+ if (pkgs == null || pkgs.size() == 0) {
+ return;
+ }
+
+ // DON't recopy local modules when build all packages to avoid duplicated copies.
+ boolean recopyLocalModules = false;
+
+ for (HeliumPackage pkg : pkgs) {
+ try {
+ buildPackage(pkg, rebuild, recopyLocalModules);
+ } catch (IOException e) {
+ logger.error("Failed to build helium package: " + pkg.getArtifact(), e);
+ }
}
- return currentCacheBundle;
}
- private void copyFrameworkModuleToInstallPath(FileFilter npmPackageCopyFilter)
+ void copyFrameworkModuleToInstallPath(boolean recopy)
throws IOException {
+
+ FileFilter npmPackageCopyFilter = new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ String fileName = pathname.getName();
+ if (fileName.startsWith(".") || fileName.startsWith("#") || fileName.startsWith("~")) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+ };
+
// install tabledata module
- File tabledataModuleInstallPath = new File(workingDirectory,
- "node_modules/zeppelin-tabledata");
+ FileUtils.forceMkdir(heliumLocalModuleDirectory);
+
+ File tabledataModuleInstallPath = new File(heliumLocalModuleDirectory,
+ "zeppelin-tabledata");
if (tabledataModulePath != null) {
- if (tabledataModuleInstallPath.exists()) {
+ if (recopy && tabledataModuleInstallPath.exists()) {
FileUtils.deleteDirectory(tabledataModuleInstallPath);
+
+ }
+
+ if (!tabledataModuleInstallPath.exists()) {
+ FileUtils.copyDirectory(
+ tabledataModulePath,
+ tabledataModuleInstallPath,
+ npmPackageCopyFilter);
}
- FileUtils.copyDirectory(
- tabledataModulePath,
- tabledataModuleInstallPath,
- npmPackageCopyFilter);
}
// install visualization module
- File visModuleInstallPath = new File(workingDirectory,
- "node_modules/zeppelin-vis");
+ File visModuleInstallPath = new File(heliumLocalModuleDirectory,
+ "zeppelin-vis");
if (visualizationModulePath != null) {
- if (visModuleInstallPath.exists()) {
- // when zeppelin-vis and zeppelin-table package is published to npm repository
- // we don't need to remove module because npm install command will take care
- // dependency version change. However, when two dependencies are copied manually
- // into node_modules directory, changing vis package version results inconsistent npm
- // install behavior.
- //
- // Remote vis package everytime and let npm download every time bundle as a workaround
+ if (recopy && visModuleInstallPath.exists()) {
FileUtils.deleteDirectory(visModuleInstallPath);
}
- FileUtils.copyDirectory(visualizationModulePath, visModuleInstallPath, npmPackageCopyFilter);
+
+ if (!visModuleInstallPath.exists()) {
+ FileUtils.copyDirectory(
+ visualizationModulePath,
+ visModuleInstallPath,
+ npmPackageCopyFilter);
+ }
}
// install spell module
- File spellModuleInstallPath = new File(workingDirectory,
- "node_modules/zeppelin-spell");
+ File spellModuleInstallPath = new File(heliumLocalModuleDirectory,
+ "zeppelin-spell");
if (spellModulePath != null) {
- if (spellModuleInstallPath.exists()) {
+ if (recopy && spellModuleInstallPath.exists()) {
FileUtils.deleteDirectory(spellModuleInstallPath);
}
- FileUtils.copyDirectory(
- spellModulePath,
- spellModuleInstallPath,
- npmPackageCopyFilter);
+ if (!spellModuleInstallPath.exists()) {
+ FileUtils.copyDirectory(
+ spellModulePath,
+ spellModuleInstallPath,
+ npmPackageCopyFilter);
+ }
}
}
private WebpackResult getWebpackResultFromOutput(String output) {
BufferedReader reader = new BufferedReader(new StringReader(output));
- String line;
boolean webpackRunDetected = false;
boolean resultJsonDetected = false;
StringBuffer sb = new StringBuffer();
try {
- while ((line = reader.readLine()) != null) {
+ String next, line = reader.readLine();
+ for (boolean last = (line == null); !last; line = next) {
+ last = ((next = reader.readLine()) == null);
+
if (!webpackRunDetected) {
- if (line.contains("webpack.js") && line.endsWith("--json")) {
+ String trimed = line.trim();
+ if (trimed.contains("webpack") && trimed.endsWith("--json")) {
webpackRunDetected = true;
}
continue;
}
if (!resultJsonDetected) {
- if (line.equals("{")) {
+ if (line.trim().equals("{")) {
sb.append(line);
resultJsonDetected = true;
}
@@ -342,10 +516,12 @@ public class HeliumBundleFactory {
}
if (resultJsonDetected && webpackRunDetected) {
- sb.append(line);
+ // yarn command always ends with `Done in ... seconds `
+ if (!last) {
+ sb.append(line);
+ }
}
}
-
Gson gson = new Gson();
return gson.fromJson(sb.toString(), WebpackResult.class);
} catch (IOException e) {
@@ -354,16 +530,6 @@ public class HeliumBundleFactory {
}
}
- public File getCurrentCacheBundle() {
- synchronized (this) {
- if (currentCacheBundle.isFile()) {
- return currentCacheBundle;
- } else {
- return null;
- }
- }
- }
-
private boolean isLocalPackage(HeliumPackage pkg) {
return (pkg.getArtifact().startsWith(".") || pkg.getArtifact().startsWith("/"));
}
@@ -423,14 +589,27 @@ public class HeliumBundleFactory {
}
private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
- installNodeAndNpm();
NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl);
npm.execute(args, env);
}
- private void configureLogger() {
+ private void npmCommand(FrontendPluginFactory fpf, String args) throws TaskRunnerException {
+ npmCommand(args, new HashMap<String, String>());
+ }
+
+ private void yarnCommand(FrontendPluginFactory fpf, String args) throws TaskRunnerException {
+ yarnCommand(fpf, args, new HashMap<String, String>());
+ }
+
+ private void yarnCommand(FrontendPluginFactory fpf,
+ String args, Map<String, String> env) throws TaskRunnerException {
+ YarnRunner yarn = fpf.getYarnRunner(getProxyConfig(), defaultNpmRegistryUrl);
+ yarn.execute(args, env);
+ }
+
+ private synchronized void configureLogger() {
org.apache.log4j.Logger npmLogger = org.apache.log4j.Logger.getLogger(
- "com.github.eirslett.maven.plugins.frontend.lib.DefaultNpmRunner");
+ "com.github.eirslett.maven.plugins.frontend.lib.DefaultYarnRunner");
Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders();
if (appenders != null) {
@@ -440,7 +619,7 @@ public class HeliumBundleFactory {
@Override
public int decide(LoggingEvent loggingEvent) {
- if (loggingEvent.getLoggerName().contains("DefaultNpmRunner")) {
+ if (loggingEvent.getLoggerName().contains("DefaultYarnRunner")) {
return DENY;
} else {
return NEUTRAL;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
index e54349b..dc4de44 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
@@ -43,7 +43,6 @@ public class HeliumLocalRegistry extends HeliumRegistry {
}
-
@Override
public synchronized List<HeliumPackage> getAll() throws IOException {
List<HeliumPackage> result = new LinkedList<>();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/resources/helium/package.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/resources/helium/package.json b/zeppelin-zengine/src/main/resources/helium/package.json
index 197ef65..6e0fe92 100644
--- a/zeppelin-zengine/src/main/resources/helium/package.json
+++ b/zeppelin-zengine/src/main/resources/helium/package.json
@@ -1,11 +1,11 @@
{
- "name": "zeppelin-helium-bundle",
- "main": "load",
+ "name": "PACKAGE_NAME",
+ "main": "MAIN_FILE",
"scripts": {
- "bundle": "node/node node_modules/webpack/bin/webpack.js --display-error-details --json"
+ "bundle": "webpack --display-error-details --json"
},
"dependencies": {
- DEPENDENCIES
+DEPENDENCIES
},
"devDependencies": {
"webpack": "^1.12.2",
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/resources/helium/webpack.config.js
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/resources/helium/webpack.config.js b/zeppelin-zengine/src/main/resources/helium/webpack.config.js
index 0163ca1..c7d3384 100644
--- a/zeppelin-zengine/src/main/resources/helium/webpack.config.js
+++ b/zeppelin-zengine/src/main/resources/helium/webpack.config.js
@@ -16,43 +16,22 @@
*/
module.exports = {
- entry: './load.js',
+ entry: './src/load.js',
output: { path: './', filename: 'helium.bundle.js', },
- module: {
- loaders: [
- {
- test: /\.js$/,
- // DON'T exclude. since zeppelin will bundle all necessary packages: `exclude: /node_modules/,`
- loader: 'babel-loader',
- query: { presets: ['es2015', 'stage-0'] },
- },
- {
- test: /(\.css)$/,
- loaders: ['style', 'css?sourceMap&importLoaders=1'],
- },
- {
- test: /\.woff(\?\S*)?$/,
- loader: 'url-loader?limit=10000&minetype=application/font-woff',
- },
- {
- test: /\.woff2(\?\S*)?$/,
- loader: 'url-loader?limit=10000&minetype=application/font-woff',
- },
- {
- test: /\.eot(\?\S*)?$/,
- loader: 'url-loader',
- }, {
- test: /\.ttf(\?\S*)?$/,
- loader: 'url-loader',
- },
- {
- test: /\.svg(\?\S*)?$/,
- loader: 'url-loader',
- },
- {
- test: /\.json$/,
- loader: 'json-loader'
- },
- ],
- }
+ module: {
+ loaders: [
+ {
+ test: /\.js$/,
+ loader: 'babel-loader',
+ exclude: /node_modules\/(?!(zeppelin-spell|zeppelin-vis|zeppelin-tabledata)\/).*/,
+ query: { presets: ['es2015', 'stage-0'] },
+ },
+ { test: /(\.css)$/, loaders: ['style', 'css?sourceMap&importLoaders=1'], },
+ { test: /\.woff(\?\S*)?$/, loader: 'url-loader?limit=10000&minetype=application/font-woff', },
+ { test: /\.woff2(\?\S*)?$/, loader: 'url-loader?limit=10000&minetype=application/font-woff', },
+ { test: /\.eot(\?\S*)?$/, loader: 'url-loader', }, {
+ test: /\.ttf(\?\S*)?$/, loader: 'url-loader', }, {
+ test: /\.svg(\?\S*)?$/, loader: 'url-loader', }, {
+ test: /\.json$/, loader: 'json-loader' }, ],
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
index 7b816e6..d55358f 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
@@ -22,6 +22,7 @@ import com.google.common.io.Resources;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
@@ -38,9 +39,16 @@ public class HeliumBundleFactoryTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private HeliumBundleFactory hbf;
+ static File nodeInstallationDir = new File(
+ System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_nodeCache");
+
+ @BeforeClass
+ static public void beforeAll() throws IOException {
+ FileUtils.deleteDirectory(nodeInstallationDir);
+ }
@Before
- public void setUp() throws InstallationException, TaskRunnerException {
+ public void setUp() throws InstallationException, TaskRunnerException, IOException {
tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
tmpDir.mkdirs();
@@ -52,10 +60,13 @@ public class HeliumBundleFactoryTest {
conf = new ZeppelinConfiguration();
hbf = new HeliumBundleFactory(conf,
+ nodeInstallationDir,
tmpDir,
new File(moduleDir, "tabledata"),
new File(moduleDir, "visualization"),
new File(moduleDir, "spell"));
+ hbf.installNodeAndNpm();
+ hbf.copyFrameworkModuleToInstallPath(true);
}
@After
@@ -65,17 +76,9 @@ public class HeliumBundleFactoryTest {
@Test
public void testInstallNpm() throws InstallationException {
- assertFalse(new File(tmpDir,
- HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
- assertFalse(new File(tmpDir,
- HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
-
- hbf.installNodeAndNpm();
-
- assertTrue(new File(tmpDir,
- HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
- assertTrue(new File(tmpDir,
- HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
+ assertTrue(new File(nodeInstallationDir, "/node/npm").isFile());
+ assertTrue(new File(nodeInstallationDir, "/node/node").isFile());
+ assertTrue(new File(nodeInstallationDir, "/node/yarn/dist/bin/yarn").isFile());
}
@Test
@@ -107,14 +110,12 @@ public class HeliumBundleFactoryTest {
"license",
"icon"
);
- List<HeliumPackage> pkgs = new LinkedList<>();
- pkgs.add(pkg);
- File bundle = hbf.buildBundle(pkgs);
+ File bundle = hbf.buildPackage(pkg, true, true);
assertTrue(bundle.isFile());
long lastModified = bundle.lastModified();
// buildBundle again and check if it served from cache
- bundle = hbf.buildBundle(pkgs);
+ bundle = hbf.buildPackage(pkg, false, true);
assertEquals(lastModified, bundle.lastModified());
}
@@ -135,9 +136,7 @@ public class HeliumBundleFactoryTest {
"license",
"fa fa-coffee"
);
- List<HeliumPackage> pkgs = new LinkedList<>();
- pkgs.add(pkg);
- File bundle = hbf.buildBundle(pkgs);
+ File bundle = hbf.buildPackage(pkg, true, true);
assertTrue(bundle.isFile());
}
@@ -157,11 +156,9 @@ public class HeliumBundleFactoryTest {
"license",
"fa fa-coffee"
);
- List<HeliumPackage> pkgs = new LinkedList<>();
- pkgs.add(pkg);
File bundle = null;
try {
- bundle = hbf.buildBundle(pkgs);
+ bundle = hbf.buildPackage(pkg, true, true);
// should throw exception
assertTrue(false);
} catch (IOException e) {
@@ -202,8 +199,8 @@ public class HeliumBundleFactoryTest {
List<HeliumPackage> pkgsV2 = new LinkedList<>();
pkgsV2.add(pkgV2);
- File bundle1 = hbf.buildBundle(pkgsV1);
- File bundle2 = hbf.buildBundle(pkgsV2);
+ File bundle1 = hbf.buildPackage(pkgV1, true, true);
+ File bundle2 = hbf.buildPackage(pkgV2, true, true);
assertNotSame(bundle1.lastModified(), bundle2.lastModified());
}