You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2019/02/11 08:39:14 UTC
[zeppelin] branch master updated: ZEPPELIN-3983. Travis fails due
to downloading spark takes a lot of time
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 727600b ZEPPELIN-3983. Travis fails due to downloading spark takes a lot of time
727600b is described below
commit 727600b5ca217367909f1090b9e2eff075cc7ed1
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Sat Feb 2 09:28:42 2019 +0800
ZEPPELIN-3983. Travis fails due to downloading spark takes a lot of time
### What is this PR for?
This PR did several refactoring.
* rename SparkDownloadUtils -> DownloadUtils as it is also used for downloading flink.
* try apache mirror just one time and fall back to apache archive when fails to download from mirror site.
### What type of PR is it?
[Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-3983
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3300 from zjffdu/ZEPPELIN-3983 and squashes the following commits:
5201746ed [Jeff Zhang] ZEPPELIN-3983. Travis fails due to downloading spark takes a lot of time
---
.travis.yml | 5 +-
.../apache/zeppelin/integration/DownloadUtils.java | 139 ++++++++++++++++
.../zeppelin/integration/FlinkIntegrationTest.java | 2 +-
.../zeppelin/integration/SparkDownloadUtils.java | 174 ---------------------
.../zeppelin/integration/SparkIntegrationTest.java | 2 +-
.../integration/ZeppelinSparkClusterTest.java | 2 +-
6 files changed, 146 insertions(+), 178 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 23529fb..b05c750 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,8 +34,11 @@ cache:
addons:
apt:
sources:
- - r-packages-trusty
+ - r-source
+ - sourceline: 'deb http://cran.rstudio.com/bin/linux/ubuntu trusty/'
+ key_url: 'keyserver.ubuntu.com/pks/lookup?op=get&search=0x51716619E084DAB9'
packages:
+ - r-base
- r-base-dev
env:
diff --git a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/DownloadUtils.java b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/DownloadUtils.java
new file mode 100644
index 0000000..4371b2b
--- /dev/null
+++ b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/DownloadUtils.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.integration;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+
+/**
+ * Utility class for downloading spark/flink. This is used for spark/flink integration test.
+ */
+public class DownloadUtils {
+ private static Logger LOGGER = LoggerFactory.getLogger(DownloadUtils.class);
+
+ private static String downloadFolder = System.getProperty("user.home") + "/.cache";
+
+ static {
+ try {
+ FileUtils.forceMkdir(new File(downloadFolder));
+ } catch (IOException e) {
+ throw new RuntimeException("Fail to create download folder: " + downloadFolder, e);
+ }
+ }
+
+
+ public static String downloadSpark(String version) {
+ String sparkDownloadFolder = downloadFolder + "/spark";
+ File targetSparkHomeFolder = new File(sparkDownloadFolder + "/spark-" + version + "-bin-hadoop2.6");
+ if (targetSparkHomeFolder.exists()) {
+ LOGGER.info("Skip to download spark as it is already downloaded.");
+ return targetSparkHomeFolder.getAbsolutePath();
+ }
+ download("spark", version, "-bin-hadoop2.6.tgz");
+ return targetSparkHomeFolder.getAbsolutePath();
+ }
+
+ public static String downloadFlink(String version) {
+ String flinkDownloadFolder = downloadFolder + "/flink";
+ File targetFlinkHomeFolder = new File(flinkDownloadFolder + "/flink-" + version);
+ if (targetFlinkHomeFolder.exists()) {
+ LOGGER.info("Skip to download flink as it is already downloaded.");
+ return targetFlinkHomeFolder.getAbsolutePath();
+ }
+ download("flink", version, "-bin-hadoop27-scala_2.11.tgz");
+ return targetFlinkHomeFolder.getAbsolutePath();
+ }
+
+ // Try mirrors first, if fails fallback to apache archive
+ private static void download(String project, String version, String postFix) {
+ String projectDownloadFolder = downloadFolder + "/" + project;
+ try {
+ String preferredMirror = IOUtils.toString(new URL("https://www.apache.org/dyn/closer.lua?preferred=true"));
+ File downloadFile = new File(projectDownloadFolder + "/" + project + "-" + version + postFix);
+ String downloadURL = preferredMirror + "/" + project + "/" + project + "-" + version + "/" + project + "-" + version + postFix;
+ runShellCommand(new String[]{"wget", downloadURL, "-P", projectDownloadFolder});
+ runShellCommand(new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", projectDownloadFolder});
+ } catch (Exception e) {
+ LOGGER.warn("Failed to download " + project + " from mirror site, fallback to use apache archive", e);
+ File downloadFile = new File(projectDownloadFolder + "/" + project + "-" + version + postFix);
+ String downloadURL =
+ "https://archive.apache.org/dist/" + project + "/" + project +"-"
+ + version
+ + "/" + project + "-"
+ + version
+ + postFix;
+ try {
+ runShellCommand(new String[]{"wget", downloadURL, "-P", projectDownloadFolder});
+ runShellCommand(
+ new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", projectDownloadFolder});
+ } catch (Exception ex) {
+ throw new RuntimeException("Fail to download " + project + " " + version, ex);
+ }
+ }
+ }
+
+ private static void runShellCommand(String[] commands) throws IOException, InterruptedException {
+ LOGGER.info("Starting shell commands: " + StringUtils.join(commands, " "));
+ Process process = Runtime.getRuntime().exec(commands);
+ StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream());
+ StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream());
+ errorGobbler.start();
+ outputGobbler.start();
+ if (process.waitFor() != 0) {
+ throw new IOException("Fail to run shell commands: " + StringUtils.join(commands, " "));
+ }
+ LOGGER.info("Complete shell commands: " + StringUtils.join(commands, " "));
+ }
+
+ private static class StreamGobbler extends Thread {
+ InputStream is;
+
+ // reads everything from is until empty.
+ StreamGobbler(InputStream is) {
+ this.is = is;
+ }
+
+ public void run() {
+ try {
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+ String line = null;
+ long startTime = System.currentTimeMillis();
+ while ((line = br.readLine()) != null) {
+ // logging per 5 seconds
+ if ((System.currentTimeMillis() - startTime) > 5000) {
+ LOGGER.info(line);
+ startTime = System.currentTimeMillis();
+ }
+ }
+ } catch (IOException ioe) {
+ LOGGER.warn("Fail to print shell output", ioe);
+ }
+ }
+ }
+}
diff --git a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/FlinkIntegrationTest.java b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/FlinkIntegrationTest.java
index 1ad2b17..526d389 100644
--- a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/FlinkIntegrationTest.java
+++ b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/FlinkIntegrationTest.java
@@ -60,7 +60,7 @@ public class FlinkIntegrationTest {
public FlinkIntegrationTest(String flinkVersion) {
LOGGER.info("Testing FlinkVersion: " + flinkVersion);
this.flinkVersion = flinkVersion;
- this.flinkHome = SparkDownloadUtils.downloadFlink(flinkVersion);
+ this.flinkHome = DownloadUtils.downloadFlink(flinkVersion);
}
@Parameterized.Parameters
diff --git a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkDownloadUtils.java b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkDownloadUtils.java
deleted file mode 100644
index 43c280e..0000000
--- a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkDownloadUtils.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.integration;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-
-/**
- * Utility class for downloading spark. This is used for spark integration test.
- *
- */
-public class SparkDownloadUtils {
- private static Logger LOGGER = LoggerFactory.getLogger(SparkDownloadUtils.class);
-
- private static String downloadFolder = System.getProperty("user.home") + "/.cache/spark";
-
- static {
- try {
- FileUtils.forceMkdir(new File(downloadFolder));
- } catch (IOException e) {
- throw new RuntimeException("Fail to create downloadFolder: " + downloadFolder, e);
- }
- }
-
-
- public static String downloadSpark(String version) {
- File targetSparkHomeFolder = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6");
- if (targetSparkHomeFolder.exists()) {
- LOGGER.info("Skip to download spark as it is already downloaded.");
- return targetSparkHomeFolder.getAbsolutePath();
- }
- // Try mirrors a few times until one succeeds
- boolean downloaded = false;
- for (int i = 0; i < 3; i++) {
- try {
- String preferredMirror = IOUtils.toString(new URL("https://www.apache.org/dyn/closer.lua?preferred=true"));
- File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz");
- String downloadURL = preferredMirror + "/spark/spark-" + version + "/spark-" + version + "-bin-hadoop2.6.tgz";
- runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder});
- runShellCommand(new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder});
- downloaded = true;
- break;
- } catch (Exception e) {
- LOGGER.warn("Failed to download Spark", e);
- }
- }
- // fallback to use apache archive
- if (!downloaded) {
- File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz");
- String downloadURL =
- "https://archive.apache.org/dist/spark/spark-"
- + version
- + "/spark-"
- + version
- + "-bin-hadoop2.6.tgz";
- try {
- runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder});
- runShellCommand(
- new String[] {"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder});
- } catch (Exception e) {
- throw new RuntimeException("Fail to download spark " + version, e);
- }
- }
- return targetSparkHomeFolder.getAbsolutePath();
- }
-
- public static String downloadFlink(String version) {
- File targetFlinkHomeFolder = new File(downloadFolder + "/flink-" + version);
- if (targetFlinkHomeFolder.exists()) {
- LOGGER.info("Skip to download flink as it is already downloaded.");
- return targetFlinkHomeFolder.getAbsolutePath();
- }
- // Try mirrors a few times until one succeeds
- boolean downloaded = false;
- // Try mirrors a few times until one succeeds
- for (int i = 0; i < 3; i++) {
- try {
- String preferredMirror = IOUtils.toString(new URL("https://www.apache.org/dyn/closer.lua?preferred=true"));
- File downloadFile = new File(downloadFolder + "/flink-" + version + "-bin-hadoop27-scala_2.11.tgz");
- String downloadURL = preferredMirror + "/flink/flink-" + version + "/flink-" + version + "-bin-hadoop27-scala_2.11.tgz";
- runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder});
- runShellCommand(new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder});
- downloaded = true;
- break;
- } catch (Exception e) {
- LOGGER.warn("Failed to download Flink", e);
- }
- }
-
- // fallback to use apache archive
- if (!downloaded) {
- File downloadFile = new File(downloadFolder + "/flink-" + version + "-bin-hadoop27-scala_2.11.tgz");
- String downloadURL =
- "https://archive.apache.org/dist/flink/flink-"
- + version
- + "/flink-"
- + version
- + "-bin-hadoop27-scala_2.11.tgz";
- try {
- runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder});
- runShellCommand(
- new String[] {"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder});
- } catch (Exception e) {
- throw new RuntimeException("Fail to download flink " + version, e);
- }
- }
- return targetFlinkHomeFolder.getAbsolutePath();
- }
-
- private static void runShellCommand(String[] commands) throws IOException, InterruptedException {
- LOGGER.info("Starting shell commands: " + StringUtils.join(commands, " "));
- Process process = Runtime.getRuntime().exec(commands);
- StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream());
- StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream());
- errorGobbler.start();
- outputGobbler.start();
- if (process.waitFor() != 0) {
- throw new IOException("Fail to run shell commands: " + StringUtils.join(commands, " "));
- }
- LOGGER.info("Complete shell commands: " + StringUtils.join(commands, " "));
- }
-
- private static class StreamGobbler extends Thread {
- InputStream is;
-
- // reads everything from is until empty.
- StreamGobbler(InputStream is) {
- this.is = is;
- }
-
- public void run() {
- try {
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- String line = null;
- long startTime = System.currentTimeMillis();
- while ( (line = br.readLine()) != null) {
- // logging per 5 seconds
- if ((System.currentTimeMillis() - startTime) > 5000) {
- LOGGER.info(line);
- startTime = System.currentTimeMillis();
- }
- }
- } catch (IOException ioe) {
- ioe.printStackTrace();
- }
- }
- }
-}
diff --git a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkIntegrationTest.java b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkIntegrationTest.java
index 369d124..0347937 100644
--- a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkIntegrationTest.java
+++ b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/SparkIntegrationTest.java
@@ -57,7 +57,7 @@ public abstract class SparkIntegrationTest {
public SparkIntegrationTest(String sparkVersion) {
LOGGER.info("Testing SparkVersion: " + sparkVersion);
this.sparkVersion = sparkVersion;
- this.sparkHome = SparkDownloadUtils.downloadSpark(sparkVersion);
+ this.sparkHome = DownloadUtils.downloadSpark(sparkVersion);
}
@BeforeClass
diff --git a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/ZeppelinSparkClusterTest.java b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/ZeppelinSparkClusterTest.java
index 53031ae..50779c9 100644
--- a/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/ZeppelinSparkClusterTest.java
+++ b/zeppelin-interpreter-integration/src/main/test/org/apache/zeppelin/integration/ZeppelinSparkClusterTest.java
@@ -73,7 +73,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
public ZeppelinSparkClusterTest(String sparkVersion) throws Exception {
this.sparkVersion = sparkVersion;
LOGGER.info("Testing SparkVersion: " + sparkVersion);
- String sparkHome = SparkDownloadUtils.downloadSpark(sparkVersion);
+ String sparkHome = DownloadUtils.downloadSpark(sparkVersion);
if (!verifiedSparkVersions.contains(sparkVersion)) {
verifiedSparkVersions.add(sparkVersion);
setupSparkInterpreter(sparkHome);