You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/09/05 22:41:48 UTC
spark git commit: [SPARK-25335][BUILD] Skip Zinc downloading if it's
installed in the system
Repository: spark
Updated Branches:
refs/heads/master 71bd79651 -> 458468ad5
[SPARK-25335][BUILD] Skip Zinc downloading if it's installed in the system
## What changes were proposed in this pull request?
Zinc is 23.5MB (tgz).
```
$ curl -LO https://downloads.lightbend.com/zinc/0.3.15/zinc-0.3.15.tgz
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 23.5M 100 23.5M 0 0 35.4M 0 --:--:-- --:--:-- --:--:-- 35.3M
```
Currently, Spark downloads Zinc once. However, it occurs too many times in build systems. This PR aims to skip Zinc downloading when the system already has it.
```
$ build/mvn clean
exec: curl --progress-bar -L https://downloads.lightbend.com/zinc/0.3.15/zinc-0.3.15.tgz
######################################################################## 100.0%
```
This will reduce many resources(CPU/Networks/DISK) at least in Mac and Docker-based build system.
## How was this patch tested?
Pass the Jenkins.
Closes #22333 from dongjoon-hyun/SPARK-25335.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Sean Owen <se...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/458468ad
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/458468ad
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/458468ad
Branch: refs/heads/master
Commit: 458468ad5163211b9275faa49cd817a974a6dc21
Parents: 71bd796
Author: Dongjoon Hyun <do...@apache.org>
Authored: Wed Sep 5 15:41:45 2018 -0700
Committer: Sean Owen <se...@databricks.com>
Committed: Wed Sep 5 15:41:45 2018 -0700
----------------------------------------------------------------------
build/mvn | 29 +++++++++++++++++++----------
1 file changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/458468ad/build/mvn
----------------------------------------------------------------------
diff --git a/build/mvn b/build/mvn
index ae4276d..2487b81 100755
--- a/build/mvn
+++ b/build/mvn
@@ -67,6 +67,9 @@ install_app() {
fi
}
+# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
+function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
+
# Determine the Maven version from the root pom.xml file and
# install maven under the build/ folder if needed.
install_mvn() {
@@ -75,8 +78,6 @@ install_mvn() {
if [ "$MVN_BIN" ]; then
local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
fi
- # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
- function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}
@@ -91,15 +92,23 @@ install_mvn() {
# Install zinc under the build/ folder
install_zinc() {
- local zinc_path="zinc-0.3.15/bin/zinc"
- [ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
- local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
+ local ZINC_VERSION=0.3.15
+ ZINC_BIN="$(command -v zinc)"
+ if [ "$ZINC_BIN" ]; then
+ local ZINC_DETECTED_VERSION="$(zinc -version | head -n1 | awk '{print $5}')"
+ fi
- install_app \
- "${TYPESAFE_MIRROR}/zinc/0.3.15" \
- "zinc-0.3.15.tgz" \
- "${zinc_path}"
- ZINC_BIN="${_DIR}/${zinc_path}"
+ if [ $(version $ZINC_DETECTED_VERSION) -lt $(version $ZINC_VERSION) ]; then
+ local zinc_path="zinc-${ZINC_VERSION}/bin/zinc"
+ [ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
+ local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
+
+ install_app \
+ "${TYPESAFE_MIRROR}/zinc/${ZINC_VERSION}" \
+ "zinc-${ZINC_VERSION}.tgz" \
+ "${zinc_path}"
+ ZINC_BIN="${_DIR}/${zinc_path}"
+ fi
}
# Determine the Scala version from the root pom.xml file, set the Scala URL,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org