You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2019/03/30 04:59:52 UTC
[hive] branch master updated: HIVE-21516 : Fix spark downloading
for q tests (Miklos Gergely via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2ce775b HIVE-21516 : Fix spark downloading for q tests (Miklos Gergely via Zoltan Haindrich)
2ce775b is described below
commit 2ce775ba75ec3823dc81aeaa704e611d92980917
Author: Miklos Gergely <mg...@hortonworks.com>
AuthorDate: Fri Mar 29 21:57:20 2019 -0700
HIVE-21516 : Fix spark downloading for q tests (Miklos Gergely via Zoltan Haindrich)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
itests/bin/download-spark.sh | 39 ++++++++++++++++++++++++++++++
itests/hive-unit/pom.xml | 2 +-
itests/pom.xml | 56 ++------------------------------------------
itests/qtest-spark/pom.xml | 2 +-
4 files changed, 43 insertions(+), 56 deletions(-)
diff --git a/itests/bin/download-spark.sh b/itests/bin/download-spark.sh
new file mode 100644
index 0000000..c68d536
--- /dev/null
+++ b/itests/bin/download-spark.sh
@@ -0,0 +1,39 @@
+download() {
+ url=$1;
+ finalName=$2
+ tarName=$(basename $url)
+ rm -rf $BASE_DIR/$finalName
+ if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
+ then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url
+ else
+ local md5File="$tarName".md5sum
+ curl -Sso $DOWNLOAD_DIR/$md5File "$url".md5sum
+ cd $DOWNLOAD_DIR
+ if type md5sum >/dev/null && ! md5sum -c $md5File; then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1
+ elif type md5 >/dev/null; then
+ md5chksum=$(md5 -r $tarName)
+ if [[ $(< $md5File) != $md5chksum ]]; then
+ curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1
+ fi
+ fi
+
+ cd -
+ fi
+
+ tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
+ mv $BASE_DIR/spark-$SPARK_VERSION-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName
+}
+
+set -x
+/bin/pwd
+BASE_DIR=./target
+HIVE_ROOT=$BASE_DIR/../../../
+DOWNLOAD_DIR=./../thirdparty
+SPARK_VERSION=$1
+
+mkdir -p $DOWNLOAD_DIR
+download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-$SPARK_VERSION-bin-hadoop3-beta1-without-hive.tgz" "spark"
+cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/
+
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index 711856e..6c83217 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -543,7 +543,7 @@
<configuration>
<target>
<exec executable="bash" dir="${basedir}" failonerror="true">
- <arg line="../target/download.sh"/>
+ <arg line="../bin/download-spark.sh ${spark.version}"/>
</exec>
</target>
</configuration>
diff --git a/itests/pom.xml b/itests/pom.xml
index 9df7379..345e922 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -53,65 +53,13 @@
<profile>
<id>spark-test</id>
<activation>
- <property>
+ <property>
<name>!skipSparkTests</name>
- </property>
+ </property>
</activation>
<modules>
<module>qtest-spark</module>
</modules>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>download-spark</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <echo file="target/download.sh">
- set -x
- /bin/pwd
- BASE_DIR=./target
- HIVE_ROOT=$BASE_DIR/../../../
- DOWNLOAD_DIR=./../thirdparty
- download() {
- url=$1;
- finalName=$2
- tarName=$(basename $url)
- rm -rf $BASE_DIR/$finalName
- if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
- then
- curl -Sso $DOWNLOAD_DIR/$tarName $url
- else
- local md5File="$tarName".md5sum
- curl -Sso $DOWNLOAD_DIR/$md5File "$url".md5sum
- cd $DOWNLOAD_DIR
- if type md5sum >/dev/null && ! md5sum -c $md5File; then
- curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1
- fi
-
- cd -
- fi
- tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
- mv $BASE_DIR/spark-${spark.version}-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName
- }
- mkdir -p $DOWNLOAD_DIR
- download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop3-beta1-without-hive.tgz" "spark"
- cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/
- </echo>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
</profile>
</profiles>
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index d77dc89..d7dff93 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -398,7 +398,7 @@
<configuration>
<target>
<exec executable="bash" dir="${basedir}" failonerror="true">
- <arg line="../target/download.sh"/>
+ <arg line="../bin/download-spark.sh ${spark.version}"/>
</exec>
</target>
</configuration>