You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2021/11/27 14:52:20 UTC
[spark] branch master updated: [SPARK-37437][BUILD] Remove unused hive profile and related CI test
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f399d0d [SPARK-37437][BUILD] Remove unused hive profile and related CI test
f399d0d is described below
commit f399d0d748dc72740ba0b3f662d658546215c71d
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Sat Nov 27 08:50:46 2021 -0600
[SPARK-37437][BUILD] Remove unused hive profile and related CI test
### What changes were proposed in this pull request?
Since we only support hive-2.3, we should remove the unused profile and related GA test.
`-Phive-2.3` still works after removing the profile
### Why are the changes needed?
Remove unused profile
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Not need
Closes #34679 from AngersZhuuuu/SPARK-37437.
Authored-by: Angerszhuuuu <an...@gmail.com>
Signed-off-by: Sean Owen <sr...@gmail.com>
---
dev/create-release/release-build.sh | 2 +-
dev/run-tests-jenkins.py | 3 ---
dev/run-tests.py | 23 +----------------------
dev/test-dependencies.sh | 8 +++-----
pom.xml | 5 -----
5 files changed, 5 insertions(+), 36 deletions(-)
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 96b8d4e..44baedd 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -192,7 +192,7 @@ SCALA_2_12_PROFILES="-Pscala-2.12"
HIVE_PROFILES="-Phive -Phive-thriftserver"
# Profiles for publishing snapshots and release to Maven Central
# We use Apache Hive 2.3 for publishing
-PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Phive-2.3 -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud"
+PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud"
# Profiles for building binary releases
BASE_RELEASE_PROFILES="$BASE_PROFILES -Psparkr"
diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py
index f24e702..67d0972 100755
--- a/dev/run-tests-jenkins.py
+++ b/dev/run-tests-jenkins.py
@@ -174,9 +174,6 @@ def main():
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.7"
if "test-hadoop3.2" in ghprb_pull_title:
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop3.2"
- # Switch the Hive profile based on the PR title:
- if "test-hive2.3" in ghprb_pull_title:
- os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive2.3"
# Switch the Scala profile based on the PR title:
if "test-scala2.13" in ghprb_pull_title:
os.environ["AMPLAB_JENKINS_BUILD_SCALA_PROFILE"] = "scala2.13"
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 55c65ed..25df8f6 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -345,24 +345,6 @@ def get_hadoop_profiles(hadoop_version):
sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
-def get_hive_profiles(hive_version):
- """
- For the given Hive version tag, return a list of Maven/SBT profile flags for
- building and testing against that Hive version.
- """
-
- sbt_maven_hive_profiles = {
- "hive2.3": ["-Phive-2.3"],
- }
-
- if hive_version in sbt_maven_hive_profiles:
- return sbt_maven_hive_profiles[hive_version]
- else:
- print("[error] Could not find", hive_version, "in the list. Valid options",
- " are", sbt_maven_hive_profiles.keys())
- sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
-
-
def build_spark_maven(extra_profiles):
# Enable all of the profiles for the build:
build_profiles = extra_profiles + modules.root.build_profile_flags
@@ -616,7 +598,6 @@ def main():
build_tool = os.environ.get("AMPLAB_JENKINS_BUILD_TOOL", "sbt")
scala_version = os.environ.get("AMPLAB_JENKINS_BUILD_SCALA_PROFILE")
hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop3.2")
- hive_version = os.environ.get("AMPLAB_JENKINS_BUILD_HIVE_PROFILE", "hive2.3")
test_env = "amplab_jenkins"
# add path for Python3 in Jenkins if we're calling from a Jenkins machine
# TODO(sknapp): after all builds are ported to the ubuntu workers, change this to be:
@@ -627,14 +608,12 @@ def main():
build_tool = "sbt"
scala_version = os.environ.get("SCALA_PROFILE")
hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop3.2")
- hive_version = os.environ.get("HIVE_PROFILE", "hive2.3")
if "GITHUB_ACTIONS" in os.environ:
test_env = "github_actions"
else:
test_env = "local"
- extra_profiles = get_hadoop_profiles(hadoop_version) + get_hive_profiles(hive_version) + \
- get_scala_profiles(scala_version)
+ extra_profiles = get_hadoop_profiles(hadoop_version) + get_scala_profiles(scala_version)
print("[info] Using build tool", build_tool, "with profiles",
*(extra_profiles + ["under environment", test_env]))
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index 156a0d3..e23a0b6 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -86,20 +86,18 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de
for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do
if [[ $HADOOP_HIVE_PROFILE == **hadoop-3.2-hive-2.3** ]]; then
HADOOP_PROFILE=hadoop-3.2
- HIVE_PROFILE=hive-2.3
else
HADOOP_PROFILE=hadoop-2.7
- HIVE_PROFILE=hive-2.3
fi
echo "Performing Maven install for $HADOOP_HIVE_PROFILE"
- $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q
+ $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q
echo "Performing Maven validate for $HADOOP_HIVE_PROFILE"
- $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q
+ $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE validate -q
echo "Generating dependency manifest for $HADOOP_HIVE_PROFILE"
mkdir -p dev/pr-deps
- $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \
+ $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly -am \
| grep "Dependencies classpath:" -A 1 \
| tail -n 1 | tr ":" "\n" | awk -F '/' '{
# For each dependency classpath, we fetch the last three parts split by "/": artifact id, version, and jar name.
diff --git a/pom.xml b/pom.xml
index f8cc8bb..87e3489 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3354,11 +3354,6 @@
</profile>
<profile>
- <id>hive-2.3</id>
- <!-- Default hive profile. Uses global properties. -->
- </profile>
-
- <profile>
<id>yarn</id>
<modules>
<module>resource-managers/yarn</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org