You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/02/08 21:28:10 UTC
spark git commit: [SPARK-19464][BUILD][HOTFIX] run-tests should use
hadoop2.6
Repository: spark
Updated Branches:
refs/heads/master 1aeb9f6cb -> c618ccdbe
[SPARK-19464][BUILD][HOTFIX] run-tests should use hadoop2.6
## What changes were proposed in this pull request?
After SPARK-19464, **SparkPullRequestBuilder** fails because it still tries to use hadoop2.3.
**BEFORE**
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/72595/console
```
========================================================================
Building Spark
========================================================================
[error] Could not find hadoop2.3 in the list. Valid options are ['hadoop2.6', 'hadoop2.7']
Attempting to post to Github...
> Post successful.
```
**AFTER**
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/72595/console
```
========================================================================
Building Spark
========================================================================
[info] Building Spark (w/Hive 1.2.1) using SBT with these arguments: -Phadoop-2.6 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive test:package streaming-kafka-0-8-assembly/assembly streaming-flume-assembly/assembly streaming-kinesis-asl-assembly/assembly
Using /usr/java/jdk1.8.0_60 as default JAVA_HOME.
Note, this will be overridden by -java-home if it is set.
```
## How was this patch tested?
Pass the existing test.
Author: Dongjoon Hyun <do...@apache.org>
Closes #16858 from dongjoon-hyun/hotfix_run-tests.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c618ccdb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c618ccdb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c618ccdb
Branch: refs/heads/master
Commit: c618ccdbe9ac103dfa3182346e2a14a1e7fca91a
Parents: 1aeb9f6
Author: Dongjoon Hyun <do...@apache.org>
Authored: Wed Feb 8 21:28:04 2017 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed Feb 8 21:28:04 2017 +0000
----------------------------------------------------------------------
R/pkg/inst/tests/testthat/test_utils.R | 3 ---
dev/run-tests-jenkins.py | 6 ------
dev/run-tests.py | 4 ++--
3 files changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c618ccdb/R/pkg/inst/tests/testthat/test_utils.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/inst/tests/testthat/test_utils.R
index c875248..6d006ec 100644
--- a/R/pkg/inst/tests/testthat/test_utils.R
+++ b/R/pkg/inst/tests/testthat/test_utils.R
@@ -231,10 +231,7 @@ test_that("varargsToStrEnv", {
test_that("basenameSansExtFromUrl", {
x <- paste0("http://people.apache.org/~pwendell/spark-nightly/spark-branch-2.1-bin/spark-2.1.1-",
"SNAPSHOT-2016_12_09_11_08-eb2d9bf-bin/spark-2.1.1-SNAPSHOT-bin-hadoop2.7.tgz")
- y <- paste0("http://people.apache.org/~pwendell/spark-releases/spark-2.1.0-rc2-bin/spark-2.1.0-",
- "bin-hadoop2.4-without-hive.tgz")
expect_equal(basenameSansExtFromUrl(x), "spark-2.1.1-SNAPSHOT-bin-hadoop2.7")
- expect_equal(basenameSansExtFromUrl(y), "spark-2.1.0-bin-hadoop2.4-without-hive")
z <- "http://people.apache.org/~pwendell/spark-releases/spark-2.1.0--hive.tar.gz"
expect_equal(basenameSansExtFromUrl(z), "spark-2.1.0--hive")
})
http://git-wip-us.apache.org/repos/asf/spark/blob/c618ccdb/dev/run-tests-jenkins.py
----------------------------------------------------------------------
diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py
index bb286af..53061bc 100755
--- a/dev/run-tests-jenkins.py
+++ b/dev/run-tests-jenkins.py
@@ -165,12 +165,6 @@ def main():
if "test-maven" in ghprb_pull_title:
os.environ["AMPLAB_JENKINS_BUILD_TOOL"] = "maven"
# Switch the Hadoop profile based on the PR title:
- if "test-hadoop2.2" in ghprb_pull_title:
- os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.2"
- if "test-hadoop2.3" in ghprb_pull_title:
- os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.3"
- if "test-hadoop2.4" in ghprb_pull_title:
- os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.4"
if "test-hadoop2.6" in ghprb_pull_title:
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.6"
if "test-hadoop2.7" in ghprb_pull_title:
http://git-wip-us.apache.org/repos/asf/spark/blob/c618ccdb/dev/run-tests.py
----------------------------------------------------------------------
diff --git a/dev/run-tests.py b/dev/run-tests.py
index ef9ab1a..0e7f5ff 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -505,14 +505,14 @@ def main():
# if we're on the Amplab Jenkins build servers setup variables
# to reflect the environment settings
build_tool = os.environ.get("AMPLAB_JENKINS_BUILD_TOOL", "sbt")
- hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.3")
+ hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.6")
test_env = "amplab_jenkins"
# add path for Python3 in Jenkins if we're calling from a Jenkins machine
os.environ["PATH"] = "/home/anaconda/envs/py3k/bin:" + os.environ.get("PATH")
else:
# else we're running locally and can use local settings
build_tool = "sbt"
- hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.3")
+ hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.6")
test_env = "local"
print("[info] Using build tool", build_tool, "with Hadoop profile", hadoop_version,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org