You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gr...@apache.org on 2017/06/01 16:10:08 UTC
flink git commit: [FLINK-6466] [build] Build Hadoop 2.8.0 convenience
binaries
Repository: flink
Updated Branches:
refs/heads/master 99c627892 -> 2438824a8
[FLINK-6466] [build] Build Hadoop 2.8.0 convenience binaries
Update Hadoop versions and replace Hadoop 2.3 with 2.8 in build and
continuous integration scripts. flink-yarn-tests can now be enabled for
all supported Hadoop versions and is made non-optional.
This closes #3832
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2438824a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2438824a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2438824a
Branch: refs/heads/master
Commit: 2438824a8f39c07783e80adf902c72eecab31347
Parents: 99c6278
Author: Greg Hogan <co...@greghogan.com>
Authored: Fri May 5 08:39:53 2017 -0400
Committer: Greg Hogan <co...@greghogan.com>
Committed: Thu Jun 1 12:06:58 2017 -0400
----------------------------------------------------------------------
.travis.yml | 24 +++----
.../flink-shaded-hadoop2/pom.xml | 6 +-
.../flink-shaded-include-yarn-tests/pom.xml | 72 --------------------
.../flink-shaded-yarn-tests/pom.xml | 72 ++++++++++++++++++++
flink-shaded-hadoop/pom.xml | 10 +--
flink-yarn-tests/pom.xml | 2 +-
pom.xml | 11 +--
tools/create_release_files.sh | 32 ++++-----
8 files changed, 107 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 44ea97d..b9251b5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,32 +16,32 @@ matrix:
include:
# Always run test groups A and B together
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk7"
- env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk7"
- env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk7"
- env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
+ env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
git:
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
index a1159d6..74754c3 100644
--- a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
+++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
@@ -681,13 +681,13 @@ under the License.
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.2.5</version>
+ <version>4.4.4</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.2.6</version>
+ <version>4.5.2</version>
</dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
deleted file mode 100644
index fe87331..0000000
--- a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop</artifactId>
- <version>1.4-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-shaded-include-yarn-tests</artifactId>
- <name>flink-shaded-include-yarn-tests</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- NOTE: We do not exclude hadoop's dependencies so that the tests start properly (in particular jetty) -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <!-- Add test-jar as well -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <version>${hadoop.version}</version>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml
new file mode 100644
index 0000000..e3ad97d
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-hadoop</artifactId>
+ <version>1.4-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-shaded-yarn-tests</artifactId>
+ <name>flink-shaded-yarn-tests</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <!-- NOTE: We do not exclude hadoop's dependencies so that the tests start properly (in particular jetty) -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <!-- Add test-jar as well -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index a549ae9..26e6077 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -37,6 +37,7 @@ under the License.
<modules>
<module>flink-shaded-hadoop2</module>
<module>flink-shaded-hadoop2-uber</module>
+ <module>flink-shaded-yarn-tests</module>
</modules>
<dependencies>
@@ -63,15 +64,6 @@ under the License.
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>include-yarn-tests</id>
- <modules>
- <module>flink-shaded-include-yarn-tests</module>
- </modules>
- </profile>
- </profiles>
-
<build>
<plugins>
<!-- Relocate the Hadoop's Guava dependency into a different namespace and
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-yarn-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index 200e5ab..f198ece 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -88,7 +88,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-include-yarn-tests</artifactId>
+ <artifactId>flink-shaded-yarn-tests</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ae92be9..ac020b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -76,6 +76,7 @@ under the License.
<module>flink-mesos</module>
<module>flink-metrics</module>
<module>flink-yarn</module>
+ <module>flink-yarn-tests</module>
<module>flink-fs-tests</module>
</modules>
@@ -84,7 +85,7 @@ under the License.
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Internal property to reduce build times on TravisCi -->
<flink-fast-tests-pattern>never-match-me</flink-fast-tests-pattern>
- <hadoop.version>2.3.0</hadoop.version>
+ <hadoop.version>2.4.1</hadoop.version>
<!-- Need to use a user property here because the surefire
forkCount is not exposed as a property. With this we can set
it on the "mvn" commandline in travis. -->
@@ -493,14 +494,6 @@ under the License.
</properties>
</profile>
- <!-- Profile to deactivate the YARN tests -->
- <profile>
- <id>include-yarn-tests</id>
- <modules>
- <module>flink-yarn-tests</module>
- </modules>
- </profile>
-
<profile>
<id>vendor-repos</id>
<!-- Add vendor maven repositories -->
http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
index 40d038c..ba99846 100755
--- a/tools/create_release_files.sh
+++ b/tools/create_release_files.sh
@@ -91,7 +91,7 @@ fi
usage() {
set +x
- echo "./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2"
+ echo "./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3"
echo ""
echo "usage:"
echo "[--scala-version <version>] [--hadoop-version <version>]"
@@ -102,12 +102,12 @@ usage() {
echo " USER_NAME=APACHEID GPG_PASSPHRASE=XXX GPG_KEY=KEYID \ "
echo " GIT_AUTHOR=\"`git config --get user.name` <`git config --get user.email`>\" \ "
echo " GIT_REPO=github.com/apache/flink.git \ "
- echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2"
+ echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3"
echo ""
echo "example 2: build local release"
echo " NEW_VERSION=1.2.0 RELEASE_BRANCH=master OLD_VERSION=1.2-SNAPSHOT \ "
echo " GPG_PASSPHRASE=XXX GPG_KEY=XXX IS_LOCAL_DIST=true \ "
- echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2"
+ echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3"
exit 1
}
@@ -272,25 +272,25 @@ make_source_release
# build dist by input parameter of "--scala-vervion xxx --hadoop-version xxx"
if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then
- make_binary_release "hadoop2" "" 2.10
- make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.10"
- make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.10"
- make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.10"
-
- make_binary_release "hadoop2" "" 2.11
- make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11"
- make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.11"
- make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.11"
+ make_binary_release "hadoop2" "" "2.10"
+ make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.10"
+ make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.10"
+ make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.10"
+
+ make_binary_release "hadoop2" "" "2.11"
+ make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.11"
+ make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.11"
+ make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.11"
elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ]
then
make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.10"
make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11"
elif [ "$SCALA_VERSION" != none ] && [ "$HADOOP_VERSION" == "none" ]
then
- make_binary_release "hadoop2" "" $SCALA_VERSION
- make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "$SCALA_VERSION"
- make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "$SCALA_VERSION"
- make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "$SCALA_VERSION"
+ make_binary_release "hadoop2" "" "$SCALA_VERSION"
+ make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "$SCALA_VERSION"
+ make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "$SCALA_VERSION"
+ make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "$SCALA_VERSION"
else
make_binary_release "hadoop2x" "-Dhadoop.version=$HADOOP_VERSION" "$SCALA_VERSION"
fi