You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2014/08/09 14:40:03 UTC
[32/39] git commit: Create release scripts,
add support for "cdh4" build.
Create release scripts, add support for "cdh4" build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/ecb829ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/ecb829ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/ecb829ed
Branch: refs/heads/travis_test
Commit: ecb829ed47544242953db8c1806338445c40713d
Parents: e4c3754
Author: Robert Metzger <rm...@apache.org>
Authored: Wed Jul 30 19:57:21 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Aug 6 11:43:05 2014 +0200
----------------------------------------------------------------------
DISCLAIMER | 11 ++
flink-addons/flink-hadoop-compatibility/pom.xml | 23 ++++
.../mapred/wrapper/HadoopDummyReporter.java | 3 +-
flink-clients/pom.xml | 6 +
flink-dist/pom.xml | 24 +++-
flink-dist/src/main/assemblies/bin.xml | 5 +-
flink-dist/src/main/assemblies/yarn.xml | 9 +-
flink-runtime/pom.xml | 25 ++++
flink-test-utils/pom.xml | 12 ++
flink-tests/pom.xml | 12 ++
pom.xml | 1 +
tools/create_release_files.sh | 134 +++++++++++++++++++
tools/how_to_release.md | 16 +++
tools/prepare_release.sh | 23 ++++
14 files changed, 291 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..48a1a19
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,11 @@
+Apache Flink is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+
+While incubation status is not necessarily a reflection of the completeness
+or stability of the code, it does indicate that the project has yet to be
+fully endorsed by the ASF.
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-addons/flink-hadoop-compatibility/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/pom.xml b/flink-addons/flink-hadoop-compatibility/pom.xml
index 746b782..52c2917 100644
--- a/flink-addons/flink-hadoop-compatibility/pom.xml
+++ b/flink-addons/flink-hadoop-compatibility/pom.xml
@@ -78,5 +78,28 @@ under the License.
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>cdh4</id>
+ <!--
+ See flink-runtime/pom.xml for details
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
index 87a6014..ac2b4b6 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
@@ -62,8 +62,7 @@ public class HadoopDummyReporter implements Reporter {
public InputSplit getInputSplit() throws UnsupportedOperationException {
return null;
}
-
- @Override
+ // There should be an @Override, but some CDH4 dependency does not contain this method
public float getProgress() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-clients/pom.xml
----------------------------------------------------------------------
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 7456209..b035c19 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -95,6 +95,12 @@ under the License.
<version>2.4</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.4</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<!-- More information on this:
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 5672fde..8c46149 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -132,8 +132,9 @@ under the License.
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<executions>
+ <!-- Uber-jar -->
<execution>
- <id>single-jar</id>
+ <id>uber-jar</id>
<phase>package</phase>
<goals>
<goal>single</goal>
@@ -154,8 +155,23 @@ under the License.
</archive>
<descriptors>
<descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+
+ <!-- yarn bin directory -->
+ <execution>
+ <id>yarn-bin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
<descriptor>src/main/assemblies/yarn.xml</descriptor>
</descriptors>
+ <finalName>flink-${project.version}-bin</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
@@ -321,13 +337,15 @@ under the License.
</profile>
</profiles>
+
+
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
- <id>generate-package</id>
+ <id>bin</id>
<phase>package</phase>
<goals>
<goal>single</goal>
@@ -336,6 +354,8 @@ under the License.
<descriptors>
<descriptor>src/main/assemblies/bin.xml</descriptor>
</descriptors>
+ <finalName>flink-${project.version}-bin</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/src/main/assemblies/bin.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index cc5850a..6ade3be 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -23,10 +23,6 @@ under the License.
<id>bin</id>
<formats>
<format>dir</format>
- <!--
- <format>tar.gz</format>
- <format>zip</format>
- -->
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
@@ -114,6 +110,7 @@ under the License.
<include>*.txt</include>
<include>LICENSE*</include>
<include>NOTICE*</include>
+ <include>DISCLAIMER*</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/src/main/assemblies/yarn.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/yarn.xml b/flink-dist/src/main/assemblies/yarn.xml
index 2fa19cb..f262ab0 100644
--- a/flink-dist/src/main/assemblies/yarn.xml
+++ b/flink-dist/src/main/assemblies/yarn.xml
@@ -23,11 +23,7 @@ under the License.
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>yarn</id>
<formats>
- <format>tar.gz</format>
- <!--
- <format>tar.gz</format>
- <format>zip</format>
- -->
+ <format>dir</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
@@ -68,6 +64,9 @@ under the License.
<includes>
<include>*.txt</include>
<include>*.properties</include>
+ <include>DISCLAIMER*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 7bb49f4..7b22708 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -217,5 +217,30 @@ under the License.
</dependency>
</dependencies>
</profile>
+
+ <profile>
+ <id>cdh4</id>
+ <!--
+
+ It seems http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/4.2.0/CDH4-Installation-Guide/cdh4ig_topic_31.html
+ that this particular CDH4 version has already separate packages for hdfs and common.
+
+ To activate the cdh4 profile, use the -Pcdh4 flag. (You probably want -Pcdh4,vendor-repos)
+ You also HAVE TO set the -Dhadoop.cdh4.hadoop.version property (for example to: 2.0.0-cdh4.2.0)
+
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
index 1a0fe1b..a9d21ed 100644
--- a/flink-test-utils/pom.xml
+++ b/flink-test-utils/pom.xml
@@ -66,4 +66,16 @@ under the License.
<version>4.11</version>
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>cdh4</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 950e78e..b4b11ea 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -236,4 +236,16 @@ under the License.
</plugins>
</pluginManagement>
</build>
+ <profiles>
+ <profile>
+ <id>cdh4</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8058539..a22c7e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -460,6 +460,7 @@ under the License.
<exclude>**/target/**</exclude>
<!-- Documentation -->
<exclude>**/docs/**</exclude>
+ <exclude>**/tools/*.md</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
new file mode 100755
index 0000000..aa8875d
--- /dev/null
+++ b/tools/create_release_files.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Quick-and-dirty automation of making maven and binary releases. Not robust at all.
+# Publishes releases to Maven and packages/copies binary release artifacts.
+# Expects to be run in a totally empty directory.
+#
+
+#
+# NOTE: The code in this file is based on code from the Apache Spark
+# project, licensed under the Apache License v 2.0
+#
+# https://github.com/apache/spark/blob/master/dev/create-release/create-release.sh
+#
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+ echo "You have to call the script from the tools/ dir"
+ exit 1
+fi
+
+GIT_USERNAME=${GIT_USERNAME:-pwendell}
+GIT_PASSWORD=${GIT_PASSWORD:-XXX}
+GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
+GPG_KEY=${GPG_KEY:-XXX}
+GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
+RELEASE_VERSION=${NEW_VERSION}
+USER_NAME=${USER_NAME:-pwendell}
+MVN=${MVN:-mvn}
+
+set -e
+
+# create source package
+
+git clone https://github.com/rmetzger/incubator-flink.git flink
+cd flink
+git checkout -b release-0.6 origin/release-0.6
+rm .gitignore
+rm .travis.yml
+rm deploysettings.xml
+rm CHANGELOG
+rm -rf .git
+cd ..
+
+
+cp -r flink flink-$RELEASE_VERSION
+tar cvzf flink-$RELEASE_VERSION.tgz flink-$RELEASE_VERSION
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --armour --output flink-$RELEASE_VERSION.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION.tgz
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md MD5 flink-$RELEASE_VERSION.tgz > \
+ flink-$RELEASE_VERSION.tgz.md5
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md SHA512 flink-$RELEASE_VERSION.tgz > \
+ flink-$RELEASE_VERSION.tgz.sha
+rm -rf flink-$RELEASE_VERSION
+
+
+make_binary_release() {
+ NAME=$1
+ FLAGS=$2
+ cp -r flink flink-$RELEASE_VERSION-bin-$NAME
+
+ cd flink-$RELEASE_VERSION-bin-$NAME
+ # make distribution
+ $MVN clean package $FLAGS -DskipTests
+ cd flink-dist/target/flink-$RELEASE_VERSION-bin/
+ tar cvzf flink-$RELEASE_VERSION-bin-$NAME.tgz flink-$RELEASE_VERSION
+ if [ -d "flink-yarn-$RELEASE_VERSION" ] ; then
+ tar cvzf flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz flink-yarn-$RELEASE_VERSION
+ fi
+ cp flink-*.tgz ../../../../
+ cd ../../../../
+ rm -rf flink-$RELEASE_VERSION
+
+ # Sign md5 and sha the tgz
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --armour \
+ --output flink-$RELEASE_VERSION-bin-$NAME.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION-bin-$NAME.tgz
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ MD5 flink-$RELEASE_VERSION-bin-$NAME.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME.tgz.md5
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ SHA512 flink-$RELEASE_VERSION-bin-$NAME.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME.tgz.sha
+
+ if [ -f "flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz" ] ; then
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --armour \
+ --output flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ MD5 flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.md5
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ SHA512 flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.sha
+ fi
+}
+
+make_binary_release "hadoop1" ""
+make_binary_release "cdh4" "-Pvendor-repos,cdh4 -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -Dhadoop.cdh4.hadoop.version=2.0.0-cdh4.2.0"
+# make_binary_release "cdh5" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.3.0+cdh5.1.0+795"
+make_binary_release "hadoop2" "-Dhadoop.profile=2"
+# make_binary_release "mapr4" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.3.0-mapr-4.0.0-FCS"
+# make_binary_release "hdp21" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.4.0.2.1.3.0-563"
+
+
+# Copy data
+echo "Copying release tarballs"
+folder=flink-$RELEASE_VERSION
+ssh $USER_NAME@people.apache.org \
+ mkdir /home/$USER_NAME/public_html/$folder
+scp flink-* \
+ $USER_NAME@people.apache.org:/home/$USER_NAME/public_html/$folder/
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/how_to_release.md
----------------------------------------------------------------------
diff --git a/tools/how_to_release.md b/tools/how_to_release.md
new file mode 100644
index 0000000..65cc3b7
--- /dev/null
+++ b/tools/how_to_release.md
@@ -0,0 +1,16 @@
+# How to release a new version of Apache Flink
+
+
+You need to have the following environment variables set:
+
+export OLD_VERSION=0.6-SNAPSHOT
+export NEW_VERSION=0.6
+export GPG_KEY=XX
+export GPG_PASSPHRASE=XX
+
+./prepare_release_files.sh
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/prepare_release.sh
----------------------------------------------------------------------
diff --git a/tools/prepare_release.sh b/tools/prepare_release.sh
new file mode 100644
index 0000000..17354b9
--- /dev/null
+++ b/tools/prepare_release.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Updating version from $OLD_VERSION to $NEW_VERSION"
+
+find .. -name 'pom.xml' -type f -exec sed -i 's#<version>$OLD_VERSION</version>#<version>$NEW_VERSION</version>#' {} \;
+