You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2015/09/24 09:50:47 UTC
[2/2] flink git commit: [FLINK-2200] [build system] Deploy Flink 2.11
with every build (nightly)
[FLINK-2200] [build system] Deploy Flink 2.11 with every build (nightly)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5dc2756d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5dc2756d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5dc2756d
Branch: refs/heads/travis_test
Commit: 5dc2756d1788b12cd97798667d11fe19e795e723
Parents: b68f05d
Author: Robert Metzger <rm...@apache.org>
Authored: Wed Sep 23 15:02:24 2015 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Thu Sep 24 09:49:08 2015 +0200
----------------------------------------------------------------------
flink-shaded-hadoop/pom.xml | 2 +-
pom.xml | 5 +++
tools/change-scala-version.sh | 24 +++++++++++
tools/deploy_to_maven.sh | 86 +++++++++++++++++++++-----------------
4 files changed, 78 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5dc2756d/flink-shaded-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index 6e75b25..a99f3c4 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -35,7 +35,7 @@ under the License.
<packaging>pom</packaging>
<modules>
- <module>${shading-artifact.name}</module>
+ <module>${shading-artifact-module.name}</module>
</modules>
<profiles>
http://git-wip-us.apache.org/repos/asf/flink/blob/5dc2756d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6603e81..6f1a137 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,10 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <!-- Mutable name of the hadoop shading artifact. The module name can contain a scala version suffix. -->
<shading-artifact.name>error</shading-artifact.name>
+ <!-- The shading artifact module name can be used as a module name. it does not get a scala version suffix -->
+ <shading-artifact-module.name>error</shading-artifact-module.name>
<hadoop-one.version>1.2.1</hadoop-one.version>
<hadoop-two.version>2.3.0</hadoop-two.version>
<!-- Need to use a user property here because the surefire/failsafe
@@ -377,6 +380,7 @@ under the License.
<properties>
<hadoop.version>${hadoop-one.version}</hadoop.version>
<shading-artifact.name>flink-shaded-hadoop1</shading-artifact.name>
+ <shading-artifact-module.name>flink-shaded-hadoop1</shading-artifact-module.name>
</properties>
</profile>
@@ -392,6 +396,7 @@ under the License.
<properties>
<hadoop.version>${hadoop-two.version}</hadoop.version>
<shading-artifact.name>flink-shaded-hadoop2</shading-artifact.name>
+ <shading-artifact-module.name>flink-shaded-hadoop2</shading-artifact-module.name>
</properties>
<modules>
<module>flink-yarn</module>
http://git-wip-us.apache.org/repos/asf/flink/blob/5dc2756d/tools/change-scala-version.sh
----------------------------------------------------------------------
diff --git a/tools/change-scala-version.sh b/tools/change-scala-version.sh
index ff38711..2c124a9 100755
--- a/tools/change-scala-version.sh
+++ b/tools/change-scala-version.sh
@@ -79,3 +79,27 @@ find "$BASEDIR/flink-dist" -name 'bin.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(<source>.*flink-dist\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g' {}" \;
find "$BASEDIR/flink-dist" -name 'bin.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(<include>org\.apache\.flink:flink-.*\)'$FROM_SUFFIX'<\/include>/\1'$TO_SUFFIX'<\/include>/g' {}" \;
+
+
+if [ "$TO_VERSION" == "2.11" ]; then
+ # set the profile activation to !scala-2.11 in parent pom, so that it activates by default
+ bash -c "sed_i 's/<name>scala-2.11<\/name>/<name>!scala-2.11<\/name>/g' $BASEDIR/pom.xml" \;
+ # set the profile activation in all sub modules to scala-2.11 (so that they are disabled by default)
+ find $BASEDIR/flink-* -name 'pom.xml' -not -path '*target*' -print \
+ -exec bash -c "sed_i 's/<name>!scala-2.11<\/name>/<name>scala-2.11<\/name>/g' {}" \;
+
+ # set the name of the shading artifact properly
+ bash -c "sed_i 's/\(shading-artifact.name>flink-shaded[a-z0-9\-]*\)'$FROM_SUFFIX'<\/shading-artifact.name>/\1'$TO_SUFFIX'<\/shading-artifact.name>/g' $BASEDIR/pom.xml" \;
+fi
+
+if [ "$TO_VERSION" == "2.10" ]; then
+ # do the opposite as above
+ bash -c "sed_i 's/<name>!scala-2.11<\/name>/<name>scala-2.11<\/name>/g' $BASEDIR/pom.xml" \;
+ # also for the other files
+ find $BASEDIR/flink-* -name 'pom.xml' -not -path '*target*' -print \
+ -exec bash -c "sed_i 's/<name>scala-2.11<\/name>/<name>!scala-2.11<\/name>/g' {}" \;
+
+ # unset shading artifact name
+ bash -c "sed_i 's/\(shading-artifact.name>flink-shaded[a-z0-9\-]*\)'$FROM_SUFFIX'<\/shading-artifact.name>/\1'$TO_SUFFIX'<\/shading-artifact.name>/g' $BASEDIR/pom.xml" \;
+fi
+
http://git-wip-us.apache.org/repos/asf/flink/blob/5dc2756d/tools/deploy_to_maven.sh
----------------------------------------------------------------------
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
index e51e0aa..ffe966f 100755
--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -28,12 +28,6 @@
# 5. Nothing
-echo "install lifecylce mapping fake plugin"
-git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
-cd dummy-lifecycle-mapping-plugin
-mvn -B install
-cd ..
-
function getVersion() {
here="`dirname \"$0\"`" # relative
here="`( cd \"$here\" && pwd )`" # absolutized and normalized
@@ -47,57 +41,73 @@ function getVersion() {
echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'`
}
-pwd
-
-# this will take a while
-CURRENT_FLINK_VERSION=`getVersion`
-if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
- CURRENT_FLINK_VERSION_HADOOP1=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop1-SNAPSHOT}
-else
- CURRENT_FLINK_VERSION_HADOOP1="$CURRENT_FLINK_VERSION-hadoop1"
-fi
+function deploy_to_s3() {
+ CURRENT_FLINK_VERSION=$1
+ HD=$2
+
+ echo "Deplyoing flink version $CURRENT_FLINK_VERSION (hadoop=$HD) to s3:"
+ mkdir flink-$CURRENT_FLINK_VERSION
+ cp -r flink-dist/target/flink-*-bin/flink-$CURRENT_FLINK_VERSION*/* flink-$CURRENT_FLINK_VERSION/
+ tar -czf flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz flink-$CURRENT_FLINK_VERSION
+ travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz --target-path /
+ # delete files again
+ rm -rf flink-$CURRENT_FLINK_VERSION
+ rm flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz
+}
-echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1 "
+pwd
# Check if push/commit is eligible for pushing
-echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST"
-if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
+echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST ; repo slug : $TRAVIS_REPO_SLUG "
+if [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $TRAVIS_REPO_SLUG == "rmetzger/flink" ]] ; then
+
+ echo "install lifecylce mapping fake plugin"
+ git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
+ cd dummy-lifecycle-mapping-plugin
+ mvn -B install
+ cd ..
+
+ # this will take a while
+ CURRENT_FLINK_VERSION=`getVersion`
+ if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
+ CURRENT_FLINK_VERSION_HADOOP1=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop1-SNAPSHOT}
+ else
+ CURRENT_FLINK_VERSION_HADOOP1="$CURRENT_FLINK_VERSION-hadoop1"
+ fi
+
+ echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1 "
#
# This script is called by travis to deploy our project to sonatype SNAPSHOTS.
# It will deploy both a hadoop v1 and a hadoop v2 (yarn) artifact
#
- if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
+ if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
# Deploy hadoop v1 to maven
echo "Generating poms for hadoop1"
./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_HADOOP1 pom.hadoop1.xml
mvn -B -f pom.hadoop1.xml -Pdocs-and-source -DskipTests -Drat.ignoreErrors=true deploy --settings deploysettings.xml;
+
+ # deploy to s3
+ deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop1"
fi
- if [[ $TRAVIS_JOB_NUMBER == *2 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
+ if [[ $TRAVIS_JOB_NUMBER == *2 ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
# deploy hadoop v2 (yarn)
echo "deploy standard version (hadoop2)"
- mvn -B -DskipTests -Pdocs-and-source -Drat.ignoreErrors=true clean deploy --settings deploysettings.xml;
- fi
+ mvn -B -DskipTests -Pdocs-and-source -Drat.skip=true -Drat.ignoreErrors=true clean deploy --settings deploysettings.xml;
+
+ deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2"
+
+ echo "deploy hadoop2 version (standard) for scala 2.11"
+ ./tools/change-scala-version.sh 2.11
+ mvn -B -DskipTests -Pdocs-and-source -Drat.skip=true -Drat.ignoreErrors=true clean deploy --settings deploysettings.xml;
+
+ deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2_2.11"
- if [[ $TRAVIS_JOB_NUMBER == *1 ]] || [[ $TRAVIS_JOB_NUMBER == *2 ]] ; then
- echo "Uploading build to amazon s3. Job Number: $TRAVIS_JOB_NUMBER"
-
- HD="hadoop1"
-
- # job nr 2 is hadoop 2
- if [[ $TRAVIS_JOB_NUMBER == *2 ]] ; then
- HD="hadoop2"
- fi
-
- mkdir flink-$CURRENT_FLINK_VERSION
- cp -r flink-dist/target/flink-*-bin/flink-$CURRENT_FLINK_VERSION*/* flink-$CURRENT_FLINK_VERSION/
- tar -czf flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz flink-$CURRENT_FLINK_VERSION
- travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz --target-path /
- echo "doing a ls -lisah:"
- ls -lisah
+ echo "Changing back to scala 2.10"
+ ./tools/change-scala-version.sh 2.10
fi
fi # pull request check