You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2020/08/20 06:00:12 UTC
[flink] branch release-1.11 updated: [FLINK-18643][Azure] Build a
Flink snapshot release with the nightly cron-job.
This is an automated email from the ASF dual-hosted git repository.
rmetzger pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.11 by this push:
new 0782d01 [FLINK-18643][Azure] Build a Flink snapshot release with the nightly cron-job.
0782d01 is described below
commit 0782d01315abebb360185a5f2c38eddcc7b14f4e
Author: Robert Metzger <rm...@apache.org>
AuthorDate: Tue Aug 4 16:49:38 2020 +0200
[FLINK-18643][Azure] Build a Flink snapshot release with the nightly cron-job.
This closes #13125
---
tools/azure-pipelines/build-apache-repo.yml | 5 ++
tools/azure-pipelines/build-nightly-dist.yml | 117 +++++++++++++++++++++++++++
tools/ci/deploy_nightly_to_s3.sh | 43 ++++++++++
tools/deploy_to_maven.sh | 99 -----------------------
tools/releasing/deploy_staging_jars.sh | 3 +-
5 files changed, 167 insertions(+), 100 deletions(-)
diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml
index dab6d95..89bbae6 100644
--- a/tools/azure-pipelines/build-apache-repo.yml
+++ b/tools/azure-pipelines/build-apache-repo.yml
@@ -73,6 +73,11 @@ stages:
dependsOn: [] # depending on an empty array makes the stages run in parallel
condition: or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['MODE'], 'nightly'))
jobs:
+ - template: build-nightly-dist.yml
+ parameters:
+ stage_name: cron_snapshot_deployment
+ environment: PROFILE=""
+ container: flink-build-container
- template: jobs-template.yml
parameters:
stage_name: cron_hadoop241
diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml
new file mode 100644
index 0000000..867605c
--- /dev/null
+++ b/tools/azure-pipelines/build-nightly-dist.yml
@@ -0,0 +1,117 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+jobs:
+ - job: ${{parameters.stage_name}}_binary
+ pool:
+ vmImage: 'ubuntu-16.04'
+ container: flink-build-container
+ workspace:
+ clean: all
+ steps:
+ - task: Cache@2
+ displayName: Cache Maven local repo
+ inputs:
+ key: $(CACHE_KEY)
+ restoreKeys: $(CACHE_FALLBACK_KEY)
+ path: $(MAVEN_CACHE_FOLDER)
+ continueOnError: true
+ - task: CmdLine@2
+ displayName: Build snapshot binary release
+ inputs:
+ script: |
+ source ./tools/ci/maven-utils.sh
+ setup_maven
+ run_mvn -version
+ export MVN="run_mvn"
+
+ export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+ echo "Determined RELEASE_VERSION as '$RELEASE_VERSION' "
+ cd tools
+ MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh
+ echo "Created files:"
+ find ./releasing/release
+ cd ..
+ - task: CmdLine@2
+ displayName: Upload artifacts to S3
+ inputs:
+ script: |
+ source ./tools/ci/deploy_nightly_to_s3.sh
+
+ upload_to_s3 ./tools/releasing/release
+ env:
+ ARTIFACTS_S3_BUCKET: $(ARTIFACTS_S3_BUCKET)
+ ARTIFACTS_AWS_ACCESS_KEY_ID: $(ARTIFACTS_AWS_ACCESS_KEY_ID)
+ ARTIFACTS_AWS_SECRET_ACCESS_KEY: $(ARTIFACTS_AWS_SECRET_ACCESS_KEY)
+ # Activate this to publish the binary release as a pipeline artifact on Azure
+ #- task: PublishPipelineArtifact@1
+ # displayName: Upload snapshot binary release
+ # inputs:
+ # targetPath: ./tools/releasing/release
+ # artifact: nightly-release
+ - job: ${{parameters.stage_name}}_maven
+ pool:
+ vmImage: 'ubuntu-16.04'
+ container: flink-build-container
+ timeoutInMinutes: 100 # 40 minutes per scala version + 20 buffer
+ workspace:
+ clean: all
+ steps:
+ - task: Cache@2
+ displayName: Cache Maven local repo
+ inputs:
+ key: $(CACHE_KEY)
+ restoreKeys: $(CACHE_FALLBACK_KEY)
+ path: $(MAVEN_CACHE_FOLDER)
+ continueOnError: true
+ # Upload snapshot
+ - task: CmdLine@2
+ displayName: Deploy maven snapshot
+ inputs:
+ script: |
+ source ./tools/ci/maven-utils.sh
+ setup_maven
+ run_mvn -version
+
+ cd tools
+ cat << EOF > deploy-settings.xml
+ <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+ https://maven.apache.org/xsd/settings-1.0.0.xsd">
+ <servers>
+ <server>
+ <id>apache.snapshots.https</id>
+ <username>${MAVEN_DEPLOY_USER}</username>
+ <password>${MAVEN_DEPLOY_PASS}</password>
+ </server>
+ </servers>
+ <mirrors>
+ <mirror>
+ <id>google-maven-central</id>
+ <name>GCS Maven Central mirror</name>
+ <url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
+ <mirrorOf>central</mirrorOf>
+ </mirror>
+ </mirrors>
+ </settings>
+ EOF
+
+ export CUSTOM_OPTIONS="-Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml"
+ export MVN_RUN_VERBOSE=true
+ ./releasing/deploy_staging_jars.sh
+ env:
+ MAVEN_DEPLOY_USER: $(MAVEN_DEPLOY_USER)
+ MAVEN_DEPLOY_PASS: $(MAVEN_DEPLOY_PASS)
diff --git a/tools/ci/deploy_nightly_to_s3.sh b/tools/ci/deploy_nightly_to_s3.sh
new file mode 100755
index 0000000..ca373bd
--- /dev/null
+++ b/tools/ci/deploy_nightly_to_s3.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# fail on errors
+set -e -x
+
+
+function upload_to_s3() {
+ local FILES_DIR=$1
+
+ echo "Installing artifacts deployment script"
+ export ARTIFACTS_DEST="$HOME/bin/artifacts"
+ curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash
+ PATH="$(dirname "$ARTIFACTS_DEST"):$PATH"
+
+ echo "Uploading contents of $FILES_DIR to S3:"
+
+
+ artifacts upload \
+ --bucket $ARTIFACTS_S3_BUCKET \
+ --key $ARTIFACTS_AWS_ACCESS_KEY_ID \
+ --secret $ARTIFACTS_AWS_SECRET_ACCESS_KEY \
+ --target-paths / $FILES_DIR
+
+}
+
+
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
deleted file mode 100755
index abc34c8..0000000
--- a/tools/deploy_to_maven.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# fail on errors
-set -e
-
-#
-# Deploys snapshot builds to Apache's snapshot repository.
-#
-
-function getVersion() {
- here="`dirname \"$0\"`" # relative
- here="`( cd \"$here\" && pwd )`" # absolutized and normalized
- if [ -z "$here" ] ; then
- # error; for some reason, the path is not accessible
- # to the script (e.g. permissions re-evaled after suid)
- exit 1 # fail
- fi
- flink_home="`dirname \"$here\"`"
- cd "$flink_home"
- echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'`
-}
-
-function deploy_to_s3() {
- local CURRENT_FLINK_VERSION=$1
- local HD=$2
-
- echo "Installing artifacts deployment script"
- export ARTIFACTS_DEST="$HOME/bin/artifacts"
- curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash
- PATH="$(dirname "$ARTIFACTS_DEST"):$PATH"
-
- echo "Deploying flink version $CURRENT_FLINK_VERSION (hadoop=$HD) to s3:"
- mkdir flink-$CURRENT_FLINK_VERSION
- cp -r flink-dist/target/flink-*-bin/flink-*/* flink-$CURRENT_FLINK_VERSION/
- tar -czf flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz flink-$CURRENT_FLINK_VERSION
-
- artifacts upload \
- --bucket $ARTIFACTS_S3_BUCKET \
- --key $ARTIFACTS_AWS_ACCESS_KEY_ID \
- --secret $ARTIFACTS_AWS_SECRET_ACCESS_KEY \
- --target-paths / \
- flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz
-
- # delete files again
- rm -rf flink-$CURRENT_FLINK_VERSION
- rm flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz
-}
-
-pwd
-
-
-echo "install lifecycle mapping fake plugin"
-git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
-cd dummy-lifecycle-mapping-plugin
-mvn -B install
-cd ..
-rm -rf dummy-lifecycle-mapping-plugin
-
-
-CURRENT_FLINK_VERSION=`getVersion`
-
-echo "detected current version as: '$CURRENT_FLINK_VERSION'"
-
-#
-# This script deploys our project to sonatype SNAPSHOTS.
-# It will deploy a hadoop v2 (yarn) artifact
-#
-
-if [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
- MVN_SNAPSHOT_OPTS="-B -Pdocs-and-source -DskipTests -Drat.skip=true -Drat.ignoreErrors=true -Dcheckstyle.skip=true \
- -DretryFailedDeploymentCount=10 clean deploy"
-
- # hadoop2 scala 2.11
- echo "deploy standard version (hadoop2) for scala 2.11"
- mvn ${MVN_SNAPSHOT_OPTS}
- deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2"
-
- exit 0
-else
- exit 1
-fi
-
diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh
index 00bf7df..6a7af33 100755
--- a/tools/releasing/deploy_staging_jars.sh
+++ b/tools/releasing/deploy_staging_jars.sh
@@ -21,6 +21,7 @@
## Variables with defaults (if not overwritten by environment)
##
MVN=${MVN:-mvn}
+CUSTOM_OPTIONS=${CUSTOM_OPTIONS:-}
# fail immediately
set -o errexit
@@ -40,7 +41,7 @@ cd ..
echo "Deploying to repository.apache.org"
-COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10"
+COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10 $CUSTOM_OPTIONS"
echo "Deploying Scala 2.11 version"
$MVN clean deploy $COMMON_OPTIONS -Dscala-2.11