You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2020/02/16 18:55:18 UTC
[flink] 05/05: [FLINK-13978][build system] Add experimental support
for building on Azure Pipelines
This is an automated email from the ASF dual-hosted git repository.
rmetzger pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 84fd23d82c2908192d58186d6e061c89b018cda5
Author: Robert Metzger <rm...@apache.org>
AuthorDate: Sun Feb 16 14:12:35 2020 +0100
[FLINK-13978][build system] Add experimental support for building on Azure Pipelines
This closes #10976
---
azure-pipelines.yml | 48 ++++--
flink-end-to-end-tests/run-nightly-tests.sh | 7 +-
flink-end-to-end-tests/test-scripts/common.sh | 47 +----
.../test-scripts/test-runner-common.sh | 2 +-
.../test-scripts/test_streaming_elasticsearch.sh | 25 +--
tools/azure-pipelines/build-apache-repo.yml | 50 ++++++
tools/azure-pipelines/google-mirror-settings.xml | 28 +++
tools/azure-pipelines/jobs-template.yml | 161 +++++++++++++++++
tools/azure-pipelines/prepare_precommit.sh | 39 +++++
.../azure-pipelines/setup_kubernetes.sh | 31 ++--
tools/azure_controller.sh | 190 +++++++++++++++++++++
tools/travis_watchdog.sh | 33 ++--
12 files changed, 549 insertions(+), 112 deletions(-)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index cc41f08..7ffc7d0 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,23 +13,47 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#
+# This file defines an Azure Pipeline build for testing Flink. It is intended to be used
+# with a free Azure Pipelines account.
+# It has the following features:
+# - default builds for pushes / pull requests
+# - end-to-end tests
+#
+#
+# For the "apache/flink" repository, we are using the pipeline definition located in
+# tools/azure-pipelines/build-apache-repo.yml
+# That file points to custom, self-hosted build agents for faster pull request build processing and
+# integration with Flinkbot.
+# The custom pipeline definition file is configured in the "Pipeline settings" screen
+# of the Azure Pipelines web ui.
+#
-trigger:
- branches:
- include:
- - '*'
resources:
containers:
- # Container with Maven 3.2.5 to have the same environment everywhere.
+ # Container with Maven 3.2.5, SSL to have the same environment everywhere.
- container: flink-build-container
- image: rmetzger/flink-ci:3
- repositories:
- - repository: templates
- type: github
- name: flink-ci/flink-azure-builds
- endpoint: flink-ci
+ image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
+
+# See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
+variables:
+ MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
+ MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+ CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
+ CACHE_FALLBACK_KEY: maven | $(Agent.OS)
+ CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
+
jobs:
-- template: flink-build-jobs.yml@templates
+ - template: tools/azure-pipelines/jobs-template.yml
+ parameters: # see template file for a definition of the parameters.
+ stage_name: ci_build
+ test_pool_definition:
+ vmImage: 'ubuntu-latest'
+ e2e_pool_definition:
+ vmImage: 'ubuntu-16.04'
+ environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+
+
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index aa51819..5907b42 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -88,8 +88,11 @@ run_test "Resuming Externalized Checkpoint after terminal failure (rocks, increm
# Docker
################################################################################
-run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
-run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
+# Ignore these tests on Azure: In these tests, the TaskManagers are not starting on YARN, probably due to memory constraints.
+if [ -z "$TF_BUILD" ] ; then
+ run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
+ run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
+fi
################################################################################
# High Availability
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index 0b9d611..ed56b3b 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -247,9 +247,7 @@ function wait_rest_endpoint_up {
echo "Waiting for ${endpoint_name} REST endpoint to come up..."
sleep 1
done
- echo "${endpoint_name} REST endpoint has not started on query url '${query_url}' within a timeout of ${TIMEOUT} sec. curl output:"
- curl ${CURL_SSL_ARGS} "$query_url"
- echo "Exiting ..."
+ echo "${endpoint_name} REST endpoint has not started within a timeout of ${TIMEOUT} sec"
exit 1
}
@@ -439,51 +437,16 @@ function wait_for_job_state_transition {
done
}
-function is_job_submitted {
- JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -a | grep "$1")
- if [[ "$JOB_LIST_RESULT" == "" ]]; then
- echo "false"
- else
- echo "true"
- fi
-}
-
-function wait_job_submitted {
- local TIMEOUT=10
- for i in $(seq 1 ${TIMEOUT}); do
- local IS_SUBMITTED=`is_job_submitted $1`
-
- if [[ "$IS_SUBMITTED" == "true" ]]; then
- echo "Job ($1) is submitted."
- return
- else
- echo "Job ($1) is not yet submitted."
- fi
- sleep 1
- done
- echo "Job ($1) has not been submitted within a timeout of ${TIMEOUT} sec"
- exit 1
-}
-
-function is_job_running {
- JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -r | grep "$1")
- if [[ "$JOB_LIST_RESULT" == "" ]]; then
- echo "false"
- else
- echo "true"
- fi
-}
-
function wait_job_running {
local TIMEOUT=10
for i in $(seq 1 ${TIMEOUT}); do
- local IS_RUNNING=`is_job_running $1`
+ JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -r | grep "$1")
- if [[ "$IS_RUNNING" == "true" ]]; then
+ if [[ "$JOB_LIST_RESULT" == "" ]]; then
+ echo "Job ($1) is not yet running."
+ else
echo "Job ($1) is running."
return
- else
- echo "Job ($1) is not yet running."
fi
sleep 1
done
diff --git a/flink-end-to-end-tests/test-scripts/test-runner-common.sh b/flink-end-to-end-tests/test-scripts/test-runner-common.sh
index 6a0840f..ecddbd9 100644
--- a/flink-end-to-end-tests/test-scripts/test-runner-common.sh
+++ b/flink-end-to-end-tests/test-scripts/test-runner-common.sh
@@ -19,7 +19,7 @@
source "${END_TO_END_DIR}"/test-scripts/common.sh
-FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
+export FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
#######################################
# Prints the given description, runs the given test and prints how long the execution took.
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh b/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh
index adabb47..e2ee273 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh
@@ -39,31 +39,10 @@ on_exit test_cleanup
TEST_ES_JAR=${END_TO_END_DIR}/flink-elasticsearch${ELASTICSEARCH_VERSION}-test/target/Elasticsearch${ELASTICSEARCH_VERSION}SinkExample.jar
# run the Flink job
-JOB_ID=$($FLINK_DIR/bin/flink run -d -p 1 $TEST_ES_JAR \
+$FLINK_DIR/bin/flink run -p 1 $TEST_ES_JAR \
--numRecords 20 \
--index index \
- --type type | awk '{print $NF}' | tail -n 1)
+ --type type
-
-# wait for 10 seconds
-wait_job_submitted ${JOB_ID}
-
-# Wait for 60 seconds for the job to finish
-MAX_RETRY_SECONDS=60
-
-start_time=$(date +%s)
-
-RUNNING=`is_job_running ${JOB_ID}`
-while [[ "$RUNNING" == "true" ]]; do
- RUNNING=`is_job_running ${JOB_ID}`
- current_time=$(date +%s)
- time_diff=$((current_time - start_time))
- if [ $time_diff -ge $MAX_RETRY_SECONDS ]; then
- echo "Job did not finish after $MAX_RETRY_SECONDS seconds. Printing logs and failing test: "
- cat $FLINK_DIR/log/*
- exit 1
- fi
-done
-
# 40 index requests and 20 final update requests
verify_result_line_number 60 index
diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml
new file mode 100644
index 0000000..87919f3
--- /dev/null
+++ b/tools/azure-pipelines/build-apache-repo.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# This file defines the Flink build for the "apache/flink" repository, including
+# the following:
+# - PR builds
+# - custom triggered e2e tests
+# - nightly builds
+
+resources:
+ containers:
+ # Container with Maven 3.2.5, SSL to have the same environment everywhere.
+ - container: flink-build-container
+ image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
+
+variables:
+ MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
+ MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+ CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
+ CACHE_FALLBACK_KEY: maven | $(Agent.OS)
+ CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
+
+stages:
+ # CI / PR triggered stage:
+ - stage: ci_build
+ displayName: "CI Build (custom builders)"
+ condition: not(eq(variables['Build.Reason'], in('Schedule', 'Manual')))
+ jobs:
+ - template: jobs-template.yml
+ parameters:
+ stage_name: ci_build
+ test_pool_definition:
+ name: Default
+ e2e_pool_definition:
+ vmImage: 'ubuntu-16.04'
+ environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
diff --git a/tools/azure-pipelines/google-mirror-settings.xml b/tools/azure-pipelines/google-mirror-settings.xml
new file mode 100644
index 0000000..49a3b71
--- /dev/null
+++ b/tools/azure-pipelines/google-mirror-settings.xml
@@ -0,0 +1,28 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<settings>
+ <mirrors>
+ <mirror>
+ <id>google-maven-central</id>
+ <name>GCS Maven Central mirror</name>
+ <url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
+ <mirrorOf>central</mirrorOf>
+ </mirror>
+ </mirrors>
+</settings>
diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
new file mode 100644
index 0000000..6a65d6f
--- /dev/null
+++ b/tools/azure-pipelines/jobs-template.yml
@@ -0,0 +1,161 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+parameters:
+ test_pool_definition: # defines the hardware pool for compilation and unit test execution.
+ e2e_pool_definion: # defines the hardware pool for end-to-end test execution
+ stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage)
+ environment: # defines environment variables for downstream scripts
+
+jobs:
+- job: compile_${{parameters.stage_name}}
+ condition: not(eq(variables['MODE'], 'e2e'))
+ pool: ${{parameters.test_pool_definition}}
+ container: flink-build-container
+ timeoutInMinutes: 240
+ cancelTimeoutInMinutes: 1
+ workspace:
+ clean: all # this cleans the entire workspace directory before running a new job
+ # It is necessary because the custom build machines are reused for tests.
+ # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
+
+ steps:
+ # The cache task is persisting the .m2 directory between builds, so that
+ # we do not have to re-download all dependencies from maven central for
+ # each build. The hope is that downloading the cache is faster than
+ # all dependencies individually.
+ # In this configuration, we use a hash over all committed (not generated) .pom files
+ # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash
+ # (usually because a pom file has changed), we'll fall back to a key without
+ # the pom files (CACHE_FALLBACK_KEY).
+ # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops
+ - task: Cache@2
+ inputs:
+ key: $(CACHE_KEY)
+ restoreKeys: $(CACHE_FALLBACK_KEY)
+ path: $(MAVEN_CACHE_FOLDER)
+ continueOnError: true # continue the build even if the cache fails.
+ displayName: Cache Maven local repo
+
+ # Compile
+ - script: STAGE=compile ${{parameters.environment}} ./tools/azure_controller.sh compile
+ displayName: Build
+
+ # upload artifacts for next stage
+ - task: PublishPipelineArtifact@1
+ inputs:
+ path: $(CACHE_FLINK_DIR)
+ artifact: FlinkCompileCacheDir-${{parameters.stage_name}}
+
+- job: test_${{parameters.stage_name}}
+ dependsOn: compile_${{parameters.stage_name}}
+ condition: not(eq(variables['MODE'], 'e2e'))
+ pool: ${{parameters.test_pool_definition}}
+ container: flink-build-container
+ timeoutInMinutes: 240
+ cancelTimeoutInMinutes: 1
+ workspace:
+ clean: all
+ strategy:
+ matrix:
+ core:
+ module: core
+ python:
+ module: python
+ libraries:
+ module: libraries
+ blink_planner:
+ module: blink_planner
+ connectors:
+ module: connectors
+ kafka_gelly:
+ module: kafka/gelly
+ tests:
+ module: tests
+ legacy_scheduler_core:
+ module: legacy_scheduler_core
+ legacy_scheduler_tests:
+ module: legacy_scheduler_tests
+ misc:
+ module: misc
+ steps:
+
+ # download artifacts
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ path: $(CACHE_FLINK_DIR)
+ artifact: FlinkCompileCacheDir-${{parameters.stage_name}}
+
+ # recreate "build-target" symlink for python tests
+ - script: |
+ ln -snf $(CACHE_FLINK_DIR)/flink-dist/target/flink-*-SNAPSHOT-bin/flink-*-SNAPSHOT $(CACHE_FLINK_DIR)/build-target
+ displayName: Recreate 'build-target' symlink
+ # Test
+ - script: STAGE=test ${{parameters.environment}} ./tools/azure_controller.sh $(module)
+ displayName: Test - $(module)
+
+ - task: PublishTestResults@2
+ inputs:
+ testResultsFormat: 'JUnit'
+
+- job: precommit_${{parameters.stage_name}}
+ dependsOn: compile_${{parameters.stage_name}}
+ # We are not running this job on a container, but in a VM.
+ pool: ${{parameters.e2e_pool_definition}}
+ timeoutInMinutes: 240
+ cancelTimeoutInMinutes: 1
+ workspace:
+ clean: all
+ steps:
+ - task: Cache@2
+ inputs:
+ key: $(CACHE_KEY)
+ restoreKeys: $(CACHE_FALLBACK_KEY)
+ path: $(MAVEN_CACHE_FOLDER)
+ displayName: Cache Maven local repo
+
+ # download artifacts
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ path: $(CACHE_FLINK_DIR)
+ artifact: FlinkCompileCacheDir-${{parameters.stage_name}}
+ - script: ./tools/travis/setup_maven.sh
+ - script: ./tools/azure-pipelines/prepare_precommit.sh
+ displayName: prepare and build Flink
+ - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh
+ displayName: Test - precommit
+
+- job: e2e_${{parameters.stage_name}}
+ condition: eq(variables['MODE'], 'e2e')
+ # We are not running this job on a container, but in a VM.
+ pool: ${{parameters.e2e_pool_definition}}
+ timeoutInMinutes: 240
+ cancelTimeoutInMinutes: 1
+ workspace:
+ clean: all
+ steps:
+ - task: Cache@2
+ inputs:
+ key: $(CACHE_KEY)
+ restoreKeys: $(CACHE_FALLBACK_KEY)
+ path: $(MAVEN_CACHE_FOLDER)
+ displayName: Cache Maven local repo
+ - script: ./tools/travis/setup_maven.sh
+ - script: ./tools/azure-pipelines/setup_kubernetes.sh
+ - script: M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -De2e-metrics -Dmaven.wagon.http.pool=false" STAGE=compile ./tools/azure_controller.sh compile
+ displayName: Build
+ - script: FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh
+ displayName: Run nightly e2e tests
+
diff --git a/tools/azure-pipelines/prepare_precommit.sh b/tools/azure-pipelines/prepare_precommit.sh
new file mode 100755
index 0000000..71a231c
--- /dev/null
+++ b/tools/azure-pipelines/prepare_precommit.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+echo "Merging cache"
+cp -RT "$CACHE_FLINK_DIR" "."
+echo "Adjusting timestamps"
+# adjust timestamps to prevent recompilation
+find . -type f -name '*.java' | xargs touch
+find . -type f -name '*.scala' | xargs touch
+# wait a bit for better odds of different timestamps
+sleep 5
+find . -type f -name '*.class' | xargs touch
+find . -type f -name '*.timestamp' | xargs touch
+
+
+export M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/
+export PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH
+mvn -version
+mvn install --settings ./tools/azure-pipelines/google-mirror-settings.xml -DskipTests -Drat.skip
+
+
+chmod -R +x build-target
+chmod -R +x flink-end-to-end-tests
diff --git a/azure-pipelines.yml b/tools/azure-pipelines/setup_kubernetes.sh
old mode 100644
new mode 100755
similarity index 66%
copy from azure-pipelines.yml
copy to tools/azure-pipelines/setup_kubernetes.sh
index cc41f08..19eb50a
--- a/azure-pipelines.yml
+++ b/tools/azure-pipelines/setup_kubernetes.sh
@@ -13,23 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-trigger:
- branches:
- include:
- - '*'
-
-resources:
- containers:
- # Container with Maven 3.2.5 to have the same environment everywhere.
- - container: flink-build-container
- image: rmetzger/flink-ci:3
- repositories:
- - repository: templates
- type: github
- name: flink-ci/flink-azure-builds
- endpoint: flink-ci
-
-jobs:
-- template: flink-build-jobs.yml@templates
-
+echo "Replace moby by docker"
+docker version
+sudo apt-get remove -y moby-engine
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+docker version
diff --git a/tools/azure_controller.sh b/tools/azure_controller.sh
new file mode 100755
index 0000000..2682fcb
--- /dev/null
+++ b/tools/azure_controller.sh
@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+echo $M2_HOME
+echo $PATH
+echo $MAVEN_OPTS
+
+mvn -version
+echo "Commit: $(git rev-parse HEAD)"
+
+
+
+HERE="`dirname \"$0\"`" # relative
+HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
+if [ -z "$HERE" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+source "${HERE}/travis/stage.sh"
+source "${HERE}/travis/shade.sh"
+
+print_system_info() {
+ echo "CPU information"
+ lscpu
+
+ echo "Memory information"
+ cat /proc/meminfo
+
+ echo "Disk information"
+ df -hH
+
+ echo "Running build as"
+ whoami
+}
+
+print_system_info
+
+
+STAGE=$1
+echo "Current stage: \"$STAGE\""
+
+EXIT_CODE=0
+
+# Set up a custom Maven settings file, configuring an Google-hosted maven central
+# mirror. We use a different mirror because the official maven central mirrors
+# often lead to connection timeouts (probably due to rate-limiting)
+
+# adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html)
+MVN="mvn clean install --settings ./tools/azure-pipelines/google-mirror-settings.xml $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -U -DskipTests $PROFILE"
+
+# Run actual compile&test steps
+if [ $STAGE == "$STAGE_COMPILE" ]; then
+ # run mvn clean install:
+ $MVN
+ EXIT_CODE=$?
+
+ if [ $EXIT_CODE == 0 ]; then
+ echo "\n\n==============================================================================\n"
+ echo "Checking scala suffixes\n"
+ echo "==============================================================================\n"
+
+ ./tools/verify_scala_suffixes.sh "${PROFILE}"
+ EXIT_CODE=$?
+ else
+ echo "\n==============================================================================\n"
+ echo "Previous build failure detected, skipping scala-suffixes check.\n"
+ echo "==============================================================================\n"
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ check_shaded_artifacts
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_s3_fs hadoop
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_s3_fs presto
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_connector_elasticsearch 2
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_connector_elasticsearch 5
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_connector_elasticsearch 6
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ else
+ echo "=============================================================================="
+ echo "Previous build failure detected, skipping shaded dependency check."
+ echo "=============================================================================="
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ echo "Creating cache build directory $CACHE_FLINK_DIR"
+
+ cp -r . "$CACHE_FLINK_DIR"
+
+ function minimizeCachedFiles() {
+ # reduces the size of the cached directory to speed up
+ # the packing&upload / download&unpacking process
+ # by removing files not required for subsequent stages
+
+ # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided)
+ find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' \
+ ! -path "$CACHE_FLINK_DIR/flink-formats/flink-csv/target/flink-csv*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-formats/flink-json/target/flink-json*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-formats/flink-avro/target/flink-avro*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-runtime/target/flink-runtime*tests.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-streaming-java/target/flink-streaming-java*tests.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-dist*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table_*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table-blink*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/opt/flink-python*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-elasticsearch-base/target/flink-*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-kafka-base/target/flink-*.jar" \
+ ! -path "$CACHE_FLINK_DIR/flink-table/flink-table-planner/target/flink-table-planner*tests.jar" | xargs rm -rf
+
+ # .git directory
+ # not deleting this can cause build stability issues
+ # merging the cached version sometimes fails
+ rm -rf "$CACHE_FLINK_DIR/.git"
+
+ # AZ Pipelines has a problem with links.
+ rm "$CACHE_FLINK_DIR/build-target"
+ }
+
+ echo "Minimizing cache"
+ minimizeCachedFiles
+ else
+ echo "=============================================================================="
+ echo "Previous build failure detected, skipping cache setup."
+ echo "=============================================================================="
+ fi
+elif [ $STAGE != "$STAGE_CLEANUP" ]; then
+ if ! [ -e $CACHE_FLINK_DIR ]; then
+ echo "Cached flink dir $CACHE_FLINK_DIR does not exist. Exiting build."
+ exit 1
+ fi
+ # merged compiled flink into local clone
+ # this prevents the cache from being re-uploaded
+ echo "Merging cache"
+ cp -RT "$CACHE_FLINK_DIR" "."
+
+ echo "Adjusting timestamps"
+ # adjust timestamps to prevent recompilation
+ find . -type f -name '*.java' | xargs touch
+ find . -type f -name '*.scala' | xargs touch
+ # wait a bit for better odds of different timestamps
+ sleep 5
+ find . -type f -name '*.class' | xargs touch
+ find . -type f -name '*.timestamp' | xargs touch
+
+ if [ $STAGE == $STAGE_PYTHON ]; then
+ echo "=============================================================================="
+ echo "Python stage found. Re-compiling (this is required on Azure for the python tests to pass)"
+ echo "=============================================================================="
+ # run mvn install (w/o "clean"):
+ PY_MVN="${MVN// clean/}"
+ PY_MVN="$PY_MVN -Drat.skip=true"
+ ${PY_MVN}
+ echo "Done compiling ... "
+ fi
+
+
+ TEST="$STAGE" "./tools/travis_watchdog.sh" 300
+ EXIT_CODE=$?
+elif [ $STAGE == "$STAGE_CLEANUP" ]; then
+ echo "Cleaning up $CACHE_BUILD_DIR"
+ rm -rf "$CACHE_BUILD_DIR"
+else
+ echo "Invalid Stage specified: $STAGE"
+ exit 1
+fi
+
+# Exit code for Azure build success/failure
+exit $EXIT_CODE
diff --git a/tools/travis_watchdog.sh b/tools/travis_watchdog.sh
index 5e3e4e3..e96934c 100755
--- a/tools/travis_watchdog.sh
+++ b/tools/travis_watchdog.sh
@@ -95,6 +95,11 @@ UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY
ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz
+if [ ! -z "$TF_BUILD" ] ; then
+ # set proper artifacts file name on Azure Pipelines
+ ARTIFACTS_FILE=${BUILD_BUILDNUMBER}.tar.gz
+fi
+
if [ $TEST == $STAGE_PYTHON ]; then
CMD=$PYTHON_TEST
CMD_PID=$PYTHON_PID
@@ -273,18 +278,22 @@ cd ../../
# only run end-to-end tests in misc because we only have flink-dist here
case $TEST in
(misc)
- if [ $EXIT_CODE == 0 ]; then
- echo "\n\n==============================================================================\n"
- echo "Running bash end-to-end tests\n"
- echo "==============================================================================\n"
-
- FLINK_DIR=build-target flink-end-to-end-tests/run-pre-commit-tests.sh
-
- EXIT_CODE=$?
- else
- echo "\n==============================================================================\n"
- echo "Previous build failure detected, skipping bash end-to-end tests.\n"
- echo "==============================================================================\n"
+ # If we are not on Azure (we are on Travis) run precommit tests in misc stage.
+ # On Azure, we run them in a separate job
+ if [ -z "$TF_BUILD" ] ; then
+ if [ $EXIT_CODE == 0 ]; then
+ echo "\n\n==============================================================================\n"
+ echo "Running bash end-to-end tests\n"
+ echo "==============================================================================\n"
+
+ FLINK_DIR=build-target flink-end-to-end-tests/run-pre-commit-tests.sh
+
+ EXIT_CODE=$?
+ else
+ echo "\n==============================================================================\n"
+ echo "Previous build failure detected, skipping bash end-to-end tests.\n"
+ echo "==============================================================================\n"
+ fi
fi
if [ $EXIT_CODE == 0 ]; then
echo "\n\n==============================================================================\n"