You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by di...@apache.org on 2020/06/15 18:24:27 UTC
[airflow] branch v1-10-test updated: Upload kind logs to Github
Actions artifact (#9230)
This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push:
new d659f33 Upload kind logs to Github Actions artifact (#9230)
d659f33 is described below
commit d659f3309a5a465cc7781c17a9c6e88af8e1bc79
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Thu Jun 11 16:00:27 2020 +0100
Upload kind logs to Github Actions artifact (#9230)
* Upload kind logs to Github Artifacts
* get_ci_environment needed to correctly dump kind logs
Without this the "setup cluster" step dumps the logs as
`kind_logs_2020-06-11_default_default.tar.gz`
Since this is now used outside of just building images, I have moved the
function to _initialization.sh
* Github already compresses artifacts for download.
If we upload a .tar.gz, the download is a .zip file containing that
.tar.gz, which isn't ideal
Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
(cherry picked from commit 9c82c5b21b2a1c19020f178bd1c38596e0603a54)
---
.github/workflows/ci.yml | 6 ++
scripts/ci/ci_deploy_app_to_kubernetes.sh | 5 ++
scripts/ci/ci_perform_kind_cluster_operation.sh | 6 ++
scripts/ci/ci_run_kubernetes_tests.sh | 5 ++
scripts/ci/libraries/_build_images.sh | 81 ------------------------
scripts/ci/libraries/_initialization.sh | 82 +++++++++++++++++++++++++
scripts/ci/libraries/_kind.sh | 23 ++-----
7 files changed, 109 insertions(+), 99 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d4f4f79..de37788 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -151,6 +151,12 @@ jobs:
${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}"
- name: "Tests"
run: ./scripts/ci/ci_run_kubernetes_tests.sh
+ - uses: actions/upload-artifact@v2
+ # Always run this, even if one of th previous steps failed.
+ if: always()
+ with:
+ name: 'kind-logs-${{matrix.kube-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}'
+ path: '/tmp/kind_logs_*'
pyfiles:
timeout-minutes: 10
diff --git a/scripts/ci/ci_deploy_app_to_kubernetes.sh b/scripts/ci/ci_deploy_app_to_kubernetes.sh
index 6061c1c..5b4a2a4 100755
--- a/scripts/ci/ci_deploy_app_to_kubernetes.sh
+++ b/scripts/ci/ci_deploy_app_to_kubernetes.sh
@@ -25,6 +25,11 @@ export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
export KUBERNETES_MODE=${KUBERNETES_MODE:="image"}
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
+
get_ci_environment
check_kind_and_kubectl_are_installed
build_kubernetes_image
diff --git a/scripts/ci/ci_perform_kind_cluster_operation.sh b/scripts/ci/ci_perform_kind_cluster_operation.sh
index 7963344..419b7af 100755
--- a/scripts/ci/ci_perform_kind_cluster_operation.sh
+++ b/scripts/ci/ci_perform_kind_cluster_operation.sh
@@ -19,6 +19,12 @@
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
+
+get_ci_environment
check_kind_and_kubectl_are_installed
perform_kind_cluster_operation "${@}"
diff --git a/scripts/ci/ci_run_kubernetes_tests.sh b/scripts/ci/ci_run_kubernetes_tests.sh
index c530112..0e2a87f 100755
--- a/scripts/ci/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/ci_run_kubernetes_tests.sh
@@ -18,6 +18,11 @@
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
+
INTERACTIVE="false"
declare -a TESTS
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 6561184..83cce06 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -309,87 +309,6 @@ function print_build_info() {
print_info
}
-# Retrieves CI environment variables needed - depending on the CI system we run it in.
-# We try to be CI - agnostic and our scripts should run the same way on different CI systems
-# (This makes it easy to move between different CI systems)
-# This function maps CI-specific variables into a generic ones (prefixed with CI_) that
-# we used in other scripts
-function get_ci_environment() {
- export CI_EVENT_TYPE="manual"
- export CI_TARGET_REPO="apache/airflow"
- export CI_TARGET_BRANCH="master"
- export CI_SOURCE_REPO="apache/airflow"
- export CI_SOURCE_BRANCH="master"
- export CI_BUILD_ID="default-build-id"
- export CI_JOB_ID="default-job-id"
- if [[ ${CI:=} != "true" ]]; then
- echo
- echo "This is not a CI environment!. Staying with the defaults"
- echo
- else
- if [[ ${TRAVIS:=} == "true" ]]; then
- export CI_TARGET_REPO="${TRAVIS_REPO_SLUG}"
- export CI_TARGET_BRANCH="${TRAVIS_BRANCH}"
- export CI_BUILD_ID="${TRAVIS_BUILD_ID}"
- export CI_JOB_ID="${TRAVIS_JOB_ID}"
- if [[ "${TRAVIS_PULL_REQUEST:=}" == "true" ]]; then
- export CI_EVENT_TYPE="pull_request"
- export CI_SOURCE_REPO="${TRAVIS_PULL_REQUEST_SLUG}"
- export CI_SOURCE_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH}"
- elif [[ "${TRAVIS_EVENT_TYPE:=}" == "cron" ]]; then
- export CI_EVENT_TYPE="schedule"
- else
- export CI_EVENT_TYPE="push"
- fi
- elif [[ ${GITHUB_ACTIONS:=} == "true" ]]; then
- export CI_TARGET_REPO="${GITHUB_REPOSITORY}"
- export CI_TARGET_BRANCH="${GITHUB_BASE_REF}"
- export CI_BUILD_ID="${GITHUB_RUN_ID}"
- export CI_JOB_ID="${GITHUB_JOB}"
- if [[ ${GITHUB_EVENT_NAME:=} == "pull_request" ]]; then
- export CI_EVENT_TYPE="pull_request"
- # default name of the source repo (assuming it's forked without rename)
- export SOURCE_AIRFLOW_REPO=${SOURCE_AIRFLOW_REPO:="airflow"}
- # For Pull Requests it's ambiguous to find the PR and we need to
- # assume that name of repo is airflow but it could be overridden in case it's not
- export CI_SOURCE_REPO="${GITHUB_ACTOR}/${SOURCE_AIRFLOW_REPO}"
- export CI_SOURCE_BRANCH="${GITHUB_HEAD_REF}"
- BRANCH_EXISTS=$(git ls-remote --heads \
- "https://github.com/${CI_SOURCE_REPO}.git" "${CI_SOURCE_BRANCH}" || true)
- if [[ ${BRANCH_EXISTS} == "" ]]; then
- echo
- echo "https://github.com/${CI_SOURCE_REPO}.git Branch ${CI_SOURCE_BRANCH} does not exist"
- echo
- echo
- echo "Fallback to https://github.com/${CI_TARGET_REPO}.git Branch ${CI_TARGET_BRANCH}"
- echo
- # Fallback to the target repository if the repo does not exist
- export CI_SOURCE_REPO="${CI_TARGET_REPO}"
- export CI_SOURCE_BRANCH="${CI_TARGET_BRANCH}"
- fi
- elif [[ ${GITHUB_EVENT_TYPE:=} == "schedule" ]]; then
- export CI_EVENT_TYPE="schedule"
- else
- export CI_EVENT_TYPE="push"
- fi
- else
- echo
- echo "ERROR! Unknown CI environment. Exiting"
- exit 1
- fi
- fi
- echo
- echo "Detected CI build environment"
- echo
- echo "CI_EVENT_TYPE=${CI_EVENT_TYPE}"
- echo "CI_TARGET_REPO=${CI_TARGET_REPO}"
- echo "CI_TARGET_BRANCH=${CI_TARGET_BRANCH}"
- echo "CI_SOURCE_REPO=${CI_SOURCE_REPO}"
- echo "CI_SOURCE_BRANCH=${CI_SOURCE_BRANCH}"
- echo "CI_BUILD_ID=${CI_BUILD_ID}"
- echo "CI_JOB_ID=${CI_JOB_ID}"
- echo
-}
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index c8cd9f7..6f66b0d 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -233,3 +233,85 @@ function set_mysql_encoding() {
export MYSQL_ENCODING=utf8
fi
}
+
+# Retrieves CI environment variables needed - depending on the CI system we run it in.
+# We try to be CI - agnostic and our scripts should run the same way on different CI systems
+# (This makes it easy to move between different CI systems)
+# This function maps CI-specific variables into a generic ones (prefixed with CI_) that
+# we used in other scripts
+function get_ci_environment() {
+ export CI_EVENT_TYPE="manual"
+ export CI_TARGET_REPO="apache/airflow"
+ export CI_TARGET_BRANCH="master"
+ export CI_SOURCE_REPO="apache/airflow"
+ export CI_SOURCE_BRANCH="master"
+ export CI_BUILD_ID="default-build-id"
+ export CI_JOB_ID="default-job-id"
+ if [[ ${CI:=} != "true" ]]; then
+ echo
+ echo "This is not a CI environment!. Staying with the defaults"
+ echo
+ else
+ if [[ ${TRAVIS:=} == "true" ]]; then
+ export CI_TARGET_REPO="${TRAVIS_REPO_SLUG}"
+ export CI_TARGET_BRANCH="${TRAVIS_BRANCH}"
+ export CI_BUILD_ID="${TRAVIS_BUILD_ID}"
+ export CI_JOB_ID="${TRAVIS_JOB_ID}"
+ if [[ "${TRAVIS_PULL_REQUEST:=}" == "true" ]]; then
+ export CI_EVENT_TYPE="pull_request"
+ export CI_SOURCE_REPO="${TRAVIS_PULL_REQUEST_SLUG}"
+ export CI_SOURCE_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH}"
+ elif [[ "${TRAVIS_EVENT_TYPE:=}" == "cron" ]]; then
+ export CI_EVENT_TYPE="schedule"
+ else
+ export CI_EVENT_TYPE="push"
+ fi
+ elif [[ ${GITHUB_ACTIONS:=} == "true" ]]; then
+ export CI_TARGET_REPO="${GITHUB_REPOSITORY}"
+ export CI_TARGET_BRANCH="${GITHUB_BASE_REF}"
+ export CI_BUILD_ID="${GITHUB_RUN_ID}"
+ export CI_JOB_ID="${GITHUB_JOB}"
+ if [[ ${GITHUB_EVENT_NAME:=} == "pull_request" ]]; then
+ export CI_EVENT_TYPE="pull_request"
+ # default name of the source repo (assuming it's forked without rename)
+ export SOURCE_AIRFLOW_REPO=${SOURCE_AIRFLOW_REPO:="airflow"}
+ # For Pull Requests it's ambiguous to find the PR and we need to
+ # assume that name of repo is airflow but it could be overridden in case it's not
+ export CI_SOURCE_REPO="${GITHUB_ACTOR}/${SOURCE_AIRFLOW_REPO}"
+ export CI_SOURCE_BRANCH="${GITHUB_HEAD_REF}"
+ BRANCH_EXISTS=$(git ls-remote --heads \
+ "https://github.com/${CI_SOURCE_REPO}.git" "${CI_SOURCE_BRANCH}" || true)
+ if [[ ${BRANCH_EXISTS} == "" ]]; then
+ echo
+ echo "https://github.com/${CI_SOURCE_REPO}.git Branch ${CI_SOURCE_BRANCH} does not exist"
+ echo
+ echo
+ echo "Fallback to https://github.com/${CI_TARGET_REPO}.git Branch ${CI_TARGET_BRANCH}"
+ echo
+ # Fallback to the target repository if the repo does not exist
+ export CI_SOURCE_REPO="${CI_TARGET_REPO}"
+ export CI_SOURCE_BRANCH="${CI_TARGET_BRANCH}"
+ fi
+ elif [[ ${GITHUB_EVENT_TYPE:=} == "schedule" ]]; then
+ export CI_EVENT_TYPE="schedule"
+ else
+ export CI_EVENT_TYPE="push"
+ fi
+ else
+ echo
+ echo "ERROR! Unknown CI environment. Exiting"
+ exit 1
+ fi
+ fi
+ echo
+ echo "Detected CI build environment"
+ echo
+ echo "CI_EVENT_TYPE=${CI_EVENT_TYPE}"
+ echo "CI_TARGET_REPO=${CI_TARGET_REPO}"
+ echo "CI_TARGET_BRANCH=${CI_TARGET_BRANCH}"
+ echo "CI_SOURCE_REPO=${CI_SOURCE_REPO}"
+ echo "CI_SOURCE_BRANCH=${CI_SOURCE_BRANCH}"
+ echo "CI_BUILD_ID=${CI_BUILD_ID}"
+ echo "CI_JOB_ID=${CI_JOB_ID}"
+ echo
+}
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index a3769a0..b89ea90 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -21,25 +21,12 @@ function dump_kind_logs() {
echo " Dumping logs from KIND"
echo "###########################################################################################"
- FILE_NAME="${1}"
- kind --name "${KIND_CLUSTER_NAME}" export logs "${FILE_NAME}"
-}
+ echo "EXIT_CODE is ${EXIT_CODE:=}"
-function send_kubernetes_logs_to_file_io() {
- echo "##############################################################################"
- echo
- echo " DUMPING LOG FILES FROM KIND AND SENDING THEM TO file.io"
- echo
- echo "##############################################################################"
- DUMP_DIR_NAME=$(date "+%Y-%m-%d")_kind_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}
- DUMP_DIR=/tmp/${DUMP_DIR_NAME}
- dump_kind_logs "${DUMP_DIR}"
- tar -cvzf "${DUMP_DIR}.tar.gz" -C /tmp "${DUMP_DIR_NAME}"
- echo
- echo " Logs saved to ${DUMP_DIR}.tar.gz"
- echo
- echo "##############################################################################"
- curl -F "file=@${DUMP_DIR}.tar.gz" https://file.io
+ local DUMP_DIR_NAME DUMP_DIR
+ DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}
+ DUMP_DIR="/tmp/${DUMP_DIR_NAME}"
+ kind --name "${KIND_CLUSTER_NAME}" export logs "${DUMP_DIR}"
}
function check_kind_and_kubectl_are_installed() {