You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/14 09:02:02 UTC
[airflow] 05/07: Combine back multiple test types into single jobs
(#11504)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 37bc926675041495033b37f3fa5371cde1b6799c
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Oct 13 20:51:08 2020 +0200
Combine back multiple test types into single jobs (#11504)
Seems that by splitting the tests into many small jobs has a bad
effect - since we only have queue size = 180 for the whole Apache
organisation, we are competing with other projects for the jobs
and with the jobs being so short we got starved much more than if
we had long jobs. Therefore we are re-combining the test types into
single jobs per Python version/Database version and run all the
tests sequentially on those machines.
(cherry picked from commit 4297abab26f2b8de48055c1838253d3b5aafcb48)
---
.github/workflows/ci.yml | 24 +++----
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 3 +-
scripts/ci/selective_tests.sh | 15 ++--
scripts/ci/testing/ci_run_airflow_testing.sh | 90 ++++++++++++++++++------
4 files changed, 87 insertions(+), 45 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e5f4897..ddec3e0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -213,28 +213,21 @@ jobs:
tests-postgres:
timeout-minutes: 60
- name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
+ name: "Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
- test-type: ${{ fromJson(needs.build-info.outputs.testTypes) }}
exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
- # Additionally include all postgres-only tests but only for default versions
- # in case the regular tests are excluded in the future
- include:
- - python_version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- postgres-version: ${{ needs.build-info.outputs.defaultPostgresVersion }}
- test-type: "Postgres"
fail-fast: false
env:
BACKEND: postgres
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
POSTGRES_VERSION: ${{ matrix.postgres-version }}
RUN_TESTS: true
- TEST_TYPE: ${{ matrix.test-type }}
+ TEST_TYPES: ${{needs.build-info.outputs.testTypes}}
if: >
needs.build-info.outputs.run-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -266,14 +259,13 @@ jobs:
tests-mysql:
timeout-minutes: 60
- name: "${{matrix.test-type}}:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}"
+ name: "MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
- test-type: ${{ fromJson(needs.build-info.outputs.testTypes) }}
exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
# Additionally include all mysql-only tests but only for default versions
# in case the regular tests are excluded in the future
@@ -287,7 +279,7 @@ jobs:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
MYSQL_VERSION: ${{ matrix.mysql-version }}
RUN_TESTS: true
- TEST_TYPE: ${{ matrix.test-type }}
+ TEST_TYPES: ${{needs.build-info.outputs.testTypes}}
if: >
needs.build-info.outputs.run-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -318,7 +310,7 @@ jobs:
tests-sqlite:
timeout-minutes: 60
- name: "${{matrix.test-type}}:Sqlite Py${{matrix.python-version}}"
+ name: "Sqlite Py${{matrix.python-version}}"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
strategy:
@@ -331,7 +323,7 @@ jobs:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
RUN_TESTS: true
- TEST_TYPE: ${{ matrix.test-type }}
+ TEST_TYPES: ${{needs.build-info.outputs.testTypes}}
if: >
needs.build-info.outputs.run-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -378,7 +370,7 @@ jobs:
MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
RUN_TESTS: true
- TEST_TYPE: Quarantined
+ TEST_TYPES: "Quarantined"
NUM_RUNS: 10
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: >
@@ -529,7 +521,7 @@ jobs:
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v2
env:
- cache-name: cache-kubernetes-tests-virtualenv-v4
+ cache-name: cache-kubernetes-tests-virtualenv-v6
with:
path: .build/.kubernetes_venv
key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}-v1"
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index 98950b2..39fafc3 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -83,10 +83,11 @@ fi
. "${virtualenv_path}/bin/activate"
+pip install --upgrade pip==20.2.3
+
pip install pytest freezegun pytest-cov \
--constraint "https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
-
pip install -e ".[kubernetes]" \
--constraint "https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
diff --git a/scripts/ci/selective_tests.sh b/scripts/ci/selective_tests.sh
index 897dc78..66cf709 100755
--- a/scripts/ci/selective_tests.sh
+++ b/scripts/ci/selective_tests.sh
@@ -61,14 +61,13 @@ function output_all_basic_variables() {
function set_outputs_run_all_tests() {
initialization::ga_output run-tests "true"
initialization::ga_output run-kubernetes-tests "true"
- initialization::ga_output test-types \
- '["Core", "Other", "API", "CLI", "Providers", "WWW", "Integration", "Heisentests"]'
+ initialization::ga_output test-types "Core Other API CLI Providers WWW Integration Heisentests"
}
function set_output_skip_all_tests() {
initialization::ga_output run-tests "false"
initialization::ga_output run-kubernetes-tests "false"
- initialization::ga_output test-types '[]'
+ initialization::ga_output test-types ""
}
function initialize_git_repo() {
@@ -312,27 +311,27 @@ function calculate_test_types_to_run() {
echo
echo "Adding API to selected files as ${COUNT_API_CHANGED_FILES} API files changed"
echo
- SELECTED_TESTS="${SELECTED_TESTS}, \"API\""
+ SELECTED_TESTS="${SELECTED_TESTS} API"
fi
if [[ ${COUNT_CLI_CHANGED_FILES} != "0" ]]; then
echo
echo "Adding CLI to selected files as ${COUNT_CLI_CHANGED_FILES} CLI files changed"
echo
- SELECTED_TESTS="${SELECTED_TESTS}, \"CLI\""
+ SELECTED_TESTS="${SELECTED_TESTS} CLI"
fi
if [[ ${COUNT_PROVIDERS_CHANGED_FILES} != "0" ]]; then
echo
echo "Adding Providers to selected files as ${COUNT_PROVIDERS_CHANGED_FILES} Provider files changed"
echo
- SELECTED_TESTS="${SELECTED_TESTS}, \"Providers\""
+ SELECTED_TESTS="${SELECTED_TESTS} Providers"
fi
if [[ ${COUNT_WWW_CHANGED_FILES} != "0" ]]; then
echo
echo "Adding WWW to selected files as ${COUNT_WWW_CHANGED_FILES} WWW files changed"
echo
- SELECTED_TESTS="${SELECTED_TESTS}, \"WWW\""
+ SELECTED_TESTS="${SELECTED_TESTS} WWW"
fi
- initialization::ga_output test-types "[ \"Integration\", \"Heisentests\" ${SELECTED_TESTS} ]"
+ initialization::ga_output test-types "Integration Heisentests ${SELECTED_TESTS}"
fi
}
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index 5c24e48..d56ae2c 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -20,23 +20,6 @@
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-INTEGRATIONS=()
-
-ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""}
-
-if [[ ${TEST_TYPE:=} == "Integration" ]]; then
- export ENABLED_INTEGRATIONS="${AVAILABLE_INTEGRATIONS}"
- export RUN_INTEGRATION_TESTS="${AVAILABLE_INTEGRATIONS}"
-fi
-
-for _INT in ${ENABLED_INTEGRATIONS}
-do
- INTEGRATIONS+=("-f")
- INTEGRATIONS+=("${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml")
-done
-
-readonly INTEGRATIONS
-
if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
echo
echo "Skipping running tests !!!!!"
@@ -51,6 +34,23 @@ function run_airflow_testing_in_docker() {
for try_num in {1..3}
do
echo
+ echo "Making sure docker-compose is down"
+ echo
+ docker-compose --log-level INFO -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
+ down --remove-orphans --timeout 10
+ echo
+ echo "System-prune docker"
+ echo
+ docker system prune --force
+ echo
+ echo "Check available space"
+ echo
+ df --human
+ echo
+ echo "Check available memory"
+ echo
+ free --human
+ echo
echo "Starting try number ${try_num}"
echo
docker-compose --log-level INFO \
@@ -127,8 +127,58 @@ echo
echo "Using docker image: ${AIRFLOW_CI_IMAGE} for docker compose runs"
echo
-RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""}
-readonly RUN_INTEGRATION_TESTS
+if [[ ${TEST_TYPE=} != "" ]]; then
+ # Handle case where test type is passed from outside
+ export TEST_TYPES="${TEST_TYPE}"
+fi
-run_airflow_testing_in_docker "${@}"
+if [[ ${TEST_TYPES=} == "" ]]; then
+ TEST_TYPES="Core Providers API CLI Integration Other WWW Heisentests"
+ echo
+ echo "Test types not specified. Running all: ${TEST_TYPES}"
+ echo
+fi
+
+if [[ ${TEST_TYPE=} != "" ]]; then
+ # Add Postgres/MySQL special test types in case we are running several test types
+ if [[ ${BACKEND} == "postgres" ]]; then
+ TEST_TYPES="${TEST_TYPES} Postgres"
+ fi
+ if [[ ${BACKEND} == "mysql" ]]; then
+ TEST_TYPES="${TEST_TYPES} MySQL"
+ fi
+fi
+readonly TEST_TYPES
+
+export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""}
+export ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""}
+
+echo "Running TEST_TYPES: ${TEST_TYPES}"
+
+for TEST_TYPE in ${TEST_TYPES}
+do
+ INTEGRATIONS=()
+ export INTEGRATIONS
+
+ if [[ ${TEST_TYPE:=} == "Integration" ]]; then
+ export ENABLED_INTEGRATIONS="${AVAILABLE_INTEGRATIONS}"
+ export RUN_INTEGRATION_TESTS="${AVAILABLE_INTEGRATIONS}"
+ fi
+
+ for _INT in ${ENABLED_INTEGRATIONS}
+ do
+ INTEGRATIONS+=("-f")
+ INTEGRATIONS+=("${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml")
+ done
+
+
+ export TEST_TYPE
+ echo "**********************************************************************************************"
+ echo
+ echo " TEST_TYPE: ${TEST_TYPE}"
+ echo
+ echo "**********************************************************************************************"
+
+ run_airflow_testing_in_docker "${@}"
+done