You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/08/03 18:01:38 UTC
[airflow] branch master updated: Status of quarantined tests is
stored in Github Issue (#10119)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 86d8e34 Status of quarantined tests is stored in Github Issue (#10119)
86d8e34 is described below
commit 86d8e349b4e810c0dca56d5202f893628969970a
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Mon Aug 3 20:01:00 2020 +0200
Status of quarantined tests is stored in Github Issue (#10119)
---
.github/workflows/ci.yml | 44 +---
.github/workflows/quarantined.yaml | 116 ++++++++++
CI.rst | 63 +++++-
scripts/ci/docker-compose/base.yml | 4 +
scripts/ci/in_container/entrypoint_ci.sh | 32 ++-
scripts/ci/in_container/quarantine_issue_header.md | 32 +++
scripts/ci/in_container/run_ci_tests.sh | 25 +++
.../in_container/update_quarantined_test_status.py | 243 +++++++++++++++++++++
scripts/ci/libraries/_initialization.sh | 8 +
scripts/ci/testing/ci_run_airflow_testing.sh | 18 ++
setup.py | 3 +-
11 files changed, 539 insertions(+), 49 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ab6c745..3735d74 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,6 +49,7 @@ jobs:
steps:
- uses: potiuk/cancel-workflow-runs@v1
with:
+ workflow: ci.yaml
token: ${{ secrets.GITHUB_TOKEN }}
static-checks:
@@ -62,7 +63,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.x'
+ python-version: '3.7'
- name: Cache pre-commit env
uses: actions/cache@v2
env:
@@ -100,7 +101,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.x'
+ python-version: '3.7'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
@@ -154,7 +155,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.x'
+ python-version: '3.7'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- uses: engineerd/setup-kind@v0.4.0
@@ -204,7 +205,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.x'
+ python-version: '3.7'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
@@ -234,7 +235,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.x'
+ python-version: '3.7'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
@@ -262,38 +263,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.x'
- - name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- - name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
-
- tests-quarantined:
- timeout-minutes: 80
- name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
- runs-on: ubuntu-latest
- continue-on-error: true
- needs: [trigger-tests]
- strategy:
- matrix:
- python-version: [3.6]
- postgres-version: [9.6]
- test-type: [Quarantined]
- fail-fast: false
- env:
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- POSTGRES_VERSION: ${{ matrix.postgres-version }}
- RUN_TESTS: "true"
- TEST_TYPE: ${{ matrix.test-type }}
- if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: '3.x'
+ python-version: '3.7'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
diff --git a/.github/workflows/quarantined.yaml b/.github/workflows/quarantined.yaml
new file mode 100644
index 0000000..75135ec
--- /dev/null
+++ b/.github/workflows/quarantined.yaml
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Quarantined Build
+on:
+ schedule:
+ # Run quarantined builds 4 times a day to gather better quarantine stats
+ - cron: '35 */6 * * *'
+ push:
+ branches: ['master', 'v1-10-test', 'v1-10-stable']
+ pull_request:
+ branches: ['master', 'v1-10-test', 'v1-10-stable']
+
+env:
+ MOUNT_LOCAL_SOURCES: "true"
+ FORCE_ANSWER_TO_QUESTIONS: "yes"
+ SKIP_CHECK_REMOTE_IMAGE: "true"
+ SKIP_CI_IMAGE_CHECK: "true"
+ DB_RESET: "true"
+ VERBOSE: "true"
+ UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }}
+ PYTHON_MAJOR_MINOR_VERSION: 3.6
+ USE_GITHUB_REGISTRY: "true"
+ CACHE_IMAGE_PREFIX: ${{ github.repository }}
+ CACHE_REGISTRY_USERNAME: ${{ github.actor }}
+ CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+
+ cancel-previous-workflow-run:
+ timeout-minutes: 60
+ name: "Cancel previous workflow run"
+ runs-on: ubuntu-latest
+ steps:
+ - uses: potiuk/cancel-workflow-runs@v1
+ with:
+ workflow: quarantined.yml
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ trigger-tests:
+ timeout-minutes: 5
+ name: "Checks if tests should be run"
+ runs-on: ubuntu-latest
+ needs: [cancel-previous-workflow-run]
+ outputs:
+ run-tests: ${{ steps.trigger-tests.outputs.run-tests }}
+ steps:
+ - uses: actions/checkout@v2
+ - name: "Check if tests should be run"
+ run: "./scripts/ci/tools/ci_check_if_tests_should_be_run.sh"
+ id: trigger-tests
+
+ tests-quarantined:
+ timeout-minutes: 80
+ name: "Quarantined tests"
+ runs-on: ubuntu-latest
+ continue-on-error: true
+ needs: [trigger-tests]
+ strategy:
+ matrix:
+ python-version: [3.6]
+ postgres-version: [9.6]
+ fail-fast: false
+ env:
+ BACKEND: postgres
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ POSTGRES_VERSION: ${{ matrix.postgres-version }}
+ RUN_TESTS: "true"
+ TEST_TYPE: Quarantined
+ NUM_RUNS: 10
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v2
+ with:
+ python-version: '3.7'
+ - name: "Set issue id for master"
+ if: github.ref == 'refs/heads/master'
+ run: |
+ echo "::set-env name=ISSUE_ID::86"
+ - name: "Set issue id for v1-10-stable"
+ if: github.ref == 'refs/heads/v1-10-stable'
+ run: |
+ echo "::set-env name=ISSUE_ID::10127"
+ - name: "Set issue id for v1-10-test"
+ if: github.ref == 'refs/heads/v1-10-test'
+ run: |
+ echo "::set-env name=ISSUE_ID::10128"
+ - name: "Free space"
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ - name: "Build CI image ${{ matrix.python-version }}"
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ - name: "Tests"
+ run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ - uses: actions/upload-artifact@v2
+ name: Upload Quarantine test results
+ if: always()
+ with:
+ name: 'quarantined_tests'
+ path: 'files/test_result.xml'
diff --git a/CI.rst b/CI.rst
index 41d22fb..2844a5c 100644
--- a/CI.rst
+++ b/CI.rst
@@ -67,8 +67,8 @@ The following components are part of the CI infrastructure
CI run types
============
-The following CI Job runs are currently run for Apache Airflow, and each of the runs have different
-purpose and context.
+The following CI Job run types are currently run for Apache Airflow (run by ci.yaml workflow and
+quarantined.yaml workflows) and each of the run types have different purpose and context.
Pull request run
----------------
@@ -126,7 +126,17 @@ DockerHub when pushing ``v1-10-stable`` manually.
All runs consist of the same jobs, but the jobs behave slightly differently or they are skipped in different
run categories. Here is a summary of the run categories with regards of the jobs they are running.
Those jobs often have matrix run strategy which runs several different variations of the jobs
-(with different Backend type / Python version, type of the tests to run for example)
+(with different Backend type / Python version, type of the tests to run for example). The following chapter
+describes the workflows that execute for each run.
+
+Workflows
+=========
+
+CI Build Workflow
+-----------------
+
+This workflow is a regular workflow that performs the regular checks - none of the jobs should fail.
+The tests to run do not contain quarantined tests.
+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run |
@@ -148,8 +158,6 @@ Those jobs often have matrix run strategy which runs several different variation
+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
| Tests Kubernetes | Run Kubernetes test | Yes (if tests-triggered) | Yes | Yes * |
+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
-| Quarantined tests | Those are tests that are flaky and we need to fix them | Yes (if tests-triggered) | Yes | Yes * |
-+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
| Test OpenAPI client gen | Tests if OpenAPIClient continues to generate | Yes | Yes | Yes * |
+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
| Helm tests | Runs tests for the Helm chart | Yes | Yes | Yes * |
@@ -164,3 +172,48 @@ Those jobs often have matrix run strategy which runs several different variation
+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
| Tag Repo nightly | Tags the repository with nightly tagIt is a lightweight tag that moves nightly | - | - | Yes. Triggers DockerHub build for public registry |
+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+
+Quarantined build workflow
+--------------------------
+
+This workflow runs only quarantined tests. Those tests do not fail the build even if some tests fail (only if
+the whole pytest execution fails). Instead this workflow updates one of the issues where we keep status
+of quarantined tests. Once the test succeeds in NUM_RUNS subsequent runs, it is marked as stable and
+can be removed from quarantine. You can read more about quarantine in `<TESTING.rst>`_
+
+The issues are only updated if the test is run as direct push or scheduled run and only in the
+``apache/airflow`` repository - so that the issues are not updated in forks.
+
+The issues that gets updated are different for different branches:
+
+* master: `Quarantine tests master <https://github.com/apache/airflow/issues/10118>`_
+* v1-10-stable: `Quarantine tests v1-10-stable <https://github.com/apache/airflow/issues/10127>`_
+* v1-10-test: `Quarantine tests v1-10-test <hhttps://github.com/apache/airflow/issues/10128>`_
+
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run |
++===========================+================================================================================================================+====================================+=================================+======================================================================+
+| Cancel previous workflow | Cancels the previously running workflow run if there is one running | Yes | Yes | Yes * |
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+| Trigger tests | Checks if tests should be triggered | Yes | Yes | Yes * |
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+| Quarantined tests | Those are tests that are flaky and we need to fix them | Yes (if tests-triggered) | Yes (Updates quarantine issue) | Yes * (updates quarantine issue) |
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+
+Cancel other workflow runs workflow
+-----------------------------------
+
+This workflow is run only on schedule (every 5 minutes) it's only purpose is to cancel other running
+``CI Build`` workflows if important jobs failed in those runs. This is to save runners for other runs
+in case we know that the build will not succeed anyway without some basic fixes to static checks or
+documentation - effectively implementing missing "fail-fast" (on a job level) in Github Actions
+similar to fail-fast in matrix strategy.
+
+The jobs that are considered as "fail-fast" are:
+
+* Static checks
+* Docs
+* Prepare Backport packages
+* Helm tests
+* Build Prod Image
+* TTest OpenAPI client gen
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 0feea60..9a364a2 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -39,6 +39,10 @@ services:
- RUN_INTEGRATION_TESTS
- ONLY_RUN_LONG_RUNNING_TESTS
- ONLY_RUN_QUARANTINED_TESTS
+ - GITHUB_TOKEN
+ - GITHUB_REPOSITORY
+ - ISSUE_ID
+ - NUM_RUNS
- BREEZE
- INSTALL_AIRFLOW_VERSION
- DB_RESET
diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh
index d006eaa..fa6091c 100755
--- a/scripts/ci/in_container/entrypoint_ci.sh
+++ b/scripts/ci/in_container/entrypoint_ci.sh
@@ -161,11 +161,12 @@ if [[ "${RUN_TESTS}" != "true" ]]; then
fi
set -u
+export RESULT_LOG_FILE="/files/test_result.xml"
+
if [[ "${CI}" == "true" ]]; then
EXTRA_PYTEST_ARGS=(
"--verbosity=0"
"--strict-markers"
- "--instafail"
"--durations=100"
"--cov=airflow/"
"--cov-config=.coveragerc"
@@ -174,6 +175,7 @@ if [[ "${CI}" == "true" ]]; then
"--maxfail=50"
"--pythonwarnings=ignore::DeprecationWarning"
"--pythonwarnings=ignore::PendingDeprecationWarning"
+ "--junitxml=${RESULT_LOG_FILE}"
)
else
EXTRA_PYTEST_ARGS=()
@@ -187,25 +189,43 @@ if [[ ${#@} -gt 0 && -n "$1" ]]; then
fi
if [[ -n ${RUN_INTEGRATION_TESTS:=""} ]]; then
+ # Integration tests
for INT in ${RUN_INTEGRATION_TESTS}
do
EXTRA_PYTEST_ARGS+=("--integration" "${INT}")
done
- EXTRA_PYTEST_ARGS+=("-rpfExX")
+ EXTRA_PYTEST_ARGS+=(
+ # timeouts in seconds for individual tests
+ "--setup-timeout=20"
+ "--execution-timeout=60"
+ "--teardown-timeout=20"
+ )
+
elif [[ ${ONLY_RUN_LONG_RUNNING_TESTS:=""} == "true" ]]; then
EXTRA_PYTEST_ARGS+=(
"-m" "long_running"
"--include-long-running"
"--verbosity=1"
- "--reruns" "3"
- "--timeout" "90")
+ "--setup-timeout=30"
+ "--execution-timeout=120"
+ "--teardown-timeout=30"
+ )
elif [[ ${ONLY_RUN_QUARANTINED_TESTS:=""} == "true" ]]; then
EXTRA_PYTEST_ARGS+=(
"-m" "quarantined"
"--include-quarantined"
"--verbosity=1"
- "--reruns" "3"
- "--timeout" "90")
+ "--setup-timeout=10"
+ "--execution-timeout=50"
+ "--teardown-timeout=10"
+ )
+else
+ # Core tests
+ EXTRA_PYTEST_ARGS+=(
+ "--setup-timeout=10"
+ "--execution-timeout=30"
+ "--teardown-timeout=10"
+ )
fi
ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}")
diff --git a/scripts/ci/in_container/quarantine_issue_header.md b/scripts/ci/in_container/quarantine_issue_header.md
new file mode 100644
index 0000000..d672a4d
--- /dev/null
+++ b/scripts/ci/in_container/quarantine_issue_header.md
@@ -0,0 +1,32 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Quarantined issues
+
+Please do not update status or list of the issues manually. It is automatically updated during
+Quarantine workflow, when the workflow executes in the context of Apache Airflow repository.
+This happens on schedule (4 times a day) or when a change has been merged or pushed
+to the relevant branch.
+
+You can update "Comment" column in the issue list - the update process will read and preserve this column.
+
+# Status update
+Last status update (UTC): {{ DATE_UTC_NOW }}
+
+# List of Quarantined issues
diff --git a/scripts/ci/in_container/run_ci_tests.sh b/scripts/ci/in_container/run_ci_tests.sh
index dc86cf0..6d36480 100755
--- a/scripts/ci/in_container/run_ci_tests.sh
+++ b/scripts/ci/in_container/run_ci_tests.sh
@@ -33,6 +33,31 @@ if [[ "${RES}" == "0" && ${CI:="false"} == "true" ]]; then
bash <(curl -s https://codecov.io/bash)
fi
+MAIN_GITHUB_REPOSITORY="apache/airflow"
+
+if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} = "true" ]]; then
+ if [[ ${GITHUB_REPOSITORY} == "${MAIN_GITHUB_REPOSITORY}" ]]; then
+ if [[ ${RES} == "1" || ${RES} == "0" ]]; then
+ echo
+ echo "Pytest exited with ${RES} result. Updating Quarantine Issue!"
+ echo
+ "${IN_CONTAINER_DIR}/update_quarantined_test_status.py" "${RESULT_LOG_FILE}"
+ else
+ echo
+ echo "Pytest exited with ${RES} result. NOT Updating Quarantine Issue!"
+ echo
+ fi
+ else
+ echo
+ echo "Github repository '${GITHUB_REPOSITORY}'. NOT Updating Quarantine Issue!"
+ echo
+ fi
+else
+ echo
+ echo "Regular tests. NOT Updating Quarantine Issue!"
+ echo
+fi
+
if [[ ${CI:=} == "true" ]]; then
send_airflow_logs_to_file_io
fi
diff --git a/scripts/ci/in_container/update_quarantined_test_status.py b/scripts/ci/in_container/update_quarantined_test_status.py
new file mode 100755
index 0000000..179ff91
--- /dev/null
+++ b/scripts/ci/in_container/update_quarantined_test_status.py
@@ -0,0 +1,243 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import sys
+from datetime import datetime
+from os.path import dirname, join, realpath
+from typing import Dict, List, NamedTuple, Optional
+from urllib.parse import urlsplit
+
+import jinja2
+from bs4 import BeautifulSoup
+from github3 import login
+from jinja2 import StrictUndefined
+from tabulate import tabulate
+
+
+class TestResult(NamedTuple):
+ test_id: str
+ file: str
+ name: str
+ classname: str
+ line: str
+ result: bool
+
+
+class TestHistory(NamedTuple):
+ test_id: str
+ name: str
+ url: str
+ states: List[bool]
+ comment: str
+
+
+test_results = []
+
+user = ""
+repo = ""
+issue_id = 0
+num_runs = 10
+
+url_pattern = re.compile(r'\[([^]]*)]\(([^)]*)\)')
+
+status_map: Dict[str, bool] = {
+ ":heavy_check_mark:": True,
+ ":x:": False,
+}
+
+reverse_status_map: Dict[bool, str] = {status_map[key]: key for key in status_map.keys()}
+
+
+def get_url(result: TestResult) -> str:
+ return f"[{result.name}](https://github.com/{user}/{repo}/blob/" \
+ f"master/{result.file}?test_id={result.test_id}#L{result.line})"
+
+
+def parse_state_history(history_string: str) -> List[bool]:
+ history_array = history_string.split(' ')
+ status_array: List[bool] = []
+ for value in history_array:
+ if value:
+ status_array.append(status_map[value])
+ return status_array
+
+
+def parse_test_history(line: str) -> Optional[TestHistory]:
+ values = line.split("|")
+ match_url = url_pattern.match(values[1].strip())
+ if match_url:
+ name = match_url.group(1)
+ url = match_url.group(0)
+ http_url = match_url.group(2)
+ parsed_url = urlsplit(http_url)
+ the_id = parsed_url[3].split("=")[1]
+ comment = values[4] if len(values) >= 5 else ""
+ # noinspection PyBroadException
+ try:
+ states = parse_state_history(values[3])
+ except Exception:
+ states = []
+ return TestHistory(
+ test_id=the_id,
+ name=name,
+ states=states,
+ url=url,
+ comment=comment,
+ )
+ return None
+
+
+def parse_body(body: str) -> Dict[str, TestHistory]:
+ parse = False
+ test_history_map: Dict[str, TestHistory] = {}
+ for line in body.splitlines(keepends=False):
+ if line.startswith("|-"):
+ parse = True
+ continue
+ if parse:
+ if not line.startswith("|"):
+ break
+ # noinspection PyBroadException
+ try:
+ status = parse_test_history(line)
+ except Exception:
+ continue
+ if status:
+ test_history_map[status.test_id] = status
+ return test_history_map
+
+
+def update_test_history(history: TestHistory, last_status: bool):
+ print(f"Adding status to test history: {history}, {last_status}")
+ return TestHistory(
+ test_id=history.test_id,
+ name=history.name,
+ url=history.url,
+ states=([last_status] + history.states)[0:num_runs],
+ comment=history.comment,
+ )
+
+
+def create_test_history(result: TestResult) -> TestHistory:
+ print(f"Creating test history {result}")
+ return TestHistory(
+ test_id=result.test_id,
+ name=result.name,
+ url=get_url(result),
+ states=[result.result],
+ comment=""
+ )
+
+
+def get_history_status(history: TestHistory):
+ if len(history.states) < num_runs:
+ if all(history.states):
+ return "So far, so good"
+ return "Flaky"
+ if all(history.states):
+ return "Stable"
+ if all(history.states[0:num_runs - 1]):
+ return "Just one more"
+ if all(history.states[0:int(num_runs / 2)]):
+ return "Almost there"
+ return "Flaky"
+
+
+def get_table(history_map: Dict[str, TestHistory]) -> str:
+ headers = ["Test", "Last run", f"Last {num_runs} runs", "Status", "Comment"]
+ the_table: List[List[str]] = []
+ for ordered_key in sorted(history_map.keys()):
+ history = history_map[ordered_key]
+ the_table.append([
+ history.url,
+ "Succeeded" if history.states[0] else "Failed",
+ " ".join([reverse_status_map[state] for state in history.states]),
+ get_history_status(history),
+ history.comment
+ ])
+ return tabulate(the_table, headers, tablefmt="github")
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print("Provide XML JUNIT FILE as first argument")
+ sys.exit(1)
+
+ with open(sys.argv[1], "r") as f:
+ text = f.read()
+ y = BeautifulSoup(text, "html.parser")
+ res = y.testsuites.testsuite.findAll("testcase")
+ for test in res:
+ print("Parsing: " + test['classname'] + "::" + test['name'])
+ if len(test.contents) > 0 and test.contents[0].name == 'skipped':
+ print(f"skipping {test['name']}")
+ continue
+ test_results.append(TestResult(
+ test_id=test['classname'] + "::" + test['name'],
+ file=test['file'],
+ line=test['line'],
+ name=test['name'],
+ classname=test['classname'],
+ result=len(test.contents) == 0
+ ))
+
+ token = os.environ.get("GITHUB_TOKEN")
+ print(f"Token: {token}")
+ github_repository = os.environ.get('GITHUB_REPOSITORY')
+ if not github_repository:
+ raise Exception("Github Repository must be defined!")
+ user, repo = github_repository.split("/")
+ print(f"User: {user}, Repo: {repo}")
+ issue_id = int(os.environ.get('ISSUE_ID', 0))
+ num_runs = int(os.environ.get('NUM_RUNS', 10))
+
+ if issue_id == 0:
+ raise Exception("You need to define ISSUE_ID as environment variable")
+
+ gh = login(token=token)
+
+ quarantined_issue = gh.issue(user, repo, issue_id)
+ print("-----")
+ print(quarantined_issue.body)
+ print("-----")
+ parsed_test_map = parse_body(quarantined_issue.body)
+ new_test_map: Dict[str, TestHistory] = {}
+
+ for test_result in test_results:
+ previous_results = parsed_test_map.get(test_result.test_id)
+ if previous_results:
+ updated_results = update_test_history(
+ previous_results, test_result.result)
+ new_test_map[previous_results.test_id] = updated_results
+ else:
+ new_history = create_test_history(test_result)
+ new_test_map[new_history.test_id] = new_history
+ table = get_table(new_test_map)
+ print()
+ print("Result:")
+ print()
+ print(table)
+ print()
+ with open(join(dirname(realpath(__file__)), "quarantine_issue_header.md"), "r") as f:
+ header = jinja2.Template(f.read(), autoescape=True, undefined=StrictUndefined).\
+ render(DATE_UTC_NOW=datetime.utcnow())
+ quarantined_issue.edit(title=None,
+ body=header + "\n\n" + str(table),
+ state='open' if len(test_results) > 0 else 'closed')
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 49afef0..08a6b69 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -287,6 +287,14 @@ function get_environment_for_builds_on_ci() {
else
export CI_EVENT_TYPE="push"
fi
+ elif [[ "${LOCAL_CI_TESTING:=}" == "true" ]]; then
+ export CI_TARGET_REPO="apache/airflow"
+ export CI_TARGET_BRANCH="${DEFAULT_BRANCH:="master"}"
+ export CI_BUILD_ID="0"
+ export CI_JOB_ID="0"
+ export CI_EVENT_TYPE="pull_request"
+ export CI_SOURCE_REPO="apache/airflow"
+ export CI_SOURCE_BRANCH="${DEFAULT_BRANCH:="master"}"
else
echo
echo "ERROR! Unknown CI environment. Exiting"
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index 884b164..091d87f 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -30,6 +30,7 @@ fi
function run_airflow_testing_in_docker() {
set +u
+ set +e
# shellcheck disable=SC2016
docker-compose --log-level INFO \
-f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
@@ -37,7 +38,18 @@ function run_airflow_testing_in_docker() {
"${INTEGRATIONS[@]}" \
"${DOCKER_COMPOSE_LOCAL[@]}" \
run airflow "${@}"
+ EXIT_CODE=$?
+ if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} == "true" ]]; then
+ if [[ ${EXIT_CODE} == "1" ]]; then
+ echo "Some Quarantined tests failed. but we recorded it in an issue"
+ EXIT_CODE="0"
+ else
+ echo "All Quarantined tests succeeded"
+ fi
+ fi
set -u
+ set -e
+ return "${EXIT_CODE}"
}
get_environment_for_builds_on_ci
@@ -93,6 +105,7 @@ elif [[ ${TEST_TYPE:=} == "Long" ]]; then
export ONLY_RUN_LONG_RUNNING_TESTS="true"
elif [[ ${TEST_TYPE:=} == "Quarantined" ]]; then
export ONLY_RUN_QUARANTINED_TESTS="true"
+ # Do not fail in quarantined tests
fi
for _INT in ${ENABLED_INTEGRATIONS}
@@ -103,4 +116,9 @@ done
RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""}
+
run_airflow_testing_in_docker "${@}"
+
+if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then
+ export ONLY_RUN_QUARANTINED_TESTS="true"
+fi
diff --git a/setup.py b/setup.py
index 60497cf..68f10f2 100644
--- a/setup.py
+++ b/setup.py
@@ -451,6 +451,7 @@ devel = [
'flake8-colors',
'flaky',
'freezegun',
+ 'github3.py',
'gitpython',
'ipdb',
'jira',
@@ -466,7 +467,7 @@ devel = [
'pytest-cov',
'pytest-instafail',
'pytest-rerunfailures',
- 'pytest-timeout',
+ 'pytest-timeouts',
'pytest-xdist',
'pywinrm',
'qds-sdk>=1.9.6',