You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/12/03 00:10:15 UTC
[airflow] 14/34: Adds possibility of forcing upgrade constraint by
setting a label (#12635)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c7ec034944bd63a1d56f6b24a017389afd376701
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu Nov 26 11:02:33 2020 +0100
Adds possibility of forcing upgrade constraint by setting a label (#12635)
You can now set a label on PR that will force upgrading to latest
dependencies in your PR. If committer sets an
"upgrade to latest dependencies" label, it will cause the PR
to upgrade all dependencies to latest versions of dependencies
matching setup.py + setup.cfg configuration.
(cherry picked from commit 8b9d52f0cc197832188f431a2b6e4eb256f9725b)
---
.github/workflows/build-images-workflow-run.yml | 18 ++-----
.github/workflows/ci.yml | 10 ++--
.github/workflows/codeql-analysis.yml | 4 +-
.../workflows/label_when_reviewed_workflow_run.yml | 4 +-
CONTRIBUTING.rst | 5 ++
scripts/ci/selective_ci_checks.sh | 56 ++++++++++++++++------
6 files changed, 60 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
index 9726c5a..c5480c6 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -30,7 +30,6 @@ env:
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
VERBOSE: "true"
- UPGRADE_TO_LATEST_CONSTRAINTS: false
USE_GITHUB_REGISTRY: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
@@ -57,7 +56,6 @@ jobs:
sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }}
cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
buildImages: ${{ steps.build-images.outputs.buildImages }}
- upgradeToLatestConstraints: ${{ steps.upgrade-constraints.outputs.upgradeToLatestConstraints }}
steps:
- name: "Get information about the original trigger of the run"
uses: potiuk/get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3
@@ -153,15 +151,6 @@ jobs:
else
echo "::set-output name=docker-cache::pulled"
fi
- - name: "Set upgrade to latest constraints"
- id: upgrade-constraints
- run: |
- if [[ ${{ steps.cancel.outputs.sourceEvent == 'push' ||
- steps.cancel.outputs.sourceEvent == 'scheduled' }} == 'true' ]]; then
- echo "::set-output name=upgradeToLatestConstraints::${{ github.sha }}"
- else
- echo "::set-output name=upgradeToLatestConstraints::false"
- fi
- name: "Cancel all duplicated 'Build Image' runs"
# We find duplicates of all "Build Image" runs - due to a missing feature
# in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching
@@ -198,6 +187,7 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
pythonVersions: ${{ steps.selective-checks.python-versions }}
+ upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }}
allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
@@ -243,12 +233,12 @@ jobs:
id: selective-checks
env:
EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- INCOMING_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
+ TARGET_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels }}
run: |
if [[ ${EVENT_NAME} == "pull_request" ]]; then
# Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
else
# Run all checks
./scripts/ci/selective_ci_checks.sh
@@ -273,7 +263,7 @@ jobs:
BACKEND: postgres
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }}
- UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.cancel-workflow-runs.outputs.upgradeToLatestConstraints }}
+ UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
steps:
- name: >
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5931135..77cbf65 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,6 @@ env:
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
VERBOSE: "true"
- UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }}
DOCKER_CACHE: "pulled"
USE_GITHUB_REGISTRY: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -69,6 +68,7 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
+ upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }}
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
@@ -131,12 +131,12 @@ jobs:
id: selective-checks
env:
EVENT_NAME: ${{ github.event_name }}
- INCOMING_COMMIT_SHA: ${{ github.sha }}
+ TARGET_COMMIT_SHA: ${{ github.sha }}
PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
run: |
if [[ ${EVENT_NAME} == "pull_request" ]]; then
# Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
else
# Run all checks
./scripts/ci/selective_ci_checks.sh
@@ -150,6 +150,7 @@ jobs:
if: needs.build-info.outputs.image-build == 'true'
env:
BACKEND: sqlite
+ UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -568,7 +569,8 @@ jobs:
needs: [build-info]
env:
BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 2bf92b7..9fa7b94 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -40,11 +40,11 @@ jobs:
id: selective-checks
env:
EVENT_NAME: ${{ github.event_name }}
- INCOMING_COMMIT_SHA: ${{ github.sha }}
+ TARGET_COMMIT_SHA: ${{ github.sha }}
run: |
if [[ ${EVENT_NAME} == "pull_request" ]]; then
# Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
else
# Run all checks
./scripts/ci/selective_ci_checks.sh
diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml
index 6e45038..6ea15b0 100644
--- a/.github/workflows/label_when_reviewed_workflow_run.yml
+++ b/.github/workflows/label_when_reviewed_workflow_run.yml
@@ -75,12 +75,12 @@ jobs:
id: selective-checks
env:
EVENT_NAME: ${{ steps.source-run-info.outputs.sourceEvent }}
- INCOMING_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }}
+ TARGET_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }}
PR_LABELS: ${{ steps.source-run-info.outputs.pullRequestLabels }}
run: |
if [[ ${EVENT_NAME} == "pull_request_review" ]]; then
# Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
else
# Run all checks
./scripts/ci/selective_ci_checks.sh
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 6d34026..61883e7 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -321,6 +321,11 @@ Step 4: Prepare PR
the "full tests needed" label is set for your PR. Additional check is set that prevents from
accidental merging of the request until full matrix of tests succeeds for the PR.
+ * when your change has "upgrade to latest dependencies" label set, constraints will be automatically
+ upgraded to latest constraints matching your setup.py. This is useful in case you want to force
+ upgrade to a latest version of dependencies. You can ask committers to set the label for you
+ when you need it in your PR.
+
More details about the PR workflow be found in `PULL_REQUEST_WORKFLOW.rst <PULL_REQUEST_WORKFLOW.rst>`_.
diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh
index 3c7132d..a6c66eb 100755
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -34,16 +34,39 @@ if [[ ${PR_LABELS=} == *"full tests needed"* ]]; then
echo
echo "Found the right PR labels in '${PR_LABELS=}': 'full tests needed''"
echo
- FULL_TESTS_NEEDED="true"
+ FULL_TESTS_NEEDED_LABEL="true"
else
echo
echo "Did not find the right PR labels in '${PR_LABELS=}': 'full tests needed'"
echo
- FULL_TESTS_NEEDED="false"
+ FULL_TESTS_NEEDED_LABEL="false"
+fi
+
+if [[ ${PR_LABELS=} == *"upgrade to latest dependencies"* ]]; then
+ echo
+ echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies''"
+ echo
+ UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="true"
+else
+ echo
+ echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies'"
+ echo
+ UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="false"
fi
function output_all_basic_variables() {
- if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+ if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS_LABEL}" == "true" ||
+ ${EVENT_NAME} == 'push' || ${EVENT_NAME} == "scheduled" ]]; then
+ # Trigger upgrading to latest constraints where label is set or when
+ # SHA of the merge commit triggers rebuilding layer in the docker image
+ # Each build that upgrades to latest constraints will get truly latest constraints, not those
+ # Cached in the image this way
+ initialization::ga_output upgrade-to-latest-constraints "${INCOMING_COMMIT_SHA}"
+ else
+ initialization::ga_output upgrade-to-latest-constraints "false"
+ fi
+
+ if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
initialization::ga_output python-versions \
"$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
initialization::ga_output all-python-versions \
@@ -60,7 +83,7 @@ function output_all_basic_variables() {
fi
initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
- if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+ if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
initialization::ga_output kubernetes-versions \
"$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")"
else
@@ -73,7 +96,7 @@ function output_all_basic_variables() {
"$(initialization::parameters_to_json "${CURRENT_KUBERNETES_MODES[@]}")"
initialization::ga_output default-kubernetes-mode "${KUBERNETES_MODE}"
- if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+ if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
initialization::ga_output postgres-versions \
"$(initialization::parameters_to_json "${CURRENT_POSTGRES_VERSIONS[@]}")"
else
@@ -82,7 +105,7 @@ function output_all_basic_variables() {
fi
initialization::ga_output default-postgres-version "${POSTGRES_VERSION}"
- if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+ if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
initialization::ga_output mysql-versions \
"$(initialization::parameters_to_json "${CURRENT_MYSQL_VERSIONS[@]}")"
else
@@ -100,7 +123,7 @@ function output_all_basic_variables() {
"$(initialization::parameters_to_json "${CURRENT_HELM_VERSIONS[@]}")"
initialization::ga_output default-helm-version "${HELM_VERSION}"
- if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+ if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
initialization::ga_output postgres-exclude '[{ "python-version": "3.6" }]'
initialization::ga_output mysql-exclude '[{ "python-version": "3.7" }]'
initialization::ga_output sqlite-exclude '[{ "python-version": "3.8" }]'
@@ -114,9 +137,6 @@ function output_all_basic_variables() {
}
function get_changed_files() {
- INCOMING_COMMIT_SHA="${1}"
- readonly INCOMING_COMMIT_SHA
-
echo
echo "Incoming commit SHA: ${INCOMING_COMMIT_SHA}"
echo
@@ -414,14 +434,20 @@ if (($# < 1)); then
echo
echo "No Commit SHA - running all tests (likely direct master merge, or scheduled run)!"
echo
- # override FULL_TESTS_NEEDED in master/scheduled run
- FULL_TESTS_NEEDED="true"
- readonly FULL_TESTS_NEEDED
+ INCOMING_COMMIT_SHA=""
+ readonly INCOMING_COMMIT_SHA
+ # override FULL_TESTS_NEEDED_LABEL in master/scheduled run
+ FULL_TESTS_NEEDED_LABEL="true"
+ readonly FULL_TESTS_NEEDED_LABEL
output_all_basic_variables
set_outputs_run_everything_and_exit
+else
+ INCOMING_COMMIT_SHA="${1}"
+ readonly INCOMING_COMMIT_SHA
fi
-readonly FULL_TESTS_NEEDED
+
+readonly FULL_TESTS_NEEDED_LABEL
output_all_basic_variables
image_build_needed="false"
@@ -429,7 +455,7 @@ docs_build_needed="false"
tests_needed="false"
kubernetes_tests_needed="false"
-get_changed_files "${1}"
+get_changed_files
run_all_tests_if_environment_files_changed
check_if_docs_should_be_generated
check_if_helm_tests_should_be_run