You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/08/11 22:34:50 UTC
[airflow] 09/32: Group CI scripts in subdirectories (#9653)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7ec2b3ace1a1a4982b0c313b89eab6aee1eb9620
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu Jul 16 18:05:35 2020 +0200
Group CI scripts in subdirectories (#9653)
Reviewed the scripts and removed some of the old unused ones.
(cherry picked from commit faec41ec9a05a037b88fd0213b1936cde2b5c454)
---
.github/workflows/ci.yml | 62 +++++-----
.pre-commit-config.yaml | 34 +++---
.rat-excludes | 1 +
BREEZE.rst | 2 +-
STATIC_CODE_CHECKS.rst | 15 ++-
TESTING.rst | 8 +-
breeze | 4 +-
docs/start_doc_server.sh | 4 +-
hooks/build | 4 +-
hooks/push | 5 -
scripts/ci/ci_load_image_to_kind.sh | 33 -----
scripts/ci/ci_perform_kind_cluster_operation.sh | 32 -----
scripts/ci/{ => docs}/ci_docs.sh | 4 +-
scripts/ci/{ => images}/ci_build_dockerhub.sh | 4 +-
.../ci/{ => images}/ci_prepare_ci_image_on_ci.sh | 4 +-
.../ci/{ => images}/ci_prepare_prod_image_on_ci.sh | 4 +-
scripts/ci/{ => images}/ci_push_ci_image.sh | 4 +-
.../ci/{ => images}/ci_push_production_images.sh | 4 +-
.../ci/in_container/_in_container_script_init.sh | 4 +-
scripts/ci/in_container/_in_container_utils.sh | 2 +-
.../in_container/deploy_airflow_to_kubernetes.sh | 23 ----
scripts/ci/in_container/entrypoint_ci.sh | 16 +--
scripts/ci/{ => in_container}/run_cli_tool.sh | 0
scripts/ci/in_container/run_system_tests.sh | 4 +-
.../ci_deploy_app_to_kubernetes.sh | 4 +-
.../ci/{ => kubernetes}/ci_run_kubernetes_tests.sh | 4 +-
scripts/ci/{ => libraries}/_all_libs.sh | 29 +++--
scripts/ci/libraries/_build_images.sh | 2 +-
scripts/ci/libraries/_initialization.sh | 6 +-
scripts/ci/libraries/_kind.sh | 40 +++---
scripts/ci/{ => libraries}/_script_init.sh | 13 +-
scripts/ci/minikdc.properties | 27 -----
.../ci/{ => pre_commit}/pre_commit_bat_tests.sh | 4 +-
.../{ => pre_commit}/pre_commit_breeze_cmd_line.sh | 10 +-
.../pre_commit_check_integrations.sh | 10 +-
.../{ => pre_commit}/pre_commit_check_license.sh | 2 +-
.../ci/pre_commit/pre_commit_check_order_setup.py | 135 +++++++++++++++++++++
scripts/ci/{ => pre_commit}/pre_commit_ci_build.sh | 4 +-
scripts/ci/{ => pre_commit}/pre_commit_flake8.sh | 4 +-
.../pre_commit_generate_requirements.sh | 4 +-
.../ci/pre_commit/pre_commit_insert_extras.py | 4 +-
.../{ => pre_commit}/pre_commit_lint_dockerfile.sh | 4 +-
.../pre_commit_local_yml_mounts.sh | 8 +-
scripts/ci/{ => pre_commit}/pre_commit_mypy.sh | 4 +-
.../ci/{ => pre_commit}/pre_commit_yaml_to_cfg.py | 7 +-
scripts/ci/pre_commit_update_extras.sh | 31 -----
.../{ => requirements}/ci_generate_requirements.sh | 4 +-
scripts/ci/{ => static_checks}/ci_bat_tests.sh | 3 +
scripts/ci/{ => static_checks}/ci_check_license.sh | 4 +-
scripts/ci/{ => static_checks}/ci_flake8.sh | 4 +-
.../ci/{ => static_checks}/ci_lint_dockerfile.sh | 4 +-
scripts/ci/{ => static_checks}/ci_mypy.sh | 4 +-
.../ci/{ => static_checks}/ci_run_static_checks.sh | 4 +-
scripts/ci/{ => testing}/ci_run_airflow_testing.sh | 33 ++---
scripts/ci/{ => tools}/ci_count_changed_files.sh | 4 +-
scripts/ci/{ => tools}/ci_fix_ownership.sh | 10 +-
scripts/ci/{ => tools}/ci_free_space_on_ci.sh | 4 +-
tests/bats/bats_utils.bash | 4 +-
tests/test_order_setup.py | 134 --------------------
59 files changed, 355 insertions(+), 494 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 134bc1f..a849d58 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -60,15 +60,15 @@ jobs:
path: ~/.cache/pre-commit
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Static checks"
if: success()
run: |
python -m pip install pre-commit \
--constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
- ./scripts/ci/ci_run_static_checks.sh
+ ./scripts/ci/static_checks/ci_run_static_checks.sh
docs:
timeout-minutes: 60
@@ -82,9 +82,9 @@ jobs:
with:
python-version: '3.6'
- name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Build docs"
- run: ./scripts/ci/ci_docs.sh
+ run: ./scripts/ci/docs/ci_docs.sh
build-prod-image:
timeout-minutes: 60
@@ -99,7 +99,7 @@ jobs:
steps:
- uses: actions/checkout@master
- name: "Build PROD image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
trigger-tests:
timeout-minutes: 10
@@ -112,7 +112,7 @@ jobs:
- name: "Get count of changed python files"
run: |
set +e
- ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} \
+ ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \
'^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests'
echo "::set-output name=count::$?"
id: trigger-tests
@@ -155,7 +155,7 @@ jobs:
with:
python-version: '3.6'
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- uses: engineerd/setup-kind@v0.4.0
name: Setup Kind Cluster
with:
@@ -163,7 +163,7 @@ jobs:
name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}}
config: "scripts/ci/kubernetes/kind-cluster-conf.yaml"
- name: "Deploy app to cluster"
- run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh
+ run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
- name: Cache virtualenv for kubernetes testing
uses: actions/cache@v2
env:
@@ -173,7 +173,7 @@ jobs:
key: "${{ env.cache-name }}-${{ github.job }}-\
${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}"
- name: "Tests"
- run: ./scripts/ci/ci_run_kubernetes_tests.sh
+ run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
- uses: actions/upload-artifact@v2
name: Upload KinD logs
# Always run this, even if one of th previous steps failed.
@@ -208,11 +208,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.6'
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
- run: ./scripts/ci/ci_run_airflow_testing.sh
+ run: ./scripts/ci/testing/ci_run_airflow_testing.sh
tests-mysql:
timeout-minutes: 80
@@ -240,11 +240,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
- run: ./scripts/ci/ci_run_airflow_testing.sh
+ run: ./scripts/ci/testing/ci_run_airflow_testing.sh
tests-sqlite:
timeout-minutes: 80
@@ -270,11 +270,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
- run: ./scripts/ci/ci_run_airflow_testing.sh
+ run: ./scripts/ci/testing/ci_run_airflow_testing.sh
tests-quarantined:
timeout-minutes: 80
@@ -303,11 +303,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
- run: ./scripts/ci/ci_run_airflow_testing.sh
+ run: ./scripts/ci/testing/ci_run_airflow_testing.sh
helm-tests:
timeout-minutes: 5
@@ -337,11 +337,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
- uses: actions/checkout@master
- uses: actions/setup-python@v1
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Generate requirements"
- run: ./scripts/ci/ci_generate_requirements.sh
+ run: ./scripts/ci/requirements/ci_generate_requirements.sh
push-prod-images-to-github-cache:
timeout-minutes: 80
@@ -364,11 +364,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
steps:
- uses: actions/checkout@master
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build PROD images ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- name: "Push PROD images ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_push_production_images.sh
+ run: ./scripts/ci/images/ci_push_production_images.sh
push-ci-images-to-github-cache:
timeout-minutes: 40
@@ -395,8 +395,8 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
steps:
- uses: actions/checkout@master
- name: "Free space"
- run: ./scripts/ci/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image"
- run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Push CI image ${{ matrix.python-version }}"
- run: ./scripts/ci/ci_push_ci_image.sh
+ run: ./scripts/ci/images/ci_push_ci_image.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 99dbbe9..eb91828 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -198,7 +198,7 @@ repos:
- id: lint-dockerfile
name: Lint dockerfile
language: system
- entry: "./scripts/ci/pre_commit_lint_dockerfile.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh"
files: ^Dockerfile.*$
pass_filenames: true
- id: setup-order
@@ -207,25 +207,25 @@ repos:
files: ^setup.py$
pass_filenames: false
require_serial: true
- entry: tests/test_order_setup.py
+ entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
- id: update-breeze-file
name: Update output of breeze command in BREEZE.rst
- entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh"
language: system
files: ^BREEZE.rst$|^breeze$|^breeze-complete$
pass_filenames: false
require_serial: true
- id: update-local-yml-file
name: Update mounts in the local yml file
- entry: "./scripts/ci/pre_commit_local_yml_mounts.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh"
language: system
files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml"
pass_filenames: false
require_serial: true
- id: update-extras
name: Update extras in documentation
- entry: "./scripts/ci/pre_commit_update_extras.sh"
- language: system
+ entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
+ language: python
files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$
pass_filenames: false
require_serial: true
@@ -265,14 +265,14 @@ repos:
^\sdef\s*\S*\(.*\):\s*\-\>\s*\S*.* # Matches -> return value syntax from Python3
)$
files: \.py$
- exclude: ^dev/
+ exclude: ^dev|^scripts
pass_filenames: true
- id: python2-compile
name: Compile code using python2
language: system
entry: python2.7 -m py_compile
files: \.py$
- exclude: ^dev/
+ exclude: ^dev|^scripts
pass_filenames: true
require_serial: true
- id: pydevd
@@ -283,7 +283,7 @@ repos:
pass_filenames: true
- id: check-integrations
name: Check if integration list is aligned
- entry: ./scripts/ci/pre_commit_check_integrations.sh
+ entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
language: system
pass_filenames: false
files: ^common/_common_values.sh$|^breeze-complete$
@@ -295,13 +295,13 @@ repos:
pass_filenames: true
- id: build
name: Check if image build is needed
- entry: ./scripts/ci/pre_commit_ci_build.sh 3.5 false
+ entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
language: system
always_run: true
pass_filenames: false
- id: check-apache-license
name: Check if licenses are OK for Apache
- entry: "./scripts/ci/pre_commit_check_license.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh"
language: system
files: ^.*LICENSE.*$|^.*LICENCE.*$
pass_filenames: false
@@ -309,28 +309,28 @@ repos:
- id: airflow-config-yaml
name: Checks for consistency between config.yml and default_config.cfg
language: python
+ entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
files: "^airflow/config_templates/config.yml$|^airflow/config_templates/default_airflow.cfg$"
pass_filenames: false
- require_serial: false
- entry: scripts/ci/pre_commit_yaml_to_cfg.py
+ require_serial: true
additional_dependencies: ['pyyaml']
- id: mypy
name: Run mypy
language: system
- entry: "./scripts/ci/pre_commit_mypy.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
files: \.py$
exclude: ^dev
require_serial: true
- id: flake8
name: Run flake8
language: system
- entry: "./scripts/ci/pre_commit_flake8.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
files: \.py$
- exclude: ^dev/
+ exclude: ^dev
pass_filenames: true
- id: bat-tests
name: Run BATS bash tests for changed bash files
language: system
- entry: "./scripts/ci/pre_commit_bat_tests.sh"
+ entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
files: ^breeze$|^breeze-complete$|\.sh$|\.bash$
pass_filenames: false
diff --git a/.rat-excludes b/.rat-excludes
index 6bef964..497d7ed 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -86,3 +86,4 @@ input_notebook.ipynb
# .git might be a file in case of worktree
.git
+tmp
diff --git a/BREEZE.rst b/BREEZE.rst
index 338777b..435b21e 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -920,7 +920,7 @@ by the root user, you can fix the ownership of those files by running this scrip
.. code-block::
- ./scripts/ci/ci_fix_ownership.sh
+ ./scripts/ci/tools/ci_fix_ownership.sh
Mounting Local Sources to Breeze
--------------------------------
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 3cb2c5e..b3b5978 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -265,13 +265,12 @@ Running Static Code Checks via Scripts from the Host
....................................................
You can trigger the static checks from the host environment, without entering the Docker container. To do
-this, run the following scripts (the same is done in the CI builds):
+this, run the following scripts:
-* `<scripts/ci/ci_check_license.sh>`_ - checks the licenses.
-* `<scripts/ci/ci_docs.sh>`_ - checks that documentation can be built without warnings.
-* `<scripts/ci/ci_flake8.sh>`_ - runs Flake8 source code style enforcement tool.
-* `<scripts/ci/ci_lint_dockerfile.sh>`_ - runs lint checker for the dockerfiles.
-* `<scripts/ci/ci_mypy.sh>`_ - runs a check for mypy type annotation consistency.
+* `<scripts/ci/static_checks/ci_check_license.sh>`_ - checks the licenses.
+* `<scripts/ci/static_checks/ci_flake8.sh>`_ - runs Flake8 source code style enforcement tool.
+* `<scripts/ci/static_checks/ci_lint_dockerfile.sh>`_ - runs lint checker for the dockerfiles.
+* `<scripts/ci/static_checks/ci_mypy.sh>`_ - runs a check for mypy type annotation consistency.
The scripts may ask you to rebuild the images, if needed.
@@ -314,8 +313,8 @@ On the host:
.. code-block::
- ./scripts/ci/ci_mypy.sh ./airflow/example_dags/
+ ./scripts/ci/static_checks/ci_mypy.sh ./airflow/example_dags/
.. code-block::
- ./scripts/ci/ci_mypy.sh ./airflow/example_dags/test_utils.py
+ ./scripts/ci/static_checks/ci_mypy.sh ./airflow/example_dags/test_utils.py
diff --git a/TESTING.rst b/TESTING.rst
index 7c761e8..02163fe 100644
--- a/TESTING.rst
+++ b/TESTING.rst
@@ -378,10 +378,10 @@ to run the tests manually one by one.
Running kubernetes tests
- ./scripts/ci/ci_run_kubernetes_tests.sh - runs all kubernetes tests
- ./scripts/ci/ci_run_kubernetes_tests.sh TEST [TEST ...] - runs selected kubernetes tests (from kubernetes_tests folder)
- ./scripts/ci/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in
- ./scripts/ci/ci_run_kubernetes_tests.sh [--help] - Prints this help message
+ ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh - runs all kubernetes tests
+ ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh TEST [TEST ...] - runs selected kubernetes tests (from kubernetes_tests folder)
+ ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in
+ ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [--help] - Prints this help message
You can also run the same tests command with Breeze, using ``kind-cluster test`` command (to run all
diff --git a/breeze b/breeze
index a10ba9c..abb95b7 100755
--- a/breeze
+++ b/breeze
@@ -62,8 +62,8 @@ function setup_default_breeze_variables() {
# load all the common functions here - those are the functions that are shared between Breeze
# and CI scripts (CI scripts do not use Breeze as execution environment)
- # shellcheck source=scripts/ci/_all_libs.sh
- . "${SCRIPTS_CI_DIR}/_all_libs.sh"
+ # shellcheck source=scripts/ci/libraries/_all_libs.sh
+ . "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
# We have different versions of images depending on the python version used. We keep up with the
# Latest patch-level changes in Python (this is done automatically during CI builds) so we have
diff --git a/docs/start_doc_server.sh b/docs/start_doc_server.sh
index 26248ec..51cc7a77 100755
--- a/docs/start_doc_server.sh
+++ b/docs/start_doc_server.sh
@@ -16,8 +16,8 @@
# specific language governing permissions and limitations
# under the License.
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-(cd "${MY_DIR}"/_build/html || exit;
+DOCS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+(cd "${DOCS_DIR}"/_build/html || exit;
# The below command works on both Python 2 and Python 3
python -m http.server 8000 && python -m SimpleHTTPServer 8000
)
diff --git a/hooks/build b/hooks/build
index a98a923..d3d3efb 100755
--- a/hooks/build
+++ b/hooks/build
@@ -20,9 +20,9 @@
# on Travis CI to potentially rebuild (and refresh layers that
# are not cached) Docker images that are used to run CI jobs
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+_HOOK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Dockerhub builds are run inside Docker container
export SKIP_IN_CONTAINER_CHECK="true"
-exec "${MY_DIR}/../scripts/ci/ci_build_dockerhub.sh"
+exec "${_HOOK_DIR}/../scripts/ci/images/ci_build_dockerhub.sh"
diff --git a/hooks/push b/hooks/push
index 91cf096..84805b0 100755
--- a/hooks/push
+++ b/hooks/push
@@ -20,11 +20,6 @@
# and it is difficult to pass list of the built images from the build to push phase
set -euo pipefail
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-echo "My dir: ${MY_DIR}"
-
-
echo
echo "Skip pushing the image. All images were built and pushed in the build hook already!"
echo
diff --git a/scripts/ci/ci_load_image_to_kind.sh b/scripts/ci/ci_load_image_to_kind.sh
deleted file mode 100755
index dda1e38..0000000
--- a/scripts/ci/ci_load_image_to_kind.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
-
-cd "${AIRFLOW_SOURCES}" || exit 1
-
-export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
-export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
-
-prepare_prod_build
-echo
-echo "Loading the ${AIRFLOW_PROD_IMAGE} to cluster ${KIND_CLUSTER_NAME} from docker"
-echo
-"${AIRFLOW_SOURCES}/.build/bin/kind" load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE}"
-echo
-echo "Loaded the ${AIRFLOW_PROD_IMAGE} to cluster ${KIND_CLUSTER_NAME}"
-echo
diff --git a/scripts/ci/ci_perform_kind_cluster_operation.sh b/scripts/ci/ci_perform_kind_cluster_operation.sh
deleted file mode 100755
index 4d3ddd2..0000000
--- a/scripts/ci/ci_perform_kind_cluster_operation.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
-
-# adding trap to exiting trap
-HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
-# shellcheck disable=SC2064
-trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
-
-get_environment_for_builds_on_ci
-make_sure_kubernetes_tools_are_installed
-initialize_kind_variables
-perform_kind_cluster_operation "${@}"
-
-check_cluster_ready_for_airflow
diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
similarity index 92%
rename from scripts/ci/ci_docs.sh
rename to scripts/ci/docs/ci_docs.sh
index 761e9a0..847cb49 100755
--- a/scripts/ci/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function run_docs() {
docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \
diff --git a/scripts/ci/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh
similarity index 95%
rename from scripts/ci/ci_build_dockerhub.sh
rename to scripts/ci/images/ci_build_dockerhub.sh
index 97f9b8c..99612a2 100755
--- a/scripts/ci/ci_build_dockerhub.sh
+++ b/scripts/ci/images/ci_build_dockerhub.sh
@@ -47,8 +47,8 @@ echo "DOCKER_TAG=${DOCKER_TAG}"
echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}"
echo
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
if [[ ${DOCKER_TAG} == *python*-ci ]]; then
echo
diff --git a/scripts/ci/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
similarity index 87%
rename from scripts/ci/ci_prepare_ci_image_on_ci.sh
rename to scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index 8ced220..5fd7913 100755
--- a/scripts/ci/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
build_ci_image_on_ci
diff --git a/scripts/ci/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
similarity index 87%
rename from scripts/ci/ci_prepare_prod_image_on_ci.sh
rename to scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index 066a43b..ab4e7c0 100755
--- a/scripts/ci/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
build_prod_image_on_ci
diff --git a/scripts/ci/ci_push_ci_image.sh b/scripts/ci/images/ci_push_ci_image.sh
similarity index 88%
rename from scripts/ci/ci_push_ci_image.sh
rename to scripts/ci/images/ci_push_ci_image.sh
index 09e2a7a..664d81a 100755
--- a/scripts/ci/ci_push_ci_image.sh
+++ b/scripts/ci/images/ci_push_ci_image.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
prepare_ci_build
diff --git a/scripts/ci/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh
similarity index 88%
rename from scripts/ci/ci_push_production_images.sh
rename to scripts/ci/images/ci_push_production_images.sh
index d6c6e7d..db439d8 100755
--- a/scripts/ci/ci_push_production_images.sh
+++ b/scripts/ci/images/ci_push_production_images.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
prepare_prod_build
diff --git a/scripts/ci/in_container/_in_container_script_init.sh b/scripts/ci/in_container/_in_container_script_init.sh
index 3540016..50a558e 100755
--- a/scripts/ci/in_container/_in_container_script_init.sh
+++ b/scripts/ci/in_container/_in_container_script_init.sh
@@ -19,10 +19,10 @@
set -euo pipefail
# This should only be sourced from in_container directory!
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+IN_CONTAINER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
-. "${MY_DIR}/_in_container_utils.sh"
+. "${IN_CONTAINER_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh
index 0eb3a8a..f2e94d4 100644
--- a/scripts/ci/in_container/_in_container_utils.sh
+++ b/scripts/ci/in_container/_in_container_utils.sh
@@ -155,7 +155,7 @@ function setup_kerberos() {
PASS="airflow"
KRB5_KTNAME=/etc/airflow.keytab
- sudo cp "${MY_DIR}/krb5/krb5.conf" /etc/krb5.conf
+ sudo cp "${AIRFLOW_SOURCES}/scripts/ci/in_container/krb5/krb5.conf" /etc/krb5.conf
echo -e "${PASS}\n${PASS}" | \
sudo kadmin -p "${ADMIN}/admin" -w "${PASS}" -q "addprinc -randkey airflow/${FQDN}" 2>&1 \
diff --git a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh b/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh
deleted file mode 100755
index a124fba..0000000
--- a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# Script to run flake8 on all code. Can be started from any working directory
-# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-
-"${MY_DIR}/kubernetes/docker/rebuild_airflow_image.sh"
-"${MY_DIR}/kubernetes/app/deploy_app.sh"
diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh
index 4d1bf0c..eb1ff51 100755
--- a/scripts/ci/in_container/entrypoint_ci.sh
+++ b/scripts/ci/in_container/entrypoint_ci.sh
@@ -22,7 +22,7 @@ fi
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. /opt/airflow/scripts/ci/in_container/_in_container_script_init.sh
-AIRFLOW_SOURCES=$(cd "${MY_DIR}/../../.." || exit 1; pwd)
+AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../../.." || exit 1; pwd)
PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.6}
BACKEND=${BACKEND:=sqlite}
@@ -47,8 +47,8 @@ INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION:=""}"
if [[ ${CI} == "false" ]]; then
# Create links for useful CLI tools
- # shellcheck source=scripts/ci/run_cli_tool.sh
- source <(bash scripts/ci/run_cli_tool.sh)
+ # shellcheck source=scripts/ci/in_container/run_cli_tool.sh
+ source <(bash scripts/ci/in_container/run_cli_tool.sh)
fi
if [[ ${AIRFLOW_VERSION} == *1.10* || ${INSTALL_AIRFLOW_VERSION} == *1.10* ]]; then
@@ -98,10 +98,10 @@ export PATH=${PATH}:${AIRFLOW_SOURCES}
unset AIRFLOW__CORE__UNIT_TEST_MODE
mkdir -pv "${AIRFLOW_HOME}/logs/"
-cp -f "${MY_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg"
+cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg"
set +e
-"${MY_DIR}/check_environment.sh"
+"${IN_CONTAINER_DIR}/check_environment.sh"
ENVIRONMENT_EXIT_CODE=$?
set -e
if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then
@@ -150,7 +150,7 @@ done
ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
# shellcheck source=scripts/ci/in_container/configure_environment.sh
-. "${MY_DIR}/configure_environment.sh"
+. "${IN_CONTAINER_DIR}/configure_environment.sh"
cd "${AIRFLOW_SOURCES}"
@@ -211,7 +211,7 @@ fi
ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}")
if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then
- "${MY_DIR}/run_system_tests.sh" "${ARGS[@]}"
+ "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}"
else
- "${MY_DIR}/run_ci_tests.sh" "${ARGS[@]}"
+ "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${ARGS[@]}"
fi
diff --git a/scripts/ci/run_cli_tool.sh b/scripts/ci/in_container/run_cli_tool.sh
similarity index 100%
rename from scripts/ci/run_cli_tool.sh
rename to scripts/ci/in_container/run_cli_tool.sh
diff --git a/scripts/ci/in_container/run_system_tests.sh b/scripts/ci/in_container/run_system_tests.sh
index 11dcf06..8cb3c3e 100755
--- a/scripts/ci/in_container/run_system_tests.sh
+++ b/scripts/ci/in_container/run_system_tests.sh
@@ -20,10 +20,10 @@
# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
set -euo pipefail
-MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
+IN_CONTAINER_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
-. "${MY_DIR}/_in_container_utils.sh"
+. "${IN_CONTAINER_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
diff --git a/scripts/ci/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
similarity index 92%
rename from scripts/ci/ci_deploy_app_to_kubernetes.sh
rename to scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
index 307bf00..d4bb4b1 100755
--- a/scripts/ci/ci_deploy_app_to_kubernetes.sh
+++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
set -euo pipefail
diff --git a/scripts/ci/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
similarity index 96%
rename from scripts/ci/ci_run_kubernetes_tests.sh
rename to scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index 4d49e9e..3d8194a 100755
--- a/scripts/ci/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
# adding trap to exiting trap
HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
diff --git a/scripts/ci/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
similarity index 68%
rename from scripts/ci/_all_libs.sh
rename to scripts/ci/libraries/_all_libs.sh
index 9869598..f70bbde 100755
--- a/scripts/ci/_all_libs.sh
+++ b/scripts/ci/libraries/_all_libs.sh
@@ -16,34 +16,33 @@
# specific language governing permissions and limitations
# under the License.
-SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}")
+LIBRARIES_DIR=$(dirname "${BASH_SOURCE[0]}")
-# must be first to initialize arrays TODO: For sure?
# shellcheck source=scripts/ci/libraries/_initialization.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_initialization.sh
+. "${LIBRARIES_DIR}"/_initialization.sh
# shellcheck source=scripts/ci/libraries/_sanity_checks.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_sanity_checks.sh
+. "${LIBRARIES_DIR}"/_sanity_checks.sh
# shellcheck source=scripts/ci/libraries/_build_images.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_build_images.sh
+. "${LIBRARIES_DIR}"/_build_images.sh
# shellcheck source=scripts/ci/libraries/_kind.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_kind.sh
+. "${LIBRARIES_DIR}"/_kind.sh
# shellcheck source=scripts/ci/libraries/_local_mounts.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_local_mounts.sh
+. "${LIBRARIES_DIR}"/_local_mounts.sh
# shellcheck source=scripts/ci/libraries/_md5sum.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_md5sum.sh
+. "${LIBRARIES_DIR}"/_md5sum.sh
# shellcheck source=scripts/ci/libraries/_parameters.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_parameters.sh
+. "${LIBRARIES_DIR}"/_parameters.sh
# shellcheck source=scripts/ci/libraries/_permissions.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_permissions.sh
+. "${LIBRARIES_DIR}"/_permissions.sh
# shellcheck source=scripts/ci/libraries/_push_pull_remove_images.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_push_pull_remove_images.sh
+. "${LIBRARIES_DIR}"/_push_pull_remove_images.sh
# shellcheck source=scripts/ci/libraries/_runs.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_runs.sh
+. "${LIBRARIES_DIR}"/_runs.sh
# shellcheck source=scripts/ci/libraries/_spinner.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_spinner.sh
+. "${LIBRARIES_DIR}"/_spinner.sh
# shellcheck source=scripts/ci/libraries/_start_end.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_start_end.sh
+. "${LIBRARIES_DIR}"/_start_end.sh
# shellcheck source=scripts/ci/libraries/_verbosity.sh
-. "${SCRIPTS_CI_DIR}"/libraries/_verbosity.sh
+. "${LIBRARIES_DIR}"/_verbosity.sh
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 352975b..01eac17 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -421,7 +421,7 @@ function rebuild_ci_image_if_needed() {
if [[ ${SYSTEM} != "Darwin" ]]; then
ROOT_FILES_COUNT=$(find "airflow" "tests" -user root | wc -l | xargs)
if [[ ${ROOT_FILES_COUNT} != "0" ]]; then
- ./scripts/ci/ci_fix_ownership.sh
+ ./scripts/ci/tools/ci_fix_ownership.sh
fi
fi
print_info
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 5f2a742..d0b14a7 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -33,10 +33,6 @@ function initialize_common_environment {
# shellcheck disable=SC2034
FILES_TO_CLEANUP_ON_EXIT=()
- # Sets to where airflow sources are located
- AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)}
- export AIRFLOW_SOURCES
-
# Sets to the build cache directory - status of build and convenience scripts are stored there
BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
export BUILD_CACHE_DIR
@@ -172,7 +168,7 @@ function initialize_common_environment {
fi
# Read airflow version from the version.py
- AIRFLOW_VERSION=$(grep version "airflow/version.py" | awk '{print $3}' | sed "s/['+]//g")
+ AIRFLOW_VERSION=$(grep version "${AIRFLOW_SOURCES}/airflow/version.py" | awk '{print $3}' | sed "s/['+]//g")
export AIRFLOW_VERSION
# default version of python used to tag the "master" and "latest" images in DockerHub
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index f24b3aa..2cf43c3 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -125,6 +125,16 @@ function delete_cluster() {
}
function perform_kind_cluster_operation() {
+ ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell ]"
+
+ set +u
+ if [[ -z "${1}" ]]; then
+ echo >&2
+ echo >&2 "Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS}"
+ echo >&2
+ exit 1
+ fi
+ set -u
OPERATION="${1}"
ALL_CLUSTERS=$(kind get clusters || true)
@@ -181,17 +191,16 @@ function perform_kind_cluster_operation() {
echo
echo "Testing with KinD"
echo
- "${AIRFLOW_SOURCES}/scripts/ci/ci_run_kubernetes_tests.sh"
+ "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh"
elif [[ ${OPERATION} == "shell" ]]; then
echo
echo "Entering an interactive shell for kubernetes testing"
echo
- "${AIRFLOW_SOURCES}/scripts/ci/ci_run_kubernetes_tests.sh" "-i"
+ "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh" "-i"
else
- echo
- echo "Wrong cluster operation: ${OPERATION}. Should be one of:"
- echo "${FORMATTED_KIND_OPERATIONS}"
- echo
+ echo >&2
+ echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of: ${ALLOWED_KIND_OPERATIONS}"
+ echo >&2
exit 1
fi
else
@@ -208,15 +217,14 @@ function perform_kind_cluster_operation() {
create_cluster
elif [[ ${OPERATION} == "stop" || ${OEPRATON} == "deploy" || \
${OPERATION} == "test" || ${OPERATION} == "shell" ]]; then
- echo
- echo "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation"
- echo
+ echo >&2
+ echo >&2 "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation"
+ echo >&2
exit 1
else
- echo
- echo "Wrong cluster operation: ${OPERATION}. Should be one of:"
- echo "${FORMATTED_KIND_OPERATIONS}"
- echo
+ echo >&2
+ echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of ${ALLOWED_KIND_OPERATIONS}"
+ echo >&2
exit 1
fi
fi
@@ -262,9 +270,9 @@ function forward_port_to_kind_webserver() {
set +e
while ! curl http://localhost:8080/health -s | grep -q healthy; do
if [[ ${num_tries} == 6 ]]; then
- echo
- echo "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting."
- echo
+ echo >&2
+ echo >&2 "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting."
+ echo >&2
exit 1
fi
echo
diff --git a/scripts/ci/_script_init.sh b/scripts/ci/libraries/_script_init.sh
similarity index 73%
rename from scripts/ci/_script_init.sh
rename to scripts/ci/libraries/_script_init.sh
index 2acbf10..804037e 100755
--- a/scripts/ci/_script_init.sh
+++ b/scripts/ci/libraries/_script_init.sh
@@ -18,17 +18,18 @@
set -euo pipefail
-# This should only be sourced from CI directory!
+_CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
-SCRIPTS_CI_DIR="$( dirname "${BASH_SOURCE[0]}" )"
+SCRIPTS_CI_DIR="$(cd "${_CURRENT_DIR}"/.. && pwd)"
export SCRIPTS_CI_DIR
-# shellcheck source=scripts/ci/_all_libs.sh
-. "${SCRIPTS_CI_DIR}"/_all_libs.sh
+# Sets to where airflow sources are located
+AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${SCRIPTS_CI_DIR}/../../" && pwd)}
+export AIRFLOW_SOURCES
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export MY_DIR
+# shellcheck source=scripts/ci/libraries/_all_libs.sh
+. "${SCRIPTS_CI_DIR}"/libraries/_all_libs.sh
initialize_common_environment
diff --git a/scripts/ci/minikdc.properties b/scripts/ci/minikdc.properties
deleted file mode 100644
index c70ff84..0000000
--- a/scripts/ci/minikdc.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-org.name=TEST
-org.domain=LOCAL
-kdc.bind.address=localhost
-kdc.port=8888
-instance=DefaultKrbServer
-max.ticket.lifetime=86400000
-max.renewable.lifetime=604800000
-transport=TCP
-debug=true
diff --git a/scripts/ci/pre_commit_bat_tests.sh b/scripts/ci/pre_commit/pre_commit_bat_tests.sh
similarity index 87%
rename from scripts/ci/pre_commit_bat_tests.sh
rename to scripts/ci/pre_commit/pre_commit_bat_tests.sh
index 4ae72c3..b820c7e 100755
--- a/scripts/ci/pre_commit_bat_tests.sh
+++ b/scripts/ci/pre_commit/pre_commit_bat_tests.sh
@@ -24,5 +24,5 @@ else
PARAMS=("${@}")
fi
-# shellcheck source=scripts/ci/ci_bat_tests.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/ci_bat_tests.sh" "${PARAMS[@]}"
+# shellcheck source=scripts/ci/static_checks/ci_bat_tests.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_bat_tests.sh" "${PARAMS[@]}"
diff --git a/scripts/ci/pre_commit_breeze_cmd_line.sh b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
similarity index 89%
rename from scripts/ci/pre_commit_breeze_cmd_line.sh
rename to scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
index eeac2e6..4d62cc1 100755
--- a/scripts/ci/pre_commit_breeze_cmd_line.sh
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
@@ -18,12 +18,14 @@
set -euo pipefail
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
+cd "${AIRFLOW_SOURCES}" || exit 1
+
+
TMP_FILE=$(mktemp)
TMP_OUTPUT=$(mktemp)
-cd "${MY_DIR}/../../" || exit;
-
echo "
.. code-block:: text
" >"${TMP_FILE}"
@@ -46,7 +48,7 @@ if (( MAX_LEN > MAX_SCREEN_WIDTH + 2 )); then
exit 1
fi
-BREEZE_RST_FILE="${MY_DIR}/../../BREEZE.rst"
+BREEZE_RST_FILE="${AIRFLOW_SOURCES}/BREEZE.rst"
LEAD='^ \.\. START BREEZE HELP MARKER$'
TAIL='^ \.\. END BREEZE HELP MARKER$'
diff --git a/scripts/ci/pre_commit_check_integrations.sh b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
similarity index 81%
rename from scripts/ci/pre_commit_check_integrations.sh
rename to scripts/ci/pre_commit/pre_commit_check_integrations.sh
index 69ace38..6871941 100755
--- a/scripts/ci/pre_commit_check_integrations.sh
+++ b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
@@ -17,12 +17,12 @@
# under the License.
set -euo pipefail
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
+cd "${AIRFLOW_SOURCES}" || exit 1
-cd "${MY_DIR}/../../" || exit;
-
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
. breeze-complete
diff --git a/scripts/ci/pre_commit_check_license.sh b/scripts/ci/pre_commit/pre_commit_check_license.sh
similarity index 92%
rename from scripts/ci/pre_commit_check_license.sh
rename to scripts/ci/pre_commit/pre_commit_check_license.sh
index 4ab6964..109df31 100755
--- a/scripts/ci/pre_commit_check_license.sh
+++ b/scripts/ci/pre_commit/pre_commit_check_license.sh
@@ -21,5 +21,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
# Hide lines between ****/**** (detailed list of files)
-"$( dirname "${BASH_SOURCE[0]}" )/ci_check_license.sh" 2>&1 | \
+"$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_check_license.sh" 2>&1 | \
(sed "/Files with Apache License headers will be marked AL.*$/,/^\**$/d" || true)
diff --git a/scripts/ci/pre_commit/pre_commit_check_order_setup.py b/scripts/ci/pre_commit/pre_commit_check_order_setup.py
new file mode 100755
index 0000000..c833e22
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_check_order_setup.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Test for an order of dependencies in setup.py
+"""
+
+import os
+import re
+import sys
+from os.path import abspath, dirname
+from typing import List
+
+errors = []
+
+
+def _check_list_sorted(the_list: List[str], message: str) -> None:
+ sorted_list = sorted(the_list)
+ if the_list == sorted_list:
+ print(f"{message} is ok")
+ return
+ i = 0
+ while sorted_list[i] == the_list[i]:
+ i += 1
+ print(f"{message} NOK")
+ errors.append(f"ERROR in {message}. First wrongly sorted element"
+ f" {the_list[i]}. Should be {sorted_list[i]}")
+
+
+def setup() -> str:
+ setup_py_file_path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir, 'setup.py'))
+ with open(setup_py_file_path) as setup_file:
+ setup_context = setup_file.read()
+ return setup_context
+
+
+def check_main_dependent_group(setup_context: str) -> None:
+ """
+ Test for an order of dependencies groups between mark
+ '# Start dependencies group' and '# End dependencies group' in setup.py
+ """
+ pattern_main_dependent_group = re.compile(
+ '# Start dependencies group\n(.*)# End dependencies group', re.DOTALL)
+ main_dependent_group = pattern_main_dependent_group.findall(setup_context)[0]
+
+ pattern_sub_dependent = re.compile(' = \\[.*?\\]\n', re.DOTALL)
+ main_dependent = pattern_sub_dependent.sub(',', main_dependent_group)
+
+ src = main_dependent.strip(',').split(',')
+ _check_list_sorted(src, "Order of dependencies")
+
+
+def check_sub_dependent_group(setup_context: str) -> None:
+ r"""
+ Test for an order of each dependencies groups declare like
+ `^dependent_group_name = [.*?]\n` in setup.py
+ """
+ pattern_dependent_group_name = re.compile('^(\\w+) = \\[', re.MULTILINE)
+ dependent_group_names = pattern_dependent_group_name.findall(setup_context)
+
+ pattern_dependent_version = re.compile('[~|><=;].*')
+
+ for group_name in dependent_group_names:
+ pattern_sub_dependent = re.compile(
+ '{group_name} = \\[(.*?)\\]'.format(group_name=group_name), re.DOTALL)
+ sub_dependent = pattern_sub_dependent.findall(setup_context)[0]
+ pattern_dependent = re.compile('\'(.*?)\'')
+ dependent = pattern_dependent.findall(sub_dependent)
+
+ src = [pattern_dependent_version.sub('', p) for p in dependent]
+ _check_list_sorted(src, f"Order of sub-dependencies group: {group_name}")
+
+
+def check_alias_dependent_group(setup_context: str) -> None:
+ """
+ Test for an order of each dependencies groups declare like
+ `alias_dependent_group = dependent_group_1 + ... + dependent_group_n` in setup.py
+ """
+ pattern = re.compile('^\\w+ = (\\w+ \\+.*)', re.MULTILINE)
+ dependents = pattern.findall(setup_context)
+
+ for dependent in dependents:
+ src = dependent.split(' + ')
+ _check_list_sorted(src, f"Order of alias dependencies group: {dependent}")
+
+
+def check_install_and_setup_requires(setup_context: str) -> None:
+ """
+ Test for an order of dependencies in function do_setup section
+ install_requires and setup_requires in setup.py
+ """
+ pattern_install_and_setup_requires = re.compile(
+ '(setup_requires) ?= ?\\[(.*?)\\]', re.DOTALL)
+ install_and_setup_requires = pattern_install_and_setup_requires.findall(setup_context)
+
+ for dependent_requires in install_and_setup_requires:
+ pattern_dependent = re.compile('\'(.*?)\'')
+ dependent = pattern_dependent.findall(dependent_requires[1])
+ pattern_dependent_version = re.compile('[~|><=;].*')
+
+ src = [pattern_dependent_version.sub('', p) for p in dependent]
+ _check_list_sorted(src, f"Order of dependencies in do_setup section: {dependent_requires[0]}")
+
+
+if __name__ == '__main__':
+ setup_context_main = setup()
+ check_main_dependent_group(setup_context_main)
+ check_alias_dependent_group(setup_context_main)
+ check_sub_dependent_group(setup_context_main)
+ check_install_and_setup_requires(setup_context_main)
+
+ print()
+ print()
+ for error in errors:
+ print(error)
+
+ print()
+
+ if errors:
+ sys.exit(1)
diff --git a/scripts/ci/pre_commit_ci_build.sh b/scripts/ci/pre_commit/pre_commit_ci_build.sh
similarity index 88%
rename from scripts/ci/pre_commit_ci_build.sh
rename to scripts/ci/pre_commit/pre_commit_ci_build.sh
index 6f25c39..861785b 100755
--- a/scripts/ci/pre_commit_ci_build.sh
+++ b/scripts/ci/pre_commit/pre_commit_ci_build.sh
@@ -18,8 +18,8 @@
export PYTHON_MAJOR_MINOR_VERSION="${1}"
export REMEMBER_LAST_ANSWER="${2}"
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
forget_last_answer
diff --git a/scripts/ci/pre_commit_flake8.sh b/scripts/ci/pre_commit/pre_commit_flake8.sh
similarity index 87%
rename from scripts/ci/pre_commit_flake8.sh
rename to scripts/ci/pre_commit/pre_commit_flake8.sh
index dd2a3da..95f1b98 100755
--- a/scripts/ci/pre_commit_flake8.sh
+++ b/scripts/ci/pre_commit/pre_commit_flake8.sh
@@ -18,5 +18,5 @@
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
-# shellcheck source=scripts/ci/ci_flake8.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/ci_flake8.sh" "${@}"
+# shellcheck source=scripts/ci/static_checks/ci_flake8.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_flake8.sh" "${@}"
diff --git a/scripts/ci/pre_commit_generate_requirements.sh b/scripts/ci/pre_commit/pre_commit_generate_requirements.sh
similarity index 85%
rename from scripts/ci/pre_commit_generate_requirements.sh
rename to scripts/ci/pre_commit/pre_commit_generate_requirements.sh
index f9e952e..d0c2deb 100755
--- a/scripts/ci/pre_commit_generate_requirements.sh
+++ b/scripts/ci/pre_commit/pre_commit_generate_requirements.sh
@@ -20,5 +20,5 @@ export REMEMBER_LAST_ANSWER="true"
export PYTHON_MAJOR_MINOR_VERSION="${1}"
-# shellcheck source=scripts/ci/ci_generate_requirements.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/ci_generate_requirements.sh"
+# shellcheck source=scripts/ci/requirements/ci_generate_requirements.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../generate_requirements/ci_generate_requirements.sh"
diff --git a/tests/insert_extras.py b/scripts/ci/pre_commit/pre_commit_insert_extras.py
old mode 100644
new mode 100755
similarity index 93%
rename from tests/insert_extras.py
rename to scripts/ci/pre_commit/pre_commit_insert_extras.py
index b38b543..9c441df
--- a/tests/insert_extras.py
+++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -20,8 +21,9 @@ from os.path import dirname
from textwrap import wrap
from typing import List
-AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir)
+AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir)
+sys.path.insert(0, AIRFLOW_SOURCES_DIR)
# flake8: noqa: F401
# pylint: disable=wrong-import-position
from setup import EXTRAS_REQUIREMENTS # isort:skip
diff --git a/scripts/ci/pre_commit_lint_dockerfile.sh b/scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh
similarity index 84%
rename from scripts/ci/pre_commit_lint_dockerfile.sh
rename to scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh
index bce941b..857c27e 100755
--- a/scripts/ci/pre_commit_lint_dockerfile.sh
+++ b/scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh
@@ -17,5 +17,5 @@
# under the License.
export REMEMBER_LAST_ANSWER="true"
-# shellcheck source=scripts/ci/ci_lint_dockerfile.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/ci_lint_dockerfile.sh" "${@}"
+# shellcheck source=scripts/ci/static_checks/ci_lint_dockerfile.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_lint_dockerfile.sh" "${@}"
diff --git a/scripts/ci/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
similarity index 86%
rename from scripts/ci/pre_commit_local_yml_mounts.sh
rename to scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
index b638a08..43ece7e 100755
--- a/scripts/ci/pre_commit_local_yml_mounts.sh
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -18,17 +18,15 @@
set -euo pipefail
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
TMP_OUTPUT=$(mktemp)
# Remove temp file if it's hanging around
trap 'rm -rf -- "${TMP_OUTPUT}" 2>/dev/null' EXIT
-LOCAL_YML_FILE="${MY_DIR}/docker-compose/local.yml"
+LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml"
LEAD=' # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh'
TAIL=' # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh'
diff --git a/scripts/ci/pre_commit_mypy.sh b/scripts/ci/pre_commit/pre_commit_mypy.sh
similarity index 87%
rename from scripts/ci/pre_commit_mypy.sh
rename to scripts/ci/pre_commit/pre_commit_mypy.sh
index 6f0025f..f5ab254 100755
--- a/scripts/ci/pre_commit_mypy.sh
+++ b/scripts/ci/pre_commit/pre_commit_mypy.sh
@@ -18,5 +18,5 @@
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
-# shellcheck source=scripts/ci/ci_mypy.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/ci_mypy.sh" "${@}"
+# shellcheck source=scripts/ci/static_checks/ci_mypy.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_mypy.sh" "${@}"
diff --git a/scripts/ci/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
similarity index 97%
rename from scripts/ci/pre_commit_yaml_to_cfg.py
rename to scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
index b826864..a578689 100755
--- a/scripts/ci/pre_commit_yaml_to_cfg.py
+++ b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -127,7 +126,9 @@ def write_config(yaml_config_file_path, default_cfg_file_path):
if __name__ == '__main__':
airflow_config_dir = os.path.join(
- os.path.dirname(__file__), "../../airflow/config_templates")
+ os.path.dirname(__file__),
+ os.pardir, os.pardir, os.pardir,
+ "airflow", "config_templates")
airflow_default_config_path = os.path.join(airflow_config_dir, "default_airflow.cfg")
airflow_config_yaml_file_path = os.path.join(airflow_config_dir, "config.yml")
diff --git a/scripts/ci/pre_commit_update_extras.sh b/scripts/ci/pre_commit_update_extras.sh
deleted file mode 100755
index be43a91..0000000
--- a/scripts/ci/pre_commit_update_extras.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euo pipefail
-
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-cd "${MY_DIR}/../../" || exit;
-
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
-
-PYTHONPATH="$(pwd)"
-export PYTHONPATH
-
-python3 tests/insert_extras.py
diff --git a/scripts/ci/ci_generate_requirements.sh b/scripts/ci/requirements/ci_generate_requirements.sh
similarity index 88%
rename from scripts/ci/ci_generate_requirements.sh
rename to scripts/ci/requirements/ci_generate_requirements.sh
index f55799f..5cc4a0e 100755
--- a/scripts/ci/ci_generate_requirements.sh
+++ b/scripts/ci/requirements/ci_generate_requirements.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
get_environment_for_builds_on_ci
diff --git a/scripts/ci/ci_bat_tests.sh b/scripts/ci/static_checks/ci_bat_tests.sh
similarity index 90%
rename from scripts/ci/ci_bat_tests.sh
rename to scripts/ci/static_checks/ci_bat_tests.sh
index 25011ce..e50c559 100755
--- a/scripts/ci/ci_bat_tests.sh
+++ b/scripts/ci/static_checks/ci_bat_tests.sh
@@ -16,6 +16,9 @@
# specific language governing permissions and limitations
# under the License.
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
function run_bats_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/static_checks/ci_check_license.sh
similarity index 94%
rename from scripts/ci/ci_check_license.sh
rename to scripts/ci/static_checks/ci_check_license.sh
index da5aebd..3d887c4 100755
--- a/scripts/ci/ci_check_license.sh
+++ b/scripts/ci/static_checks/ci_check_license.sh
@@ -18,8 +18,8 @@
export MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true"
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function run_check_license() {
echo
diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/static_checks/ci_flake8.sh
similarity index 94%
rename from scripts/ci/ci_flake8.sh
rename to scripts/ci/static_checks/ci_flake8.sh
index 33504c0..4ebd060 100755
--- a/scripts/ci/ci_flake8.sh
+++ b/scripts/ci/static_checks/ci_flake8.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function run_flake8() {
FILES=("$@")
diff --git a/scripts/ci/ci_lint_dockerfile.sh b/scripts/ci/static_checks/ci_lint_dockerfile.sh
similarity index 92%
rename from scripts/ci/ci_lint_dockerfile.sh
rename to scripts/ci/static_checks/ci_lint_dockerfile.sh
index 29a5f68..2e48043 100755
--- a/scripts/ci/ci_lint_dockerfile.sh
+++ b/scripts/ci/static_checks/ci_lint_dockerfile.sh
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function run_docker_lint() {
FILES=("$@")
diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/static_checks/ci_mypy.sh
similarity index 93%
rename from scripts/ci/ci_mypy.sh
rename to scripts/ci/static_checks/ci_mypy.sh
index 8cc3028..b6fc56e 100755
--- a/scripts/ci/ci_mypy.sh
+++ b/scripts/ci/static_checks/ci_mypy.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=3.6
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function run_mypy() {
FILES=("$@")
diff --git a/scripts/ci/ci_run_static_checks.sh b/scripts/ci/static_checks/ci_run_static_checks.sh
similarity index 91%
rename from scripts/ci/ci_run_static_checks.sh
rename to scripts/ci/static_checks/ci_run_static_checks.sh
index bfdae1a..6b7f124 100755
--- a/scripts/ci/ci_run_static_checks.sh
+++ b/scripts/ci/static_checks/ci_run_static_checks.sh
@@ -17,8 +17,8 @@
# under the License.
export PYTHON_MAJOR_MINOR_VERSION=3.6
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
echo
diff --git a/scripts/ci/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
similarity index 82%
rename from scripts/ci/ci_run_airflow_testing.sh
rename to scripts/ci/testing/ci_run_airflow_testing.sh
index 09dfb7d..7949996 100755
--- a/scripts/ci/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -17,28 +17,29 @@
# under the License.
export VERBOSE=${VERBOSE:="false"}
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
+ echo
+ echo "Skipping running tests !!!!!"
+ echo
+ exit
+fi
+
+
function run_airflow_testing_in_docker() {
set +u
# shellcheck disable=SC2016
docker-compose --log-level INFO \
- -f "${MY_DIR}/docker-compose/base.yml" \
- -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \
+ -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
+ -f "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml" \
"${INTEGRATIONS[@]}" \
"${DOCKER_COMPOSE_LOCAL[@]}" \
run airflow "${@}"
set -u
}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
-
-if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
- echo
- echo "Skipping running tests !!!!!"
- echo
- exit
-fi
-
get_environment_for_builds_on_ci
prepare_ci_build
@@ -66,17 +67,17 @@ export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"}
export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""}
if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then
- DOCKER_COMPOSE_LOCAL=("-f" "${MY_DIR}/docker-compose/local.yml")
+ DOCKER_COMPOSE_LOCAL=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml")
else
DOCKER_COMPOSE_LOCAL=()
fi
if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
- DOCKER_COMPOSE_LOCAL+=("-f" "${MY_DIR}/docker-compose/forward-credentials.yml")
+ DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml")
fi
if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
- DOCKER_COMPOSE_LOCAL+=("-f" "${MY_DIR}/docker-compose/remove-sources.yml")
+ DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml")
fi
echo
@@ -99,7 +100,7 @@ fi
for _INT in ${ENABLED_INTEGRATIONS}
do
INTEGRATIONS+=("-f")
- INTEGRATIONS+=("${MY_DIR}/docker-compose/integration-${_INT}.yml")
+ INTEGRATIONS+=("${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml")
done
RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""}
diff --git a/scripts/ci/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh
similarity index 92%
rename from scripts/ci/ci_count_changed_files.sh
rename to scripts/ci/tools/ci_count_changed_files.sh
index d1ccd4b..e3d2c9e 100755
--- a/scripts/ci/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_count_changed_files.sh
@@ -22,8 +22,8 @@
# $1: Revision to compare
# $2: Pattern to match
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
get_environment_for_builds_on_ci
diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh
similarity index 82%
rename from scripts/ci/ci_fix_ownership.sh
rename to scripts/ci/tools/ci_fix_ownership.sh
index 7e85152..8cde42d 100755
--- a/scripts/ci/ci_fix_ownership.sh
+++ b/scripts/ci/tools/ci_fix_ownership.sh
@@ -21,8 +21,8 @@
#
export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
export AIRFLOW_CI_IMAGE=\
${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci
@@ -39,7 +39,7 @@ export HOST_OS
export BACKEND="sqlite"
docker-compose \
- -f "${MY_DIR}/docker-compose/base.yml" \
- -f "${MY_DIR}/docker-compose/local.yml" \
- -f "${MY_DIR}/docker-compose/forward-credentials.yml" \
+ -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
+ -f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \
+ -f "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml" \
run airflow /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh
diff --git a/scripts/ci/ci_free_space_on_ci.sh b/scripts/ci/tools/ci_free_space_on_ci.sh
similarity index 87%
rename from scripts/ci/ci_free_space_on_ci.sh
rename to scripts/ci/tools/ci_free_space_on_ci.sh
index 5d8f851..a50add7 100755
--- a/scripts/ci/ci_free_space_on_ci.sh
+++ b/scripts/ci/tools/ci_free_space_on_ci.sh
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
sudo swapoff -a
sudo rm -f /swapfile
diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash
index 7a29da9..c06b182 100644
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/bats_utils.bash
@@ -18,5 +18,5 @@
AIRFLOW_SOURCES=$(pwd)
export AIRFLOW_SOURCES
export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-# shellcheck source=scripts/ci/_all_libs.sh
-source "${SCRIPTS_CI_DIR}/_all_libs.sh"
+# shellcheck source=scripts/ci/libraries/_all_libs.sh
+source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
diff --git a/tests/test_order_setup.py b/tests/test_order_setup.py
deleted file mode 100755
index 5c40ea3..0000000
--- a/tests/test_order_setup.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Test for an order of dependencies in setup.py
-"""
-
-import os
-import re
-import unittest
-
-
-class TestOrderSetup(unittest.TestCase):
-
- def setUp(self):
- current_dir = os.path.dirname(os.path.abspath(__file__))
- parent_dir = os.path.dirname(current_dir)
- self.setup_file = open('{parent_dir}/setup.py'.format(parent_dir=parent_dir))
- self.setup_context = self.setup_file.read()
-
- def tearDown(self):
- self.setup_file.close()
-
- def test_main_dependent_group(self):
- """
- Test for an order of dependencies groups between mark
- '# Start dependencies group' and '# End dependencies group' in setup.py
- """
- pattern_main_dependent_group = re.compile(
- '# Start dependencies group\n(.*)# End dependencies group', re.DOTALL)
- main_dependent_group = pattern_main_dependent_group.findall(self.setup_context)[0]
-
- pattern_sub_dependent = re.compile(' = \\[.*?\\]\n', re.DOTALL)
- main_dependent = pattern_sub_dependent.sub(',', main_dependent_group)
-
- src = main_dependent.strip(',').split(',')
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_sub_dependent_group(self):
- """
- Test for an order of each dependencies groups declare like
- `^dependent_group_name = [.*?]\n` in setup.py
- """
- pattern_dependent_group_name = re.compile('^(\\w+) = \\[', re.MULTILINE)
- dependent_group_names = pattern_dependent_group_name.findall(self.setup_context)
-
- pattern_dependent_version = re.compile('[~|>|<|=|;].*')
- for group_name in dependent_group_names:
- pattern_sub_dependent = re.compile(
- '{group_name} = \\[(.*?)\\]'.format(group_name=group_name), re.DOTALL)
- sub_dependent = pattern_sub_dependent.findall(self.setup_context)[0]
- pattern_dependent = re.compile('\'(.*?)\'')
- dependent = pattern_dependent.findall(sub_dependent)
-
- src = [pattern_dependent_version.sub('', p) for p in dependent]
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_alias_dependent_group(self):
- """
- Test for an order of each dependencies groups declare like
- `alias_dependent_group = dependent_group_1 + ... + dependent_group_n` in setup.py
- """
- pattern = re.compile('^\\w+ = (\\w+ \\+.*)', re.MULTILINE)
- dependents = pattern.findall(self.setup_context)
- for dependent in dependents:
- src = dependent.split(' + ')
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_devel_all(self):
- """
- Test for an order of dependencies groups
- devel_all = (dependent_group_1 + ... + dependent_group_n) in setup.py
- """
- pattern = re.compile('devel_all = \\((.*?)\\)', re.DOTALL)
- dependent = pattern.findall(self.setup_context)[0]
- pattern_new_line = re.compile('\\n *')
-
- src = pattern_new_line.sub(' ', dependent).split(' + ')
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_install_and_setup_requires(self):
- """
- Test for an order of dependencies in function do_setup section
- install_requires and setup_requires in setup.py
- """
- pattern_install_and_setup_requires = re.compile(
- '(INSTALL_REQUIREMENTS|setup_requires) ?= ?\\[(.*?)\\]', re.DOTALL)
- install_and_setup_requires = pattern_install_and_setup_requires.findall(self.setup_context)
-
- for dependent_requires in install_and_setup_requires:
- pattern_dependent = re.compile('\'(.*?)\'')
- dependent = pattern_dependent.findall(dependent_requires[1])
- pattern_dependent_version = re.compile('[~|>|<|=|;].*')
-
- src = [pattern_dependent_version.sub('', p) for p in dependent]
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
- def test_extras_require(self):
- """
- Test for an order of dependencies in function do_setup section
- extras_require in setup.py
- """
- pattern_extras_requires = re.compile('EXTRAS_REQUIREMENTS = \\{(.*?)\\}', re.DOTALL)
- extras_requires = pattern_extras_requires.findall(self.setup_context)[0]
-
- pattern_dependent = re.compile('\'(.*?)\'')
- src = pattern_dependent.findall(extras_requires)
- alphabetical = sorted(src)
- self.assertListEqual(alphabetical, src)
-
-
-if __name__ == '__main__':
- unittest.main(verbosity=2)