You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/01/21 20:19:50 UTC
[airflow] 25/44: Allows to install Airflow in Breeze from PIP with
configurable extras (#13055)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f7ab31f12217c926fa56e915b864d763371d30a8
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Mon Dec 14 15:10:06 2020 +0100
Allows to install Airflow in Breeze from PIP with configurable extras (#13055)
The extras configured by --extras Breeze switch are now
passed to pip install command in case airfow is installed via
--install-airflow-version or --install-airflow-reference switch.
(cherry picked from commit 26c685425996a2e8127d349a7b98ec92c74ee3f2)
---
.github/workflows/ci.yml | 2 +-
breeze | 10 ++---
.../ci/{libraries => docker-compose}/_docker.env | 52 ++++++++++++++++------
scripts/ci/docker-compose/base.yml | 43 ++----------------
scripts/ci/docker-compose/local-prod.yml | 9 +---
scripts/ci/libraries/_initialization.sh | 5 +--
scripts/ci/static_checks/check_license.sh | 2 +-
scripts/ci/tools/ci_clear_tmp.sh | 2 +-
scripts/ci/tools/ci_fix_ownership.sh | 2 +-
scripts/in_container/_in_container_utils.sh | 7 ---
scripts/in_container/entrypoint_ci.sh | 12 ++---
11 files changed, 58 insertions(+), 88 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bb06e0d..33fb5d4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -207,6 +207,7 @@ jobs:
runs-on: ubuntu-20.04
needs: [build-info, ci-images]
env:
+ SKIP: "pylint"
MOUNT_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.basic-checks-only == 'false'
@@ -673,7 +674,6 @@ jobs:
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
- ENABLE_KIND_CLUSTER: "true"
PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}"
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
diff --git a/breeze b/breeze
index 2565ad5..f6a45a5 100755
--- a/breeze
+++ b/breeze
@@ -58,7 +58,6 @@ export EXTRA_STATIC_CHECK_OPTIONS
# SCREEN_WIDTH
# MOUNT_LOCAL_SOURCES
# FORCE_PULL_IMAGES
-# ENABLE_KIND_CLUSTER
# FORWARD_CREDENTIALS
# DB_RESET
# START_AIRFLOW
@@ -101,9 +100,6 @@ function breeze::setup_default_breeze_constants() {
# This can be overridden by '--force-pull-images' flag
export FORCE_PULL_IMAGES="false"
- # Do not enable Kind Kubernetes cluster by default
- export ENABLE_KIND_CLUSTER="false"
-
# Forward common host credentials to docker (gcloud, aws etc.).
export FORWARD_CREDENTIALS="false"
@@ -824,7 +820,9 @@ function breeze::parse_arguments() {
INSTALL_AIRFLOW_VERSION="${2}"
# Reference is mutually exclusive with version
INSTALL_AIRFLOW_REFERENCE=""
- echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}"
+ # Skip mounting local sources when airflow is installed from remote
+ INSTALL_PROVIDERS_FROM_SOURCES="false"
+ echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION} and skip mounting sources"
echo
shift 2
;;
@@ -832,6 +830,8 @@ function breeze::parse_arguments() {
INSTALL_AIRFLOW_REFERENCE="${2}"
# Reference is mutually exclusive with version
INSTALL_AIRFLOW_VERSION=""
+ # Skip mounting local sources when airflow is installed from remote
+ INSTALL_PROVIDERS_FROM_SOURCES="false"
echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}"
echo
shift 2
diff --git a/scripts/ci/libraries/_docker.env b/scripts/ci/docker-compose/_docker.env
similarity index 76%
rename from scripts/ci/libraries/_docker.env
rename to scripts/ci/docker-compose/_docker.env
index 6bc53dc..1ab933c 100644
--- a/scripts/ci/libraries/_docker.env
+++ b/scripts/ci/docker-compose/_docker.env
@@ -14,25 +14,49 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-INSTALL_AIRFLOW_VERSION
-PYTHONDONTWRITEBYTECODE
-VERBOSE
-VERBOSE_COMMANDS
+AIRFLOW_CI_IMAGE
+AIRFLOW_EXTRAS
+BACKEND
+BACKPORT_PACKAGES
+BREEZE
+CI
+CI_BUILD_ID
+CI_JOB_ID
+CI_EVENT_TYPE
+CI_TARGET_REPO
+CI_TARGET_BRANCH
+COMMIT_SHA
+DB_RESET
+DEFAULT_CONSTRAINTS_BRANCH
+DISABLE_RBAC
+ENABLED_INTEGRATIONS
+ENABLED_SYSTEMS
+GITHUB_REGISTRY_PULL_IMAGE_TAG
HOST_USER_ID
HOST_GROUP_ID
HOST_OS
HOST_HOME
HOST_AIRFLOW_SOURCES
-PYTHON_MAJOR_MINOR_VERSION
-BACKEND
-VERSION_SUFFIX_FOR_PYPI
-VERSION_SUFFIX_FOR_SVN
-PRINT_INFO_FROM_SCRIPTS
-CI
+INIT_SCRIPT_FILE
+INSTALL_AIRFLOW_VERSION
+INSTALL_PROVIDERS_FROM_SOURCES
+INSTALL_PACKAGES_FROM_DIST
+ISSUE_ID
LOAD_DEFAULT_CONNECTIONS
LOAD_EXAMPLES
-DEFAULT_CONSTRAINTS_BRANCH
-GITHUB_REGISTRY_PULL_IMAGE_TAG
-POSTGRES_VERSION
MYSQL_VERSION
-DISABLE_RBAC
+NUM_RUNS
+PACKAGE_FORMAT
+POSTGRES_VERSION
+PRINT_INFO_FROM_SCRIPTS
+PYTHONDONTWRITEBYTECODE
+PYTHON_MAJOR_MINOR_VERSION
+RUN_TESTS
+RUN_INTEGRATION_TESTS
+RUN_SYSTEM_TESTS
+START_AIRFLOW
+TEST_TYPE
+VERBOSE
+VERBOSE_COMMANDS
+VERSION_SUFFIX_FOR_PYPI
+VERSION_SUFFIX_FOR_SVN
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 96bf0f6..f4f9851 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -24,47 +24,10 @@ services:
- ADDITIONAL_PATH=~/.local/bin
- CELERY_BROKER_URLS=amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0
- KUBECONFIG=/files/.kube/config
- - BACKEND
- - CI
- - CI_BUILD_ID
- - CI_JOB_ID
- - CI_EVENT_TYPE
- - CI_TARGET_REPO
- - CI_TARGET_BRANCH
- - COMMIT_SHA
- - RUN_TESTS
- - VERBOSE
- - VEROSE_COMMANDS
- - AIRFLOW_CI_IMAGE
- - ENABLE_KIND_CLUSTER
- - ENABLED_INTEGRATIONS
- - RUN_INTEGRATION_TESTS
- - TEST_TYPE
- - GITHUB_TOKEN
- - GITHUB_REPOSITORY
- - ISSUE_ID
- - NUM_RUNS
- - BREEZE
- - INSTALL_AIRFLOW_VERSION
- - DB_RESET
- - START_AIRFLOW
- - LOAD_EXAMPLES
- - LOAD_DEFAULT_CONNECTIONS
- - INSTALL_WHEELS
- - DISABLE_RBAC
- - ENABLED_SYSTEMS
- - RUN_SYSTEM_TESTS
- - PYTHON_MAJOR_MINOR_VERSION
- - HOST_USER_ID
- - HOST_GROUP_ID
- - HOST_HOME=${HOME}
- HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
- - HOST_OS
- - PYTHONDONTWRITEBYTECODE
- - INIT_SCRIPT_FILE
- - GITHUB_REGISTRY_PULL_IMAGE_TAG
- - POSTGRES_VERSION
- - MYSQL_VERSION
+ - HOST_HOME=${HOME}
+ env_file:
+ - _docker.env
volumes:
# Pass docker to inside of the container so that Kind and Moto tests can use it.
- /var/run/docker.sock:/var/run/docker.sock
diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml
index 5a4bc8a..79476d0 100644
--- a/scripts/ci/docker-compose/local-prod.yml
+++ b/scripts/ci/docker-compose/local-prod.yml
@@ -29,16 +29,9 @@ services:
- ../../../.github:/opt/airflow/.github:cached
- ../../../.inputrc:/root/.inputrc:cached
- ../../../.kube:/root/.kube:cached
- - ../../../dist:/dist:cached
- ../../../scripts/in_container/prod/entrypoint_prod.sh:/entrypoint:cached
- ../../../setup.cfg:/opt/airflow/setup.cfg:cached
- ../../../setup.py:/opt/airflow/setup.py:cached
- ../../../tests:/opt/airflow/tests:cached
- ../../../tmp:/tmp:cached
- environment:
- - HOST_USER_ID
- - HOST_GROUP_ID
- - HOST_HOME=${HOME}
- - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
- - HOST_OS
- - PYTHONDONTWRITEBYTECODE
+ - ../../../metastore_browser:/opt/airflow/metastore_browser:cached
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index ff7abe9..50cdbf0 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -258,7 +258,7 @@ function initialization::initialize_mount_variables() {
EXTRA_DOCKER_FLAGS+=(
"--rm"
- "--env-file" "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env"
+ "--env-file" "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env"
)
export EXTRA_DOCKER_FLAGS
}
@@ -404,8 +404,6 @@ function initialization::initialize_version_suffixes_for_package_building() {
# Determine versions of kubernetes cluster and tools used
function initialization::initialize_kubernetes_variables() {
- # By default we assume the kubernetes cluster is not being started
- export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
# Currently supported versions of Kubernetes
CURRENT_KUBERNETES_VERSIONS+=("v1.18.6" "v1.17.5" "v1.16.9")
export CURRENT_KUBERNETES_VERSIONS
@@ -679,7 +677,6 @@ function initialization::make_constants_read_only() {
readonly HOST_HOME
readonly HOST_OS
- readonly ENABLE_KIND_CLUSTER
readonly KUBERNETES_MODE
readonly KUBERNETES_VERSION
readonly KIND_VERSION
diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh
index a185a5a..03858e3 100755
--- a/scripts/ci/static_checks/check_license.sh
+++ b/scripts/ci/static_checks/check_license.sh
@@ -33,7 +33,7 @@ function run_check_license() {
# We mount ALL airflow files for the licence check. We want to check them all!
if ! docker run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \
--user "$(id -ur):$(id -gr)" \
- --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" \
+ --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
apache/airflow:apache-rat-2020.07.10-0.13 \
--exclude-file /opt/airflow/.rat-excludes \
--d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then
diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh
index 1806c96..d367967 100755
--- a/scripts/ci/tools/ci_clear_tmp.sh
+++ b/scripts/ci/tools/ci_clear_tmp.sh
@@ -29,6 +29,6 @@ read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker
docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
--rm \
- --env-file "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" \
+ --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
"${AIRFLOW_CI_IMAGE}" \
-c /opt/airflow/scripts/in_container/run_clear_tmp.sh
diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh
index 2e3a909..2d57d65 100755
--- a/scripts/ci/tools/ci_fix_ownership.sh
+++ b/scripts/ci/tools/ci_fix_ownership.sh
@@ -35,6 +35,6 @@ read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker
docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
--rm \
- --env-file "${AIRFLOW_SOURCES}/scripts/ci/libraries/_docker.env" \
+ --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
"${AIRFLOW_CI_IMAGE}" \
-c /opt/airflow/scripts/in_container/run_fix_ownership.sh
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index e2410fd..1dbcdd4 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -276,13 +276,6 @@ function uninstall_airflow_and_providers() {
uninstall_airflow
}
-function install_all_airflow_dependencies() {
- echo
- echo "Installing dependencies from 'all' extras"
- echo
- pip install ".[all]" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
-}
-
function install_released_airflow_version() {
local version="${1}"
local extras="${2}"
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index 55962aa..ba9c2ee 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -108,23 +108,23 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "none" ]]; then
uninstall_airflow_and_providers
elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel" ]]; then
echo
- echo "Install airflow from wheel package with [all] extras but uninstalling providers."
+ echo "Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers."
echo
uninstall_airflow_and_providers
- install_airflow_from_wheel "[all]"
+ install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]"
uninstall_providers
elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist" ]]; then
echo
- echo "Install airflow from sdist package with [all] extras but uninstalling providers."
+ echo "Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers."
echo
uninstall_airflow_and_providers
- install_airflow_from_sdist "[all]"
+ install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]"
uninstall_providers
else
echo
- echo "Install airflow from PyPI including [all] extras"
+ echo "Install airflow from PyPI including [${AIRFLOW_EXTRAS}] extras"
echo
- install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[all]"
+ install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[${AIRFLOW_EXTRAS}]"
fi
if [[ ${INSTALL_PACKAGES_FROM_DIST=} == "true" ]]; then
echo