You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/14 00:46:53 UTC
[airflow] 10/14: Adds --install-wheels flag to breeze command line
(#11317)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c986c1be7ac8818b71ea9e3ddb8d938ee19a0918
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu Oct 8 10:06:53 2020 +0200
Adds --install-wheels flag to breeze command line (#11317)
If this flag is specified it will look for wheel packages placed in dist
folder and it will install the wheels from there after installing
Airflow. This is useful for testing backport packages as well as in the
future for testing provider packages for 2.0.
(cherry picked from commit de07d135ae1bda3f71dd83951bcfafc2b3ad9f89)
---
BREEZE.rst | 37 ++++++++++++++++++---------
breeze | 44 ++++++++++++++++++++++-----------
breeze-complete | 1 +
scripts/ci/docker-compose/base.yml | 3 ++-
scripts/ci/libraries/_initialization.sh | 10 ++++++++
scripts/in_container/entrypoint_ci.sh | 3 +++
6 files changed, 70 insertions(+), 28 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index cd87a73..3002f05 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1184,6 +1184,14 @@ This is the current syntax for `./breeze <./breeze>`_:
If specified, installs Airflow directly from reference in GitHub. This happens at
image building time in production image and at container entering time for CI image.
+ --no-rbac-ui
+ Disables RBAC UI when Airflow 1.10.* is installed.
+
+ --install-wheels
+ If specified it will look for wheel packages placed in dist folder and it will install the
+ wheels from there after installing Airflow. This is useful for testing backport
+ packages as well as in the future for testing provider packages for 2.0.
+
-I, --production-image
Use production image for entering the environment and builds (not for tests).
@@ -1212,6 +1220,9 @@ This is the current syntax for `./breeze <./breeze>`_:
--image-tag TAG
Additional tag in the image.
+ --disable-pip-cache
+ Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
+
--additional-extras ADDITIONAL_EXTRAS
Additional extras to pass to build images The default is no additional extras.
@@ -1262,9 +1273,6 @@ This is the current syntax for `./breeze <./breeze>`_:
constraint file placed in 'docker-context-files' folder, in which case it has to be
in the form of '/docker-context-files/<NAME_OF_THE_FILE>'
- --disable-pip-cache
- Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
-
--install-local-pip-wheels
This flag is only used in production image building. If it is used then instead of
installing Airflow from PyPI, the packages are installed from the .whl packages placed
@@ -1710,6 +1718,9 @@ This is the current syntax for `./breeze <./breeze>`_:
--image-tag TAG
Additional tag in the image.
+ --disable-pip-cache
+ Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
+
--additional-extras ADDITIONAL_EXTRAS
Additional extras to pass to build images The default is no additional extras.
@@ -1760,9 +1771,6 @@ This is the current syntax for `./breeze <./breeze>`_:
constraint file placed in 'docker-context-files' folder, in which case it has to be
in the form of '/docker-context-files/<NAME_OF_THE_FILE>'
- --disable-pip-cache
- Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
-
--install-local-pip-wheels
This flag is only used in production image building. If it is used then instead of
installing Airflow from PyPI, the packages are installed from the .whl packages placed
@@ -1950,9 +1958,6 @@ This is the current syntax for `./breeze <./breeze>`_:
cassandra kerberos mongo openldap presto rabbitmq redis all
- --rbac-ui
- Enable RBAC UI.
-
--init-script INIT_SCRIPT_FILE
Initialization script name - Sourced from files/airflow-breeze-config. Default value
init.sh. It will be executed after the environment is configured and started.
@@ -2036,6 +2041,14 @@ This is the current syntax for `./breeze <./breeze>`_:
If specified, installs Airflow directly from reference in GitHub. This happens at
image building time in production image and at container entering time for CI image.
+ --no-rbac-ui
+ Disables RBAC UI when Airflow 1.10.* is installed.
+
+ --install-wheels
+ If specified it will look for wheel packages placed in dist folder and it will install the
+ wheels from there after installing Airflow. This is useful for testing backport
+ packages as well as in the future for testing provider packages for 2.0.
+
****************************************************************************************************
Credentials
@@ -2071,6 +2084,9 @@ This is the current syntax for `./breeze <./breeze>`_:
--image-tag TAG
Additional tag in the image.
+ --disable-pip-cache
+ Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
+
--additional-extras ADDITIONAL_EXTRAS
Additional extras to pass to build images The default is no additional extras.
@@ -2121,9 +2137,6 @@ This is the current syntax for `./breeze <./breeze>`_:
constraint file placed in 'docker-context-files' folder, in which case it has to be
in the form of '/docker-context-files/<NAME_OF_THE_FILE>'
- --disable-pip-cache
- Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
-
--install-local-pip-wheels
This flag is only used in production image building. If it is used then instead of
installing Airflow from PyPI, the packages are installed from the .whl packages placed
diff --git a/breeze b/breeze
index 522378f..e7a2ca8 100755
--- a/breeze
+++ b/breeze
@@ -63,7 +63,6 @@ export EXTRA_STATIC_CHECK_OPTIONS
# FORWARD_CREDENTIALS
# DB_RESET
# START_AIRFLOW
-# RBAC_UI
# INSTALL_AIRFLOW_VERSION
# INSTALL_AIRFLOW_REFERENCE
# FORCE_BUILD_IMAGES
@@ -124,8 +123,8 @@ function breeze::setup_default_breeze_constants() {
# If set to true, the sample dags will be created
export LOAD_EXAMPLES="false"
- # If set to true, RBAC mode is enabled
- export RBAC_UI="false"
+ # If set to true, RBAC UI will not be used for 1.10 version
+ export DISABLE_RBAC="false"
# If it set is set to specified version, then the source version of Airflow
# is removed and the specified version of Airflow is installed from PyPi
@@ -135,6 +134,10 @@ function breeze::setup_default_breeze_constants() {
# of Airflow is removed and the specified version of Airflow is installed from GitHub
export INSTALL_AIRFLOW_REFERENCE=""
+ # if set to true, the ci image will look for wheel packages in dist folder and will install them
+ # during entering the container
+ export INSTALL_WHEELS="false"
+
# Determines whether to force build without checking if it is needed
# Can be overridden by '--force-build-images' flag.
export FORCE_BUILD_IMAGES="false"
@@ -1078,12 +1081,6 @@ function breeze::parse_arguments() {
echo
shift 2
;;
- --rbac-ui)
- echo "Enable RBAC UI"
- echo
- export RBAC_UI="true"
- shift
- ;;
--load-example-dags)
export LOAD_EXAMPLES="true"
echo "Include Airflow sample dags"
@@ -1096,6 +1093,18 @@ function breeze::parse_arguments() {
echo
shift
;;
+ --no-rbac-ui)
+ export DISABLE_RBAC="true"
+ echo "When installing Airflow 1.10, RBAC UI will be disabled."
+ echo
+ shift
+ ;;
+ --install-wheels)
+ export INSTALL_WHEELS="true"
+ echo "Install wheels found in dist folder"
+ echo
+ shift
+ ;;
--)
shift
break
@@ -1978,9 +1987,6 @@ function breeze::flag_breeze_actions() {
${FORMATTED_INTEGRATIONS}
---rbac-ui
- Enable RBAC UI.
-
--init-script INIT_SCRIPT_FILE
Initialization script name - Sourced from files/airflow-breeze-config. Default value
init.sh. It will be executed after the environment is configured and started.
@@ -2078,6 +2084,14 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS}
-t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE
If specified, installs Airflow directly from reference in GitHub. This happens at
image building time in production image and at container entering time for CI image.
+
+--no-rbac-ui
+ Disables RBAC UI when Airflow 1.10.* is installed.
+
+--install-wheels
+ If specified it will look for wheel packages placed in dist folder and it will install the
+ wheels from there after installing Airflow. This is useful for testing backport
+ packages as well as in the future for testing provider packages for 2.0.
"
}
@@ -2186,6 +2200,9 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
--image-tag TAG
Additional tag in the image.
+--disable-pip-cache
+ Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
+
--additional-extras ADDITIONAL_EXTRAS
Additional extras to pass to build images The default is no additional extras.
@@ -2236,9 +2253,6 @@ Build options:
constraint file placed in 'docker-context-files' folder, in which case it has to be
in the form of '/docker-context-files/<NAME_OF_THE_FILE>'
---disable-pip-cache
- Disables GitHub PIP cache during the build. Useful if github is not reachable during build.
-
--install-local-pip-wheels
This flag is only used in production image building. If it is used then instead of
installing Airflow from PyPI, the packages are installed from the .whl packages placed
diff --git a/breeze-complete b/breeze-complete
index 8b94cef..2c57a79 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -140,6 +140,7 @@ additional-extras: additional-python-deps:
dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env:
runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env:
load-default-connections load-example-dags
+install-wheels no-rbac-ui
"
_breeze_commands="
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 706d90c..f89aefd 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -52,7 +52,8 @@ services:
- START_AIRFLOW
- LOAD_EXAMPLES
- LOAD_DEFAULT_CONNECTIONS
- - RBAC_UI
+ - INSTALL_WHEELS
+ - DISABLE_RBAC
- ENABLED_SYSTEMS
- RUN_SYSTEM_TESTS
- PYTHON_MAJOR_MINOR_VERSION
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 4831f34..576e80c 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -114,6 +114,14 @@ function initialization::initialize_base_variables() {
export LOAD_EXAMPLES=${LOAD_EXAMPLES:="false"}
export LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS:="false"}
+
+ # If set to true, RBAC UI will not be used for 1.10 version
+ export DISABLE_RBAC=${DISABLE_RBAC:="false"}
+
+ # if set to true, the ci image will look for wheel packages in dist folder and will install them
+ # during entering the container
+ export INSTALL_WHEELS=${INSTALL_WHEELS:="false"}
+
# If set the specified file will be used to initialized Airflow after the environment is created,
# otherwise it will use files/airflow-breeze-config/init.sh
export INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE:=""}
@@ -551,6 +559,8 @@ Initialization variables:
INIT_SCRIPT_FILE: ${INIT_SCRIPT_FILE}
LOAD_DEFAULT_CONNECTIONS: ${LOAD_DEFAULT_CONNECTIONS}
LOAD_EXAMPLES: ${LOAD_EXAMPLES}
+ INSTALL_WHEELS: ${INSTALL_WHEELS}
+ DISABLE_RBAC: ${DISABLE_RBAC}
EOF
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index e74fe40..f59e58e 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -88,6 +88,9 @@ else
install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}"
fi
+if [[ ${INSTALL_WHEELS} == "true" ]]; then
+ pip install /dist/*.whl || true
+fi
export RUN_AIRFLOW_1_10=${RUN_AIRFLOW_1_10:="false"}