You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/09/18 08:42:13 UTC
[airflow] branch main updated: Added more automation on latest
version (#18335)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bfe7726 Added more automation on latest version (#18335)
bfe7726 is described below
commit bfe772634861d8a8967aa8030ae4a2cc33f977bb
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Sep 18 10:41:57 2021 +0200
Added more automation on latest version (#18335)
Some of the versions stored in docs have to be stored as
verbatim, rather than as |version| placeholder - they have to refer
to latest released airflow version.
This PR automates that:
When a new version of airflow is set in setup.py, the Dockerfiles
and relevant other documentation files are using that version.
However for tests, the version in Dockerfile is replaced with
lastest version that has been released - just to make
sure that the Dockerfiles are still valid and they can build
with the latest released version.
---
.pre-commit-config.yaml | 6 +++
BREEZE.rst | 2 +-
STATIC_CODE_CHECKS.rst | 2 +-
breeze-complete | 1 +
.../extending/add-apt-packages/Dockerfile | 2 +-
.../add-build-essential-extend/Dockerfile | 2 +-
.../extending/add-providers/Dockerfile | 2 +-
.../extending/add-pypi-packages/Dockerfile | 2 +-
.../extending/embedding-dags/Dockerfile | 2 +-
.../extending/writable-directory/Dockerfile | 2 +-
.../restricted/restricted_environments.sh | 4 +-
docs/docker-stack/entrypoint.rst | 50 +++++++++++++++---
scripts/ci/images/ci_run_prod_image_test.sh | 8 ++-
.../ci/pre_commit/pre_commit_update_versions.py | 61 ++++++++++++++++++++++
14 files changed, 127 insertions(+), 19 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 8cfbfc0..3f9d6db 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -323,6 +323,12 @@ repos:
language: python
files: ^setup\.py$|^INSTALL$|^CONTRIBUTING\.rst$
pass_filenames: false
+ - id: update-version
+ name: Update version to the latest version in the documentation
+ entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
+ language: python
+ files: ^docs
+ pass_filenames: false
- id: pydevd
language: pygrep
name: Check for pydevd debug statements accidentally left
diff --git a/BREEZE.rst b/BREEZE.rst
index e0ce35e..7408982 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2187,7 +2187,7 @@ This is the current syntax for `./breeze <./breeze>`_:
pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages
shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace
ui-lint update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
- verify-db-migrations-documented version-sync www-lint yamllint yesqa
+ update-versions verify-db-migrations-documented version-sync www-lint yamllint yesqa
You can pass extra arguments including options to the pre-commit framework as
<EXTRA_ARGS> passed after --. For example:
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 8d533b8..aae29d7 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -264,7 +264,7 @@ require Breeze Docker images to be installed locally.
------------------------------------ ---------------------------------------------------------------- ------------
``update-setup-cfg-file`` Update setup.cfg file with all licenses
------------------------------------ ---------------------------------------------------------------- ------------
-``update-extras`` Updates extras in the documentation
+``update-versions`` Updates latest versions in the documentation
------------------------------------ ---------------------------------------------------------------- ------------
``verify-db-migrations-documented`` Verify DB Migrations have been documented
------------------------------------ ---------------------------------------------------------------- ------------
diff --git a/breeze-complete b/breeze-complete
index aac8210..b7e00b7 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -144,6 +144,7 @@ update-breeze-file
update-extras
update-local-yml-file
update-setup-cfg-file
+update-versions
verify-db-migrations-documented
version-sync
www-lint
diff --git a/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile
index f11e87a..4540e8f 100644
--- a/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile
+++ b/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile
@@ -15,7 +15,7 @@
# This is an example Dockerfile. It is not intended for PRODUCTION use
# [START Dockerfile]
-FROM apache/airflow:2.1.2
+FROM apache/airflow:2.2.0.dev0
USER root
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
diff --git a/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile b/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile
index 47ac51f..114bf5d 100644
--- a/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile
+++ b/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile
@@ -15,7 +15,7 @@
# This is an example Dockerfile. It is not intended for PRODUCTION use
# [START Dockerfile]
-FROM apache/airflow:2.1.2
+FROM apache/airflow:2.2.0.dev0
USER root
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
diff --git a/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile b/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile
index b65262f..d4cbbce 100644
--- a/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile
+++ b/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile
@@ -15,6 +15,6 @@
# This is an example Dockerfile. It is not intended for PRODUCTION use
# [START Dockerfile]
-FROM apache/airflow:2.1.2
+FROM apache/airflow:2.2.0.dev0
RUN pip install --no-cache-dir apache-airflow-providers-docker==2.1.0
# [END Dockerfile]
diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile
index 310f84c..09df51e 100644
--- a/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile
+++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile
@@ -15,6 +15,6 @@
# This is an example Dockerfile. It is not intended for PRODUCTION use
# [START Dockerfile]
-FROM apache/airflow:2.1.2
+FROM apache/airflow:2.2.0.dev0
RUN pip install --no-cache-dir lxml
# [END Dockerfile]
diff --git a/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile b/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile
index 48701aa..7a5d89e 100644
--- a/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile
+++ b/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile
@@ -15,7 +15,7 @@
# This is an example Dockerfile. It is not intended for PRODUCTION use
# [START Dockerfile]
-FROM apache/airflow:2.1.2
+FROM apache/airflow:2.2.0.dev0
COPY --chown=airflow:root test_dag.py /opt/airflow/dags
diff --git a/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile b/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile
index 8fbb98d..ebd8be6 100644
--- a/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile
+++ b/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile
@@ -15,7 +15,7 @@
# This is an example Dockerfile. It is not intended for PRODUCTION use
# [START Dockerfile]
-FROM apache/airflow:2.1.2
+FROM apache/airflow:2.2.0.dev0
RUN umask 0002; \
mkdir -p ~/writeable-directory
# [END Dockerfile]
diff --git a/docs/docker-stack/docker-examples/restricted/restricted_environments.sh b/docs/docker-stack/docker-examples/restricted/restricted_environments.sh
index f3adb45..4c96fb7 100755
--- a/docs/docker-stack/docker-examples/restricted/restricted_environments.sh
+++ b/docs/docker-stack/docker-examples/restricted/restricted_environments.sh
@@ -25,7 +25,7 @@ cd "${AIRFLOW_SOURCES}"
rm docker-context-files/*.whl docker-context-files/*.tar.gz docker-context-files/*.txt || true
curl -Lo "docker-context-files/constraints-3.7.txt" \
- https://raw.githubusercontent.com/apache/airflow/constraints-2.1.2/constraints-3.7.txt
+ https://raw.githubusercontent.com/apache/airflow/constraints-2.2.0.dev0/constraints-3.7.txt
# For Airflow pre 2.1 you need to use PIP 20.2.4 to install/download Airflow packages.
pip install pip==20.2.4
@@ -39,7 +39,7 @@ pip download --dest docker-context-files \
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
- --build-arg AIRFLOW_VERSION="2.1.2" \
+ --build-arg AIRFLOW_VERSION="2.2.0.dev0" \
--build-arg INSTALL_MYSQL_CLIENT="false" \
--build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" \
--build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="true" \
diff --git a/docs/docker-stack/entrypoint.rst b/docs/docker-stack/entrypoint.rst
index 7db5b5d..18104e6 100644
--- a/docs/docker-stack/entrypoint.rst
+++ b/docs/docker-stack/entrypoint.rst
@@ -132,7 +132,7 @@ if you specify extra arguments. For example:
.. code-block:: bash
- docker run -it apache/airflow:2.1.2-python3.6 bash -c "ls -la"
+ docker run -it apache/airflow:2.2.0.dev0-python3.6 bash -c "ls -la"
total 16
drwxr-xr-x 4 airflow root 4096 Jun 5 18:12 .
drwxr-xr-x 1 root root 4096 Jun 5 18:12 ..
@@ -144,7 +144,7 @@ you pass extra parameters. For example:
.. code-block:: bash
- > docker run -it apache/airflow:2.1.2-python3.6 python -c "print('test')"
+ > docker run -it apache/airflow:2.2.0.dev0-python3.6 python -c "print('test')"
test
If first argument equals to "airflow" - the rest of the arguments is treated as an airflow command
@@ -152,14 +152,48 @@ to execute. Example:
.. code-block:: bash
- docker run -it apache/airflow:2.1.2-python3.6 airflow webserver
+ docker run -it apache/airflow:2.2.0.dev0-python3.6 airflow webserver
If there are any other arguments - they are simply passed to the "airflow" command
.. code-block:: bash
- > docker run -it apache/airflow:2.1.2-python3.6 version
- 2.1.2
+ > docker run -it apache/airflow:2.2.0.dev0-python3.6 help
+ usage: airflow [-h] GROUP_OR_COMMAND ...
+
+ positional arguments:
+ GROUP_OR_COMMAND
+
+ Groups:
+ celery Celery components
+ config View configuration
+ connections Manage connections
+ dags Manage DAGs
+ db Database operations
+ jobs Manage jobs
+ kubernetes Tools to help run the KubernetesExecutor
+ pools Manage pools
+ providers Display providers
+ roles Manage roles
+ tasks Manage tasks
+ users Manage users
+ variables Manage variables
+
+ Commands:
+ cheat-sheet Display cheat sheet
+ info Show information about current Airflow and environment
+ kerberos Start a kerberos ticket renewer
+ plugins Dump information about loaded plugins
+ rotate-fernet-key
+ Rotate encrypted connection credentials and variables
+ scheduler Start a scheduler instance
+ sync-perm Update permissions for existing roles and optionally DAGs
+ version Show the version
+ webserver Start a Airflow webserver instance
+
+ optional arguments:
+ -h, --help show this help message and exit
+
Signal propagation
------------------
@@ -263,7 +297,7 @@ database and creating an ``admin/admin`` Admin user with the following command:
--env "_AIRFLOW_DB_UPGRADE=true" \
--env "_AIRFLOW_WWW_USER_CREATE=true" \
--env "_AIRFLOW_WWW_USER_PASSWORD=admin" \
- apache/airflow:main-python3.8 webserver
+ apache/airflow:2.2.0.dev0-python3.8 webserver
.. code-block:: bash
@@ -272,7 +306,7 @@ database and creating an ``admin/admin`` Admin user with the following command:
--env "_AIRFLOW_DB_UPGRADE=true" \
--env "_AIRFLOW_WWW_USER_CREATE=true" \
--env "_AIRFLOW_WWW_USER_PASSWORD_CMD=echo admin" \
- apache/airflow:main-python3.8 webserver
+ apache/airflow:2.2.0.dev0-python3.8 webserver
The commands above perform initialization of the SQLite database, create admin user with admin password
and Admin role. They also forward local port ``8080`` to the webserver port and finally start the webserver.
@@ -312,6 +346,6 @@ Example:
--env "_AIRFLOW_DB_UPGRADE=true" \
--env "_AIRFLOW_WWW_USER_CREATE=true" \
--env "_AIRFLOW_WWW_USER_PASSWORD_CMD=echo admin" \
- apache/airflow:master-python3.8 webserver
+ apache/airflow:2.2.0.dev0-python3.8 webserver
This method is only available starting from Docker image of Airflow 2.1.1 and above.
diff --git a/scripts/ci/images/ci_run_prod_image_test.sh b/scripts/ci/images/ci_run_prod_image_test.sh
index 3039eca..24c30e0 100755
--- a/scripts/ci/images/ci_run_prod_image_test.sh
+++ b/scripts/ci/images/ci_run_prod_image_test.sh
@@ -23,6 +23,8 @@ initialization::set_output_color_variables
job_name=$1
file=$2
+latest_airflow_version_released=$(curl "https://pypi.org/pypi/apache-airflow/json" | jq '.info.version' -r)
+
set +e
if [[ ${file} == *".sh" ]]; then
@@ -30,7 +32,11 @@ if [[ ${file} == *".sh" ]]; then
res=$?
elif [[ ${file} == *"Dockerfile" ]]; then
cd "$(dirname "${file}")" || exit 1
- docker build . --tag "${job_name}"
+ echo
+ echo "${COLOR_BLUE}Replacing the airflow image version in ${file} with ${latest_airflow_version_released} for testing.${COLOR_RESET}"
+ echo
+ sed "s/FROM apache\/airflow:.*$/FROM apache\/airflow:${latest_airflow_version_released}/" <Dockerfile | \
+ docker build . --tag "${job_name}" -f -
res=$?
docker rmi --force "${job_name}"
else
diff --git a/scripts/ci/pre_commit/pre_commit_update_versions.py b/scripts/ci/pre_commit/pre_commit_update_versions.py
new file mode 100755
index 0000000..ee12bb6
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_update_versions.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import glob
+import os
+import re
+import sys
+from os.path import abspath, dirname, join
+
+AIRFLOW_SOURCES_DIR = abspath(join(dirname(__file__), os.pardir, os.pardir, os.pardir))
+
+sys.path.insert(0, AIRFLOW_SOURCES_DIR)
+# flake8: noqa: F401
+
+from setup import version # isort:skip
+
+
+def update_version(pattern, v: str, file_path: str):
+ print(f"Replacing {pattern} to {version} in {file_path}")
+ with open(file_path, "r+") as f:
+ file_contents = f.read()
+ lines = file_contents.splitlines(keepends=True)
+ for i in range(0, len(lines)):
+ lines[i] = re.sub(pattern, fr'\g<1>{v}\g<2>', lines[i])
+ file_contents = "".join(lines)
+ f.seek(0)
+ f.truncate()
+ f.write(file_contents)
+
+
+REPLACEMENTS = {
+ r'(FROM apache/airflow:).*($)': "docs/docker-stack/docker-examples/extending/*/Dockerfile",
+ r'(apache/airflow:)[^-]*(\-)': "docs/docker-stack/entrypoint.rst",
+ r'(/constraints-)[^-]*(/constraints)': "docs/docker-stack/docker-examples/"
+ "restricted/restricted_environments.sh",
+ r'(AIRFLOW_VERSION=")[^"]*(" \\)': "docs/docker-stack/docker-examples/"
+ "restricted/restricted_environments.sh",
+}
+
+if __name__ == '__main__':
+ for regexp, p in REPLACEMENTS.items():
+ text_pattern = re.compile(regexp)
+ files = glob.glob(join(AIRFLOW_SOURCES_DIR, p), recursive=True)
+ if not files:
+ print(f"ERROR! No files matched on {p}")
+ for file in glob.glob(join(AIRFLOW_SOURCES_DIR, p), recursive=True):
+ update_version(text_pattern, version, file)