You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/01/23 17:20:32 UTC
[airflow] 01/04: Add image labels required by ArtifactHub (#21040)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b334800de8e06426f2cada239cc27a3e3a79d368
Author: Kamil BreguĊa <mi...@users.noreply.github.com>
AuthorDate: Sun Jan 23 14:15:45 2022 +0100
Add image labels required by ArtifactHub (#21040)
(cherry picked from commit 7f02b4718bc9e282c1d59b0aab5dd46972b6f79c)
---
Dockerfile | 6 +-
docs/docker-stack/README.md | 75 ++++++++++++++++++++++
docs/docker-stack/index.rst | 5 +-
scripts/ci/libraries/_build_images.sh | 1 +
.../ci/pre_commit/pre_commit_update_versions.py | 7 +-
5 files changed, 91 insertions(+), 3 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 0edb037..86e6a0d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,6 +48,7 @@ ARG PYTHON_BASE_IMAGE="python:3.7-slim-buster"
ARG AIRFLOW_PIP_VERSION=21.3.1
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
+ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"
# By default latest released version of airflow is installed (when empty) but this value can be overridden
# and we can install version according to specification (For example ==2.0.2 or <3.0.0).
@@ -396,6 +397,7 @@ ARG AIRFLOW_HOME
# production image is prepared from sources rather than from package
ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
ARG AIRFLOW_IMAGE_REPOSITORY
+ARG AIRFLOW_IMAGE_README_URL
ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \
@@ -524,7 +526,9 @@ LABEL org.apache.airflow.distro="debian" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.ref.name="airflow" \
org.opencontainers.image.title="Production Airflow Image" \
- org.opencontainers.image.description="Reference, production-ready Apache Airflow image"
+ org.opencontainers.image.description="Reference, production-ready Apache Airflow image" \
+ io.artifacthub.package.license='Apache-2.0' \
+ io.artifacthub.package.readme-url='${AIRFLOW_IMAGE_README_URL}'
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
CMD []
diff --git a/docs/docker-stack/README.md b/docs/docker-stack/README.md
new file mode 100644
index 0000000..d6f1d81
--- /dev/null
+++ b/docs/docker-stack/README.md
@@ -0,0 +1,75 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!--
+This file was created for the purpose of publishing an image to ArtifactHub.
+Please try to keep it in sync with index.rst
+-->
+
+# Docker Image for Apache Airflow
+
+For the ease of deployment in production, the community releases a production-ready reference container
+image.
+
+The Apache Airflow community, releases Docker Images which are `reference images` for Apache Airflow.
+Every time a new version of Airflow is released, the images are prepared in the
+[apache/airflow DockerHub](https://hub.docker.com/r/apache/airflow)
+for all the supported Python versions.
+
+You can find the following images there (Assuming Airflow version `2.2.4`):
+
+* `apache/airflow:latest` - the latest released Airflow image with default Python version (3.7 currently)
+* `apache/airflow:latest-pythonX.Y` - the latest released Airflow image with specific Python version
+* `apache/airflow:2.2.4` - the versioned Airflow image with default Python version (3.7 currently)
+* `apache/airflow:2.2.4-pythonX.Y` - the versioned Airflow image with specific Python version
+
+Those are "reference" images. They contain the most common set of extras, dependencies and providers that are
+often used by the users and they are good to "try-things-out" when you want to just take Airflow for a spin,
+
+The Apache Airflow image provided as convenience package is optimized for size, and
+it provides just a bare minimal set of the extras and dependencies installed and in most cases
+you want to either extend or customize the image. You can see all possible extras in [Reference for package extras](https://airflow.apache.org/docs/apache-airflow/stable/extra-packages-ref.html).
+The set of extras used in Airflow Production image are available in the
+[Dockerfile](https://github.com/apache/airflow/blob/2c6c7fdb2308de98e142618836bdf414df9768c8/Dockerfile#L37).
+
+However, Airflow has more than 60 community-managed providers (installable via extras) and some of the
+default extras/providers installed are not used by everyone, sometimes others extras/providers
+are needed, sometimes (very often actually) you need to add your own custom dependencies,
+packages or even custom providers. You can learn how to do it in [Building the image](https://airflow.apache.org/docs/docker-stack/build.html#build-build-image).
+
+The production images are build in DockerHub from released version and release candidates. There
+are also images published from branches but they are used mainly for development and testing purpose.
+See [Airflow Git Branching](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#airflow-git-branches)
+for details.
+
+## Usage
+
+The [`AIRFLOW_HOME`](https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html#envvar-AIRFLOW_HOME) is set by default to ``/opt/airflow/`` - this means that DAGs
+are in default in the ``/opt/airflow/dags`` folder and logs are in the ``/opt/airflow/logs``
+
+The working directory is ``/opt/airflow`` by default.
+
+If no `AIRFLOW__CORE__SQL_ALCHEMY_CONN` variable is set then SQLite database is created in
+``${AIRFLOW_HOME}/airflow.db``.
+
+For example commands that start Airflow see: [Executing commands](https://airflow.apache.org/docs/docker-stack/entrypoint.html#entrypoint-commands).
+
+Airflow requires many components to function as it is a distributed application. You may therefore also be interested
+in launching Airflow in the Docker Compose environment, see: [Quick Start](https://airflow.apache.org/docs/apache-airflow/stable/start/index.html).
+
+You can use this image in [Helm Chart](https://airflow.apache.org/docs/helm-chart/stable/index.html) as well.
diff --git a/docs/docker-stack/index.rst b/docs/docker-stack/index.rst
index 3701844..411af82 100644
--- a/docs/docker-stack/index.rst
+++ b/docs/docker-stack/index.rst
@@ -15,6 +15,9 @@
specific language governing permissions and limitations
under the License.
+ .. WARNING:
+ IF YOU ARE UPDATING THIS FILE, CONSIDER UPDATING README.MD TOO.
+
.. image:: /img/docker-logo.png
:width: 100
@@ -44,7 +47,7 @@ Every time a new version of Airflow is released, the images are prepared in the
`apache/airflow DockerHub <https://hub.docker.com/r/apache/airflow>`_
for all the supported Python versions.
-You can find the following images there (Assuming Airflow version |airflow-version|):
+You can find the following images there (Assuming Airflow version :subst-code:`|airflow-version|`):
* :subst-code:`apache/airflow:latest` - the latest released Airflow image with default Python version (3.7 currently)
* :subst-code:`apache/airflow:latest-pythonX.Y` - the latest released Airflow image with specific Python version
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 2d84a92..c052261 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -673,6 +673,7 @@ function build_images::build_prod_images() {
--build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \
--build-arg AIRFLOW_IMAGE_REPOSITORY="https://github.com/${GITHUB_REPOSITORY}" \
--build-arg AIRFLOW_IMAGE_DATE_CREATED="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+ --build-arg AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/${COMMIT_SHA}/docs/docker-stack/README.md" \
"${additional_dev_args[@]}" \
"${additional_runtime_args[@]}" \
"${docker_cache_prod_directive[@]}" \
diff --git a/scripts/ci/pre_commit/pre_commit_update_versions.py b/scripts/ci/pre_commit/pre_commit_update_versions.py
index 2af0698..3898d64 100755
--- a/scripts/ci/pre_commit/pre_commit_update_versions.py
+++ b/scripts/ci/pre_commit/pre_commit_update_versions.py
@@ -30,7 +30,7 @@ from setup import version # isort:skip
def update_version(pattern: re.Pattern, v: str, file_path: str):
- print(f"Replacing {pattern} to {version} in {file_path}")
+ print(f"Checking {pattern} in {file_path}")
with open(file_path, "r+") as f:
file_content = f.read()
if not pattern.search(file_content):
@@ -38,6 +38,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: str):
new_content = pattern.sub(fr'\g<1>{v}\g<2>', file_content)
if file_content == new_content:
return
+ print(" Updated.")
f.seek(0)
f.truncate()
f.write(new_content)
@@ -46,8 +47,12 @@ def update_version(pattern: re.Pattern, v: str, file_path: str):
REPLACEMENTS = {
r'^(FROM apache\/airflow:).*($)': "docs/docker-stack/docker-examples/extending/*/Dockerfile",
r'(apache\/airflow:)[^-]*(\-)': "docs/docker-stack/entrypoint.rst",
+ r'(`apache/airflow:)[0-9].*?((?:-pythonX.Y)?`)': "docs/docker-stack/README.md",
+ r'(\(Assuming Airflow version `).*(`\))': "docs/docker-stack/README.md",
}
+print(f"Current version: {version}")
+
if __name__ == '__main__':
for regexp, p in REPLACEMENTS.items():
text_pattern = re.compile(regexp, flags=re.MULTILINE)