You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2022/08/16 15:35:13 UTC
[airflow] 06/11: Build and publish Airflow wheels (#3)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to tag v2.3.3+astro.2
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7e48c8b02358ae8f094e84d183ea2716d1270a7a
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Wed Jul 20 15:31:10 2022 +0100
Build and publish Airflow wheels (#3)
* Build and publish Airflow wheels
Build Airflow wheels and publish at pip.astronomer.io/v2.
The workflow runs whenever there's a commit on branches starting
with 'v'
* Make version dev
---
.github/workflows/build-wheel.yml | 57 ++++++++++++++++++++++++++++++++
scripts/ci/astronomer-release.sh | 68 +++++++++++++++++++++++++++++++++++++++
setup.py | 37 ++++++++++++++++++++-
3 files changed, 161 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml
new file mode 100644
index 0000000000..a2a6e30166
--- /dev/null
+++ b/.github/workflows/build-wheel.yml
@@ -0,0 +1,57 @@
+---
+name: 'Build and publish wheel'
+on: # yamllint disable-line rule:truthy
+ push:
+ branches: ['v[0-9]+-[0-9]+-[0-9]']
+ tags: ['v[0-9]+\.[0-9]+\.[0-9]+\+astro.*']
+permissions:
+ # All other permissions are set to none
+ contents: read
+ packages: read
+ pull-requests: read
+
+jobs:
+ deploy:
+ timeout-minutes: 10
+ name: "Publish wheels"
+ runs-on: ubuntu-latest
+ if: github.event_name != 'pull_request'
+ env:
+ PULL_PYTHON_BASE_IMAGES_FROM_CACHE: "false"
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ CI_JOB_TYPE: "Prod image"
+ INSTALL_PROVIDERS_FROM_SOURCES: false
+ steps:
+ - uses: actions/checkout@master
+ - uses: actions/setup-python@v2
+ with:
+ python-version: '3.7'
+ - run: |
+ pip install --upgrade pip==21.2.4
+ pip install packaging GitPython wheel
+ - name: Create Python Wheel
+ run: ./scripts/ci/astronomer-release.sh
+ - name: "Deploy Python Wheel and tar.gz"
+ uses: actions-hub/gcloud@330.0.0
+ env:
+ APPLICATION_CREDENTIALS: ${{secrets.ASTRONOMER_PROD}}
+ PROJECT_ID: ${{secrets.GCLOUD_PROJECT_ID}}
+ with:
+ args: cp -a public-read -r ./dist/* gs://pip.astronomer.io/v2
+ cli: gsutil
+ - name: "Deploy '.build' file (containing deployed version number)"
+ uses: actions-hub/gcloud@330.0.0
+ env:
+ APPLICATION_CREDENTIALS: ${{secrets.ASTRONOMER_PROD}}
+ PROJECT_ID: ${{secrets.GCLOUD_PROJECT_ID}}
+ with:
+ args: >
+ -h "Cache-Control:no-cache,max-age=0"
+ cp -a public-read -r "./**/*.build" gs://pip.astronomer.io/v2/apache-airflow
+ cli: gsutil
+ - name: Index Pages
+ env:
+ GOOGLE_APPLICATION_CREDENTIALS_CONTENT: ${{secrets.ASTRONOMER_PROD}}
+ run: |
+ git clone https://github.com/astronomer/pip-release-utilities.git pip-release-utilities
+ ./pip-release-utilities/build-index-page.sh pip.astronomer.io v2/apache-airflow
diff --git a/scripts/ci/astronomer-release.sh b/scripts/ci/astronomer-release.sh
new file mode 100755
index 0000000000..1032386d5d
--- /dev/null
+++ b/scripts/ci/astronomer-release.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# This file is Not licensed to ASF
+# SKIP LICENSE INSERTION
+
+export PYTHON_VERSION=3.7
+
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/libraries/_script_init.sh"
+
+# shellcheck disable=SC2153
+export TAG=${GITHUB_REF/refs\/tags\//}
+if [[ "$GITHUB_REF" != *"tags"* ]]; then
+ export TAG=""
+fi
+echo "TAG: $TAG"
+# shellcheck disable=SC2153
+export BRANCH=${GITHUB_REF#refs/*/}
+echo "BRANCH: $BRANCH"
+
+# If the build is not a tag/release, build a dev version
+if [[ -z ${TAG:=} ]]; then
+ echo
+ AIRFLOW_VERSION=$(awk '/version =/{print $NF; exit}' setup.py | tr -d \')
+ export AIRFLOW_VERSION
+ echo "Current Version is: ${AIRFLOW_VERSION}"
+
+ if [[ ${AIRFLOW_VERSION} == *"dev"* ]]; then
+ echo "Building and releasing a DEV Package"
+ echo
+ git fetch --unshallow
+
+ COMMITS_SINCE_LAST_TAG="$(git rev-list "$(git describe --abbrev=0 --tags)"..HEAD --count)"
+ sed -i -E "s/dev[0-9]+/dev${COMMITS_SINCE_LAST_TAG}/g" airflow/version.py
+ sed -i -E "s/dev[0-9]+\+astro/dev${COMMITS_SINCE_LAST_TAG}\+astro/g" setup.py
+
+ UPDATED_AIRFLOW_VERSION=$(awk '/version = /{print $NF}' setup.py | tr -d \')
+ export UPDATED_AIRFLOW_VERSION
+ echo "Updated Airflow Version: $UPDATED_AIRFLOW_VERSION"
+
+ python3 setup.py --quiet verify compile_assets sdist --dist-dir dist/apache-airflow/ bdist_wheel --dist-dir dist/apache-airflow/
+ else
+ echo "Version does not contain 'dev' in airflow/version.py"
+ echo "Skipping build and release process"
+ echo
+ exit 1
+ fi
+elif [[ ${TAG:=} == "${BRANCH:=}" ]]; then
+ python3 setup.py --quiet verify compile_assets sdist --dist-dir dist/apache-airflow/ bdist_wheel --dist-dir dist/apache-airflow/
+fi
+
+ls -altr dist/*/*
+
+
+# Get the version of AC (Example 2.3.3.post7)
+CURRENT_AC_VERSION=$(echo dist/apache-airflow/apache_airflow-*.whl | sed -E 's|.*apache_airflow-(.+)-py3-none-any.whl|\1|')
+export CURRENT_AC_VERSION
+echo "AC Version: $CURRENT_AC_VERSION"
+
+# Get the version of Apache Airflow (Example 2.3.3)
+AIRFLOW_BASE_VERSION=$(echo "$CURRENT_AC_VERSION" | sed -E 's|([0-9]+\.[0-9]+\.[0-9]+).*|\1|')
+export AIRFLOW_BASE_VESION
+echo "Airflow Base Version: $AIRFLOW_BASE_VERSION"
+
+# Store the latest version info in a separate file
+# Example: 'apache-airflow/latest-2.3.3.build' contains '2.3.3.post7'
+mkdir apache-airflow
+echo "${CURRENT_AC_VERSION}" > apache-airflow/latest-"$AIRFLOW_BASE_VERSION".build
diff --git a/setup.py b/setup.py
index 03c3065d98..f5d68aab12 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ PY39 = sys.version_info >= (3, 9)
logger = logging.getLogger(__name__)
-version = '2.3.3+astro.1'
+version = '2.3.3.dev0+astro.1'
my_dir = dirname(__file__)
@@ -132,6 +132,40 @@ class ListExtras(Command):
print("\n".join(wrap(", ".join(EXTRAS_REQUIREMENTS.keys()), 100)))
+class VerifyVersionCommand(Command):
+ """Custom command to verify that the git tag matches our version"""
+
+ description = 'verify that the git tag matches our version'
+ user_options = [] # type: ignore
+
+ def initialize_options(self):
+ """Set default values for options."""
+
+ def finalize_options(self):
+ """Set final values for options."""
+
+ def run(self):
+ """Run Verify Version Command"""
+ tag = os.getenv('TAG')
+ branch = os.getenv('BRANCH')
+
+ if tag and tag != "v" + version:
+ info = f"Git tag: {tag} does not match the version of this app: v{version}"
+ sys.exit(info)
+ if branch and not version.startswith(branch.lstrip('v').replace('-', '.')):
+ info = f"Git Branch: {branch} does not match the version of this app: v{version}"
+ sys.exit(info)
+ import re
+
+ # Check the version matches the expected format of 1.10.7+astro.1
+ if not re.match(r'^\d\.\d+\.\d+(\.(dev|rc)\d+)?\+astro\.\d+$', version):
+ info = (
+ f"Version: {version} does not match the expected format of '1.2.3+astro.4'"
+ "or '1.2.3.(dev|rc)4+astro.5'"
+ )
+ sys.exit(info)
+
+
def git_version(version_: str) -> str:
"""
Return a version to identify the state of the underlying git repo. The version will
@@ -1170,6 +1204,7 @@ def do_setup() -> None:
'list_extras': ListExtras,
'install': Install, # type: ignore
'develop': Develop,
+ 'verify': VerifyVersionCommand,
},
test_suite='setup.airflow_test_suite',
**setup_kwargs, # type: ignore