You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/05/26 11:55:29 UTC
[airflow] branch master updated: Better content of backport
packages CHANGELOG and INSTALL files (#9013)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 9764c90 Better content of backport packages CHANGELOG and INSTALL files (#9013)
9764c90 is described below
commit 9764c90b925a710506f43423302f3fa512490c03
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue May 26 13:54:58 2020 +0200
Better content of backport packages CHANGELOG and INSTALL files (#9013)
* Better content of backport packages CHANGELOG and INSTALL files
The content of Backport Packages CHANGELOG.txt and INSTALL files
has been updated to reflect that those are not full Airflow
releases.
1) Source package:
- INSTALL contains only references to preparing backport packages
- CHANGELOG.txt contains combined change log of all the packages
2) Binary packages:
- No INSTALL
- CHANGELOG.txt contains changelog for this package only
3) Whl packages
No change
* Update backport_packages/INSTALL
---
.gitignore | 1 +
backport_packages/CHANGELOG.txt | 1 -
backport_packages/INSTALL | 15 +++
backport_packages/build_source_package.sh | 135 +++++++++++++++++++++
dev/BACKPORT_PACKAGES.md | 19 +--
.../in_container/run_prepare_backport_packages.sh | 37 ++++++
6 files changed, 191 insertions(+), 17 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4673049..a675fca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -186,3 +186,4 @@ dmypy.json
/.kube
/.inputrc
log.txt*
+/backport_packages/CHANGELOG.txt
diff --git a/backport_packages/CHANGELOG.txt b/backport_packages/CHANGELOG.txt
deleted file mode 120000
index 3502868..0000000
--- a/backport_packages/CHANGELOG.txt
+++ /dev/null
@@ -1 +0,0 @@
-../CHANGELOG.txt
\ No newline at end of file
diff --git a/backport_packages/INSTALL b/backport_packages/INSTALL
new file mode 100644
index 0000000..57727ee
--- /dev/null
+++ b/backport_packages/INSTALL
@@ -0,0 +1,15 @@
+## INSTALL / BUILD instructions for Apache Airflow Backport packages
+
+NOTE! Those sources are only intended to be used to build Apache Airflow Backport packages, not the Apache Airflow. They have been generated using the unreleased version of Apache Airflow.
+(from master)
+
+This ia a generic installation method that requires docker and docker-compose to be installed.
+
+# [required] fetch the tarball and untar the sources. Move into the directory that was un-tarred.
+
+# [optional] run Apache RAT (release audit tool) to validate license headers
+# RAT docs here: https://creadur.apache.org/rat/. Requires Java and Apache Rat
+java -jar apache-rat.jar -E ./.rat-excludes -d .
+
+The only prerequisites to build the packages is docker and docker-compose.
+The instructions on how to build the packages are described in dev/BACKPORT_PACKAGES.md
diff --git a/backport_packages/build_source_package.sh b/backport_packages/build_source_package.sh
new file mode 100755
index 0000000..ad19e67
--- /dev/null
+++ b/backport_packages/build_source_package.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Use this to sign the tar balls generated from
+# python setup.py sdist --formats=gztar
+# ie. sign.sh <my_tar_ball>
+# you will still be required to type in your signing key password
+# or it needs to be available in your keychain
+set -euo pipefail
+
+MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "${MY_DIR}"/..
+
+function check_version() {
+ if [[ ${VERSION:=} == "" ]]; then
+ echo
+ echo "Please export VERSION variable with the version of source package to prepare"
+ echo
+ exit 1
+ fi
+}
+
+function tag_release() {
+ echo
+ echo "Tagging the sources with backport-providers-${VERSION} tag"
+ echo
+
+ git tag "backport-providers-${VERSION}"
+}
+
+function clean_repo() {
+ ./confirm "Cleaning the repository sources - that might remove some of your unchanged files"
+
+ git clean -fxd
+}
+
+
+function prepare_combined_changelog() {
+ echo
+ echo "Preparing the changelog"
+ echo
+ CHANGELOG_FILE="backport_packages/CHANGELOG.txt"
+ PATTERN="airflow\/providers\/(.*)\/PROVIDERS_CHANGES_.*.md"
+ echo > "${CHANGELOG_FILE}"
+ CHANGES_FILES=$(find "airflow/providers/" -name 'PROVIDERS_CHANGES_*.md' | sort -r)
+ LAST_PROVIDER_ID=""
+ for FILE in ${CHANGES_FILES}
+ do
+ echo "Adding ${FILE}"
+ [[ ${FILE} =~ ${PATTERN} ]]
+ PROVIDER_ID=${BASH_REMATCH[1]//\//.}
+ {
+ if [[ ${LAST_PROVIDER_ID} != "${PROVIDER_ID}" ]]; then
+ echo
+ echo "Provider: ${BASH_REMATCH[1]//\//.}"
+ echo
+ LAST_PROVIDER_ID=${PROVIDER_ID}
+ else
+ echo
+ fi
+ cat "${FILE}"
+ echo
+ } >> "${CHANGELOG_FILE}"
+ done
+
+
+ echo
+ echo "Changelog prepared in ${CHANGELOG_FILE}"
+ echo
+}
+
+function prepare_archive(){
+ echo
+ echo "Preparing the archive ${ARCHIVE_FILE_NAME}"
+ echo
+
+ git archive \
+ --format=tar.gz \
+ "backport-providers-${VERSION}" \
+ "--prefix=apache-airflow-backport-providers-${VERSION}/" \
+ -o "${ARCHIVE_FILE_NAME}"
+
+ echo
+ echo "Prepared the archive ${ARCHIVE_FILE_NAME}"
+ echo
+
+}
+
+
+function replace_install_changelog(){
+ DIR=$(mktemp -d)
+
+ echo
+ echo "Replacing INSTALL CHANGELOG.txt in ${ARCHIVE_FILE_NAME} "
+ echo
+ tar -f "apache-airflow-backport-providers-${VERSION}-source.tar.gz" -xz -C "${DIR}"
+
+ cp "backport_packages/INSTALL" "backport_packages/CHANGELOG.txt" \
+ "${DIR}/apache-airflow-backport-providers-${VERSION}/"
+
+ tar -f "apache-airflow-backport-providers-${VERSION}-source.tar.gz" -cz -C "${DIR}" \
+ "apache-airflow-backport-providers-${VERSION}/"
+
+ echo
+ echo "Replaced INSTALL CHANGELOG.txt in ${ARCHIVE_FILE_NAME} "
+ echo
+
+}
+
+
+
+check_version
+
+export ARCHIVE_FILE_NAME="apache-airflow-backport-providers-${VERSION}-source.tar.gz"
+
+tag_release
+clean_repo
+prepare_archive
+prepare_combined_changelog
+replace_install_changelog
diff --git a/dev/BACKPORT_PACKAGES.md b/dev/BACKPORT_PACKAGES.md
index 11bfa65..5f44793 100644
--- a/dev/BACKPORT_PACKAGES.md
+++ b/dev/BACKPORT_PACKAGES.md
@@ -253,27 +253,14 @@ export AIRFLOW_REPO_ROOT=$(pwd)
```
-* Tag the release
+* Build the source package:
-```bash
-git tag backport-providers-${VERSION}
```
+./backport_packages/build_source_package.sh
-* Clean the checkout
-
-```bash
-git clean -fxd
```
-* Tarball the sources
-
-```bash
-git archive \
- --format=tar.gz \
- backport-providers-${VERSION} \
- --prefix=apache-airflow-backport-providers-${VERSION}/ \
- -o apache-airflow-backport-providers-${VERSION}-source.tar.gz
-```
+It will generate `apache-airflow-backport-providers-${VERSION}-source.tar.gz`
* Generate the packages - since we are preparing packages for SVN repo, we should use the right switch. Note
that this will clean up dist folder before generating the packages, so it will only contain the packages
diff --git a/scripts/ci/in_container/run_prepare_backport_packages.sh b/scripts/ci/in_container/run_prepare_backport_packages.sh
index 7664807..a70d966 100755
--- a/scripts/ci/in_container/run_prepare_backport_packages.sh
+++ b/scripts/ci/in_container/run_prepare_backport_packages.sh
@@ -105,6 +105,42 @@ do
cat "${LOG_FILE}"
exit "${RES}"
fi
+ echo > "${LOG_FILE}"
+
+ PACKAGE_DIR=${BACKPORT_PACKAGE//./\/}
+
+ PATTERN="airflow\/providers\/(.*)\/PROVIDERS_CHANGES_.*.md"
+ CHANGELOG_FILE="CHANGELOG.txt"
+
+ echo > "${CHANGELOG_FILE}"
+ CHANGES_FILES=$(find "airflow/providers/${PACKAGE_DIR}" -name 'PROVIDERS_CHANGES_*.md' | sort -r)
+ LAST_PROVIDER_ID=""
+ for FILE in ${CHANGES_FILES}
+ do
+ [[ ${FILE} =~ ${PATTERN} ]]
+ PROVIDER_ID=${BASH_REMATCH[1]//\//.}
+ {
+ if [[ ${LAST_PROVIDER_ID} != "${PROVIDER_ID}" ]]; then
+ echo
+ echo "Provider: ${BASH_REMATCH[1]//\//.}"
+ echo
+ LAST_PROVIDER_ID=${PROVIDER_ID}
+ else
+ echo
+ fi
+ cat "${FILE}"
+ echo
+ } >> "${CHANGELOG_FILE}"
+ done
+
+ echo "Changelog prepared in ${CHANGELOG_FILE} for ${PACKAGE_DIR}"
+ set +e
+ python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" clean --all >"${LOG_FILE}" 2>&1
+ RES="${?}"
+ if [[ ${RES} != "0" ]]; then
+ cat "${LOG_FILE}"
+ exit "${RES}"
+ fi
python3 setup_backport_packages.py --version-suffix "${VERSION_SUFFIX_FOR_PYPI} " \
"${BACKPORT_PACKAGE}" sdist bdist_wheel >"${LOG_FILE}" 2>&1
RES="${?}"
@@ -114,6 +150,7 @@ do
fi
set -e
echo " Prepared backport package ${BACKPORT_PACKAGE}"
+ echo "==================================================================================="
done
cd "${AIRFLOW_SOURCES}" || exit 1