You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/06/22 19:25:21 UTC
[airflow] 43/47: Adds automated generation of provider issue to
track test progress (#16419)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c462ddaba1fefad55720fe3e634bee2a615f1368
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Jun 20 16:31:28 2021 +0200
Adds automated generation of provider issue to track test progress (#16419)
Allows to automatically generate draft of the issue which can be
used to track progress of testing released providers.
(cherry picked from commit f40d5557880660863c69d146e7ed6ae585932936)
---
.github/workflows/ci.yml | 5 +-
BREEZE.rst | 4 +
breeze | 17 ++-
breeze-complete | 2 +-
dev/README_RELEASE_PROVIDER_PACKAGES.md | 24 +++-
.../PROVIDER_ISSUE_TEMPLATE.md.jinja2 | 35 ++++++
dev/provider_packages/prepare_provider_packages.py | 127 ++++++++++++++++++++-
scripts/ci/docker-compose/forward-credentials.yml | 2 +
scripts/ci/libraries/_runs.sh | 2 +
.../run_prepare_provider_documentation.sh | 27 ++++-
setup.py | 1 +
11 files changed, 233 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9edec76..7b72347 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -318,8 +318,6 @@ jobs:
uses: actions/checkout@v2
with:
persist-credentials: false
- # Needs to fetch all history to be able to verify changelog generation
- fetch-depth: 0
- name: "Setup python"
uses: actions/setup-python@v2
with:
@@ -520,6 +518,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
VERSION_SUFFIX_FOR_PYPI: ".dev0"
GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
NON_INTERACTIVE: "true"
+ GENERATE_PROVIDERS_ISSUE: "true"
if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -567,6 +566,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
VERSION_SUFFIX_FOR_PYPI: ".dev0"
GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
+ NON_INTERACTIVE: "true"
+ GENERATE_PROVIDERS_ISSUE: "true"
if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
diff --git a/BREEZE.rst b/BREEZE.rst
index 428bc92..4e13653 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2141,6 +2141,10 @@ This is the current syntax for `./breeze <./breeze>`_:
Runs the command in non-interactive mode.
+ --generate-providers-issue
+
+ Generate providers issue that should be created.
+
-v, --verbose
Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
debugging - when you run breeze with --verbose flags you will be able to see the commands
diff --git a/breeze b/breeze
index bbe28e3..48021f9 100755
--- a/breeze
+++ b/breeze
@@ -1244,6 +1244,13 @@ function breeze::parse_arguments() {
echo
shift
;;
+ --generate-providers-issue)
+ export GENERATE_PROVIDERS_ISSUE="true"
+ export FORWARD_CREDENTIALS="true"
+ echo "Generating providers issue (includes forwarding credentials)"
+ echo
+ shift
+ ;;
--installation-method)
export AIRFLOW_INSTALLATION_METHOD="${2}"
echo "Airflow installation method: ${AIRFLOW_INSTALLATION_METHOD}"
@@ -1829,7 +1836,7 @@ ${CMDNAME} prepare-provider-documentation [FLAGS] [PACKAGE_ID ...]
Flags:
$(breeze::flag_version_suffix)
$(breeze::flag_packages)
-$(breeze::flag_non_interactive)
+$(breeze::prepare_providers_documentation)
$(breeze::flag_verbosity)
"
readonly DETAILED_USAGE_PREPARE_PROVIDER_DOCUMENTATION
@@ -2556,17 +2563,21 @@ ${FORMATTED_PACKAGE_FORMATS}
#######################################################################################################
#
-# Prints flags for non-interactive run
+# Prints flags for prepare-providers-documentation
#
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_non_interactive() {
+function breeze::prepare_providers_documentation() {
echo "
--non-interactive
Runs the command in non-interactive mode.
+--generate-providers-issue
+
+ Generate providers issue that should be created.
+
"
}
diff --git a/breeze-complete b/breeze-complete
index a9d4d04..f4445da 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -183,7 +183,7 @@ additional-extras: additional-python-deps: disable-pypi-when-building skip-insta
dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env:
runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env:
load-default-connections load-example-dags
-use-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies installation-method: continue-on-pip-check-failure non-interactive
+use-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies installation-method: continue-on-pip-check-failure non-interactive generate-providers-issue
use-airflow-version:
cleanup-docker-context-files
test-type: preserve-volumes dry-run-docker
diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md
index 619083c..dcb729a 100644
--- a/dev/README_RELEASE_PROVIDER_PACKAGES.md
+++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md
@@ -31,6 +31,7 @@
- [Publish the Regular convenience package to PyPI](#publish-the-regular-convenience-package-to-pypi)
- [Add tags in git](#add-tags-in-git)
- [Prepare documentation](#prepare-documentation)
+ - [Prepare issue in GitHub to keep status of testing](#prepare-issue-in-github-to-keep-status-of-testing)
- [Prepare voting email for Providers release candidate](#prepare-voting-email-for-providers-release-candidate)
- [Verify the release by PMC members](#verify-the-release-by-pmc-members)
- [Verify by Contributors](#verify-by-contributors)
@@ -109,6 +110,17 @@ are updated, run it in non-interactive mode:
./breeze --non-interactive prepare-provider-documentation [packages]
```
+When you run the command and documentation generation is successful you will get a command that you can run to
+create GitHub issue where you will be tracking status of tests for the providers you release.
+
+You can also trigger automated execution of the issue by running:
+
+```shell script
+./breeze --non-interactive --generate-providers-issue prepare-provider-documentation [packages]
+```
+
+Once you release packages, you should create the issue with the content specified and link to it in
+the email sent to the devlist.
## Build provider packages for SVN apache upload
@@ -338,6 +350,11 @@ git commit -m "Add documentation for packages - $(date "+%Y-%m-%d%n")"
git push --set-upstream origin "${branch}"
```
+## Prepare issue in GitHub to keep status of testing
+
+Create GitHub issue with the content generated via prepare-provider-documentation or manual
+execution of the script above. You will use link to that issue in the next step.
+
## Prepare voting email for Providers release candidate
Make sure the packages are in https://dist.apache.org/repos/dist/dev/airflow/providers/
@@ -350,7 +367,7 @@ subject:
```shell script
cat <<EOF
-[VOTE] Airflow Providers - release prepared $(date "+%Y-%m-%d%n")
+[VOTE] Airflow Providers prepared on $(date "+%B %d, %Y")
EOF
```
@@ -363,7 +380,7 @@ which will last for 72 hours - which means that it will end on $(date -d '+3 day
Consider this my (binding) +1.
-<PASTE ANY HIGH-LEVEL DESCRIPTION OF THE CHANGES HERE!>
+<ADD ANY HIGH-LEVEL DESCRIPTION OF THE CHANGES HERE!>
Airflow Providers are available at:
https://dist.apache.org/repos/dist/dev/airflow/providers/
@@ -399,6 +416,9 @@ Please note that the version number excludes the 'rcX' string.
This will allow us to rename the artifact without modifying
the artifact checksums when we actually release.
+The status of testing the providers by the community is kept here:
+<TODO COPY LINK TO THE ISSUE CREATED>
+
You can find packages as well as detailed changelog following the below links:
<PASTE TWINE UPLOAD LINKS HERE. SORT THEM BEFORE!>
diff --git a/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 b/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2
new file mode 100644
index 0000000..a436bcb
--- /dev/null
+++ b/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2
@@ -0,0 +1,35 @@
+I have a kind request for all the contributors to the latest provider packages release.
+Could you help us to test the RC versions of the providers and let us know in the comment,
+if the issue is addressed there.
+
+## Providers that need testing
+
+Those are providers that require testing as there were some substantial changes introduced:
+
+{% for provider_id, provider_pr_info in interesting_providers.items() %}
+### Provider [{{ provider_id }}: {{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}](https://pypi.org/project/{{ provider_pr_info.provider_details.pypi_package_name }}/{{ provider_pr_info.provider_details.versions[0] }}{{ suffix }})
+{%- for pr in provider_pr_info.pr_list %}
+ - [ ] [{{ pr.title }} (#{{ pr.number }})]({{ pr.html_url }}): @{{ pr.user.login }}
+{%- endfor %}
+{%- endfor %}
+
+## Providers that do not need testing
+
+Those are providers that were either doc-only or had changes that do not require testing.
+
+{% for provider_id, provider_pr_info in non_interesting_providers.items() %}
+* Provider [{{ provider_id }}: {{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}](https://pypi.org/project/{{ provider_pr_info.provider_details.pypi_package_name }}/{{ provider_pr_info.provider_details.versions[0] }}{{ suffix }})
+{%- endfor %}
+
+<!--
+
+NOTE TO RELEASE MANAGER:
+
+Please move here the providers that have doc-only changes or for which changes are trivial and
+you could asses that they are OK. In case
+
+The providers are automatically installed on Airflow 2.1 and latest `main` during the CI, so we know they
+are installable. Also all classes within the providers are imported during the CI run so we know all
+providers can be imported.
+
+-->
diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 00eec60..87aab2d 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -44,9 +44,11 @@ from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple,
import click
import jsonschema
import yaml
+from github import Github, PullRequest, UnknownObjectException
from packaging.version import Version
from rich import print
from rich.console import Console
+from rich.progress import Progress
from rich.syntax import Syntax
try:
@@ -155,6 +157,7 @@ option_force = click.option(
)
argument_package_id = click.argument('package_id')
argument_changelog_files = click.argument('changelog_files', nargs=-1)
+argument_package_ids = click.argument('package_ids', nargs=-1)
@contextmanager
@@ -200,6 +203,7 @@ class VerifiedEntities(NamedTuple):
class ProviderPackageDetails(NamedTuple):
provider_package_id: str
full_package_name: str
+ pypi_package_name: str
source_provider_package_path: str
documentation_provider_package_path: str
provider_description: str
@@ -1436,6 +1440,7 @@ def get_provider_details(provider_package_id: str) -> ProviderPackageDetails:
return ProviderPackageDetails(
provider_package_id=provider_package_id,
full_package_name=f"airflow.providers.{provider_package_id}",
+ pypi_package_name=f"apache-airflow-providers-{provider_package_id.replace('.', '-')}",
source_provider_package_path=get_source_package_path(provider_package_id),
documentation_provider_package_path=get_documentation_package_path(provider_package_id),
provider_description=provider_info['description'],
@@ -1797,7 +1802,7 @@ def get_all_providers() -> List[str]:
return list(PROVIDERS_REQUIREMENTS.keys())
-def verify_provider_package(provider_package_id: str) -> None:
+def verify_provider_package(provider_package_id: str) -> str:
"""
Verifies if the provider package is good.
:param provider_package_id: package id to verify
@@ -2246,6 +2251,7 @@ def get_package_from_changelog(changelog_path: str):
@option_git_update
@option_verbose
def update_changelogs(changelog_files: List[str], git_update: bool, verbose: bool):
+ """Updates changelogs for multiple packages."""
if git_update:
make_sure_remote_apache_exists_and_fetch(git_update, verbose)
for changelog_file in changelog_files:
@@ -2253,6 +2259,125 @@ def update_changelogs(changelog_files: List[str], git_update: bool, verbose: boo
_update_changelog(package_id=package_id, verbose=verbose)
+def get_prs_for_package(package_id: str) -> List[int]:
+ pr_matcher = re.compile(r".*\(#([0-9]*)\)``$")
+ verify_provider_package(package_id)
+ changelog_path = verify_changelog_exists(package_id)
+ provider_details = get_provider_details(package_id)
+ current_release_version = provider_details.versions[0]
+ prs = []
+ with open(changelog_path) as changelog_file:
+ changelog_lines = changelog_file.readlines()
+ extract_prs = False
+ skip_line = False
+ for line in changelog_lines:
+ if skip_line:
+ # Skip first "....." header
+ skip_line = False
+ continue
+ if line.strip() == current_release_version:
+ extract_prs = True
+ skip_line = True
+ continue
+ if extract_prs:
+ if all(c == '.' for c in line):
+ # Header for next version reached
+ break
+ if line.startswith('.. Below changes are excluded from the changelog'):
+ # The reminder of PRs is not important skipping it
+ break
+ match_result = pr_matcher.match(line.strip())
+ if match_result:
+ prs.append(int(match_result.group(1)))
+ return prs
+
+
+class ProviderPRInfo(NamedTuple):
+ provider_details: ProviderPackageDetails
+ pr_list: List[PullRequest.PullRequest]
+
+
+@cli.command()
+@click.option('--github-token', envvar='GITHUB_TOKEN')
+@click.option('--suffix', default='rc1')
+@click.option('--excluded-pr-list', type=str, help="Coma-separated list of PRs to exclude from the issue.")
+@argument_package_ids
+def generate_issue_content(package_ids: List[str], github_token: str, suffix: str, excluded_pr_list: str):
+ if not package_ids:
+ package_ids = get_all_providers()
+ """Generates content for issue to test the release."""
+ with with_group("Generates GitHub issue content with people who can test it"):
+ if excluded_pr_list:
+ excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")]
+ else:
+ excluded_prs = []
+ console = Console(width=200, color_system="standard")
+ all_prs: Set[int] = set()
+ provider_prs: Dict[str, List[int]] = {}
+ for package_id in package_ids:
+ console.print(f"Extracting PRs for provider {package_id}")
+ prs = get_prs_for_package(package_id)
+ provider_prs[package_id] = list(filter(lambda pr: pr not in excluded_prs, prs))
+ all_prs.update(provider_prs[package_id])
+ g = Github(github_token)
+ repo = g.get_repo("apache/airflow")
+ pull_requests: Dict[int, PullRequest.PullRequest] = {}
+ with Progress(console=console) as progress:
+ task = progress.add_task(f"Retrieving {len(all_prs)} PRs ", total=len(all_prs))
+ pr_list = list(all_prs)
+ for i in range(len(pr_list)):
+ pr_number = pr_list[i]
+ progress.console.print(
+ f"Retrieving PR#{pr_number}: " f"https://github.com/apache/airflow/pull/{pr_number}"
+ )
+ try:
+ pull_requests[pr_number] = repo.get_pull(pr_number)
+ except UnknownObjectException:
+ # Fallback to issue if PR not found
+ try:
+ pull_requests[pr_number] = repo.get_issue(pr_number) # noqa (same fields as PR)
+ except UnknownObjectException:
+ console.print(f"[red]The PR #{pr_number} could not be found[/]")
+ progress.advance(task)
+ interesting_providers: Dict[str, ProviderPRInfo] = {}
+ non_interesting_providers: Dict[str, ProviderPRInfo] = {}
+ for package_id in package_ids:
+ pull_request_list = [pull_requests[pr] for pr in provider_prs[package_id] if pr in pull_requests]
+ provider_details = get_provider_details(package_id)
+ if pull_request_list:
+ interesting_providers[package_id] = ProviderPRInfo(provider_details, pull_request_list)
+ else:
+ non_interesting_providers[package_id] = ProviderPRInfo(provider_details, pull_request_list)
+ context = {
+ 'interesting_providers': interesting_providers,
+ 'date': datetime.now(),
+ 'suffix': suffix,
+ 'non_interesting_providers': non_interesting_providers,
+ }
+ issue_content = render_template(template_name="PROVIDER_ISSUE", context=context, extension=".md")
+ console.print()
+ console.print(
+ "[green]Below you can find the issue content that you can use "
+ "to ask contributor to test providers![/]"
+ )
+ console.print()
+ console.print()
+ console.print(
+ "Issue title: [yellow]Status of testing Providers that were "
+ f"prepared on { datetime.now().strftime('%B %d, %Y') }[/]"
+ )
+ console.print()
+ syntax = Syntax(issue_content, "markdown", theme="ansi_dark")
+ console.print(syntax)
+ console.print()
+ users: Set[str] = set()
+ for provider_info in interesting_providers.values():
+ for pr in provider_info.pr_list:
+ users.add("@" + pr.user.login)
+ console.print("All users involved in the PRs:")
+ console.print(" ".join(users))
+
+
if __name__ == "__main__":
# The cli exit code is:
# * 0 in case of success
diff --git a/scripts/ci/docker-compose/forward-credentials.yml b/scripts/ci/docker-compose/forward-credentials.yml
index 0eb0dbb..ebbd2ec 100644
--- a/scripts/ci/docker-compose/forward-credentials.yml
+++ b/scripts/ci/docker-compose/forward-credentials.yml
@@ -24,6 +24,8 @@ services:
# Everything you install in Docker
# If you add it here - also add it to "in_container_fix_ownership" method in
# the _in_container_utils.sh file to make it friendly for Linux users
+ environment:
+ - GITHUB_TOKEN
volumes:
- ${HOME}/.aws:/root/.aws:cached
- ${HOME}/.azure:/root/.azure:cached
diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh
index d15e68b..f7bb69f 100644
--- a/scripts/ci/libraries/_runs.sh
+++ b/scripts/ci/libraries/_runs.sh
@@ -73,6 +73,8 @@ function runs::run_prepare_provider_documentation() {
"${term_flag}" \
-v "${AIRFLOW_SOURCES}:/opt/airflow" \
-e "NON_INTERACTIVE" \
+ -e "GENERATE_PROVIDERS_ISSUE" \
+ -e "GITHUB_TOKEN" \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh" "${@}"
}
diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh
index bbc889b..9130c79 100755
--- a/scripts/in_container/run_prepare_provider_documentation.sh
+++ b/scripts/in_container/run_prepare_provider_documentation.sh
@@ -96,19 +96,19 @@ function run_prepare_documentation() {
echo
if [[ "${#prepared_documentation[@]}" != "0" ]]; then
echo "${COLOR_GREEN} Success:${COLOR_RESET}"
- echo "${prepared_documentation[@]}" | fold -w 100
+ echo "${prepared_documentation[@]}" | fold -sw 100
fi
if [[ "${#skipped_documentation[@]}" != "0" ]]; then
echo "${COLOR_YELLOW} Skipped:${COLOR_RESET}"
- echo "${skipped_documentation[@]}" | fold -w 100
+ echo "${skipped_documentation[@]}" | fold -sw 100
fi
if [[ "${#doc_only_documentation[@]}" != "0" ]]; then
echo "${COLOR_YELLOW} Marked as doc-only (please commit those!):${COLOR_RESET}"
- echo "${doc_only_documentation[@]}" | fold -w 100
+ echo "${doc_only_documentation[@]}" | fold -sw 100
fi
if [[ "${#error_documentation[@]}" != "0" ]]; then
echo "${COLOR_RED} Errors:${COLOR_RESET}"
- echo "${error_documentation[@]}" | fold -w 100
+ echo "${error_documentation[@]}" | fold -sw 100 | sed "s/^/ /" | sed "s/$/\\/"
fi
echo
echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
@@ -117,9 +117,28 @@ function run_prepare_documentation() {
echo "${COLOR_RED}There were errors when preparing documentation. Exiting! ${COLOR_RESET}"
exit 1
else
+ if [[ ${GENERATE_PROVIDERS_ISSUE=} == "true" ]]; then
+ echo
+ python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content "${prepared_documentation[@]}"
+ echo
+ fi
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ echo
+ echo "${COLOR_YELLOW}You can separately generate content of the issue to create to track testing status by running this command${COLOR_RESET}"
+ echo "${COLOR_YELLOW}You can optionally exclude some PRs via --excluded-pr-list option containing coma-separated list of pr numbers${COLOR_RESET}"
+ echo
+ echo "python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content \\"
+ echo "${prepared_documentation[@]}" | fold -sw 100 | sed "s/^/ /" | sed "s/$/ \\\\/"
+ echo " [--excluded-pr-list=\"\" ]"
+ echo
+ echo "${COLOR_YELLOW}You can also run it here by rerunning the prepare-providers-documentation${COLOR_RESET}"
+ echo "${COLOR_YELLOW}with --generate-providers-issue and --forward-credentials flag (you have to have GITHUB_TOKEN variable set)${COLOR_RESET}"
+ echo
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
echo
echo "${COLOR_YELLOW}Please review the updated files, classify the changelog entries and commit the changes!${COLOR_RESET}"
echo
+
fi
}
diff --git a/setup.py b/setup.py
index de1c405..cad6fc3 100644
--- a/setup.py
+++ b/setup.py
@@ -522,6 +522,7 @@ devel = [
'paramiko',
'pipdeptree',
'pre-commit',
+ 'pygithub',
'pylint~=2.8.1',
'pysftp',
'pytest~=6.0',