You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by vi...@apache.org on 2026/10/13 20:53:57 UTC
(superset) branch master updated: fix(releasing): various release tooling improvements (#45253)
This is an automated email from the ASF dual-hosted git repository.
villebro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 48eb96aa843 fix(releasing): various release tooling improvements (#45253)
48eb96aa843 is described below
commit 48eb96aa843b587405250080af370da02bf93b73
Author: Ville Brofeldt <33...@users.noreply.github.com>
AuthorDate: Tue Oct 13 13:53:50 2026 -0700
fix(releasing): various release tooling improvements (#45253)
---
RELEASING/Dockerfile.from_local_tarball | 13 +-
RELEASING/Dockerfile.from_svn_tarball | 13 +-
RELEASING/README.md | 23 +++-
RELEASING/changelog.py | 220 ++++++++++++++++++++++++--------
RELEASING/requirements.txt | 2 +
5 files changed, 197 insertions(+), 74 deletions(-)
diff --git a/RELEASING/Dockerfile.from_local_tarball b/RELEASING/Dockerfile.from_local_tarball
index e7ee676e12c..bfec1eb8831 100644
--- a/RELEASING/Dockerfile.from_local_tarball
+++ b/RELEASING/Dockerfile.from_local_tarball
@@ -32,13 +32,14 @@ RUN apt-get install -y apt-transport-https apt-utils
RUN apt-get install -y build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium zstd
-# Install nodejs for custom build
-# https://nodejs.org/en/download/package-manager/
+# Install nodejs for custom build, from the same image the main Dockerfile builds on
+COPY --from=node:24-trixie-slim /usr/local/bin/node /usr/local/bin/node
+COPY --from=node:24-trixie-slim /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm
RUN set -eux; \
- curl -sL https://deb.nodesource.com/setup_20.x | bash -; \
- apt-get install -y nodejs; \
- node --version;
-RUN if ! which npm; then apt-get install -y npm; fi
+ ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm; \
+ ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx; \
+ node --version; \
+ npm --version;
RUN mkdir -p /home/superset
RUN chown superset /home/superset
diff --git a/RELEASING/Dockerfile.from_svn_tarball b/RELEASING/Dockerfile.from_svn_tarball
index e858c7adacd..c5fb3101363 100644
--- a/RELEASING/Dockerfile.from_svn_tarball
+++ b/RELEASING/Dockerfile.from_svn_tarball
@@ -32,13 +32,14 @@ RUN apt-get install -y apt-transport-https apt-utils
RUN apt-get install -y subversion build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium zstd
-# Install nodejs for custom build
-# https://nodejs.org/en/download/package-manager/
+# Install nodejs for custom build, from the same image the main Dockerfile builds on
+COPY --from=node:24-trixie-slim /usr/local/bin/node /usr/local/bin/node
+COPY --from=node:24-trixie-slim /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm
RUN set -eux; \
- curl -sL https://deb.nodesource.com/setup_20.x | bash -; \
- apt-get install -y nodejs; \
- node --version;
-RUN if ! which npm; then apt-get install -y npm; fi
+ ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm; \
+ ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx; \
+ node --version; \
+ npm --version;
RUN mkdir -p /home/superset
RUN chown superset /home/superset
diff --git a/RELEASING/README.md b/RELEASING/README.md
index 23f3f116aca..fd92596ea87 100644
--- a/RELEASING/README.md
+++ b/RELEASING/README.md
@@ -223,7 +223,17 @@ Next, update the `CHANGELOG/<version>.md` with all the changes that are included
Make sure the branch has been pushed to `origin` to ensure the changelog generator
can pick up changes since the previous release.
Similar to `cherrytree`, the change log script requires a github token, either as an env var
-(`GITHUB_TOKEN`) or as the parameter `--access_token`.
+(`GITHUB_TOKEN`) or as the parameter `--access_token`. If you use the GitHub CLI, you can
+reuse its token with `export GITHUB_TOKEN=$(gh auth token)`.
+
+Fetching the pull request details from github can take hours for a large release, as the
+script makes two API requests per pull request and has to wait whenever it hits github's
+rate limit. The fetched details are cached in a file (see `--cache_file`), so if a run is
+interrupted, rerunning the same command resumes where it left off. Pull requests that github
+no longer serves, for example ones hidden along with a suspended account, are listed using
+their commit details instead, and the script prints a warning for each so they can be
+checked by hand. Pull requests reverted on the release branch itself with `git revert` are
+left out, and pull requests merged more than once are listed once.
#### Initial release (e.g. 1.5.0)
@@ -232,7 +242,7 @@ the previous release (in the example, the previous release branch is `1.4`, so r
update it accordingly):
```bash
-python changelog.py --previous_version 1.4 --current_version ${SUPERSET_GITHUB_BRANCH} changelog
+python changelog.py --previous_version 1.4 --current_version ${SUPERSET_GITHUB_BRANCH} changelog > ../CHANGELOG/${SUPERSET_VERSION}.md
```
You can get a list of pull requests with labels started with blocking, risk, hold, revert and security by using the parameter `--risk`.
@@ -242,9 +252,10 @@ Example:
python changelog.py --previous_version 0.37 --current_version 0.38 changelog --access_token {GITHUB_TOKEN} --risk
```
-The script will checkout both branches, compare all the PRs, and output the lines that are needed to be added to the
-`CHANGELOG/<version>.md` file in the root of the repo. Remember to also make sure to update the branch id (with the above command
-`1.5` needs to be changed to `1.5.0`)
+The script will checkout both branches, compare all the PRs, and print the complete
+`CHANGELOG/<version>.md` file. The version in its heading is taken from `SUPERSET_VERSION`
+(set by `set_release_env.sh`), or from `--release_version` if given. Add the new file to the
+list in `CHANGELOG.md`.
Then, in `UPDATING.md`, a file that contains a list of notifications around
deprecations and upgrading-related topics,
@@ -257,7 +268,7 @@ To compare the forthcoming patch release with the latest release from the same b
`--previous_version` as the tag of the previous release (in this example `1.5.0`; remember to update accordingly)
```bash
-python changelog.py --previous_version 1.5.0 --current_version ${SUPERSET_GITHUB_BRANCH} changelog
+python changelog.py --previous_version 1.5.0 --current_version ${SUPERSET_GITHUB_BRANCH} changelog > ../CHANGELOG/${SUPERSET_VERSION}.md
```
### Set version number
diff --git a/RELEASING/changelog.py b/RELEASING/changelog.py
index 06c97ffeb27..6d638ccf2dd 100644
--- a/RELEASING/changelog.py
+++ b/RELEASING/changelog.py
@@ -14,9 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import csv as lib_csv
+import json # noqa: TID251 - standalone release script, not part of superset
import os
import re
import sys
+import tempfile
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Any, Optional, Union
@@ -25,7 +27,15 @@ import click
from click.core import Context
try:
- from github import BadCredentialsException, Github, PullRequest, Repository
+ from github import (
+ Auth,
+ BadCredentialsException,
+ Commit,
+ Github,
+ PullRequest,
+ Repository,
+ UnknownObjectException,
+ )
except ModuleNotFoundError:
print("PyGitHub is a required package for this script")
exit(1)
@@ -33,6 +43,27 @@ except ModuleNotFoundError:
SUPERSET_REPO = "apache/superset"
SUPERSET_PULL_REQUEST_TYPES = r"^(fix|feat|chore|refactor|docs|build|ci|/gmi)"
SUPERSET_RISKY_LABELS = r"^(blocking|risk|hold|revert|security vulnerability)"
+# subject of a commit made with `git revert`, which unlike a merged revert PR has no
+# PR number of its own
+REVERT_SUBJECT = re.compile(r'^Revert "(.*)"$')
+CHANGELOG_HEADER = """<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->"""
@dataclass
@@ -47,6 +78,7 @@ class GitLog:
message: str
pr_number: Union[int, None] = None
author_email: str = ""
+ is_revert: bool = False
def __eq__(self, other: object) -> bool:
"""A log entry is considered equal if it has the same PR number"""
@@ -62,7 +94,7 @@ class GitChangeLog:
"""
Helper class to output a list of logs entries on a superset changelog format
- We want to map a git author to a github login, for that we call github's API
+ Each PR is credited to its author's github login, for that we call github's API
"""
def __init__(
@@ -71,29 +103,69 @@ class GitChangeLog:
logs: list[GitLog],
access_token: Optional[str] = None,
risk: Optional[bool] = False,
+ cache_file: Optional[str] = None,
) -> None:
self._version = version
self._logs = logs
- self._pr_logs_with_details: dict[int, dict[str, Any]] = {}
- self._github_login_cache: dict[str, Optional[str]] = {}
- self._github_prs: dict[int, Any] = {}
+ self._cache_file = cache_file
+ self._pr_logs_with_details = self._load_cached_pull_request_details()
self._wait = 10
github_token = access_token or os.environ.get("GITHUB_TOKEN")
- self._github = Github(github_token)
+ self._github = Github(auth=Auth.Token(github_token) if github_token else None)
self._show_risk = risk
self._superset_repo: Repository = None
- def _fetch_github_pr(self, pr_number: int) -> PullRequest:
+ def _load_cached_pull_request_details(self) -> dict[int, dict[str, Any]]:
"""
- Fetches a github PR info
+ Loads the PR details persisted by previous runs, so an interrupted run
+ resumes where it left off instead of fetching everything from github again
+ """
+ details: dict[int, dict[str, Any]] = {}
+ if not self._cache_file or not os.path.exists(self._cache_file):
+ return details
+ truncated = False
+ with open(self._cache_file) as cache:
+ for line in cache:
+ try:
+ detail = json.loads(line)
+ except json.JSONDecodeError:
+ truncated = True
+ continue
+ details[detail["id"]] = detail
+ if truncated:
+ # a run killed mid-write leaves a truncated line behind, so rewrite the
+ # cache without it to keep new records from being appended onto it
+ with open(self._cache_file, "w") as cache:
+ cache.writelines(
+ f"{json.dumps(detail)}\n" for detail in details.values()
+ )
+ print(
+ f"Loaded {len(details)} cached pull requests from {self._cache_file}",
+ file=sys.stderr,
+ )
+ return details
+
+ def _cache_pull_request_details(self, detail: dict[str, Any]) -> None:
+ self._pr_logs_with_details[detail["id"]] = detail
+ if self._cache_file:
+ with open(self._cache_file, "a") as cache:
+ cache.write(f"{json.dumps(detail)}\n")
+
+ def _fetch_github_pr(self, pr_number: int) -> Optional[PullRequest]:
+ """
+ Fetches a github PR info, or None if github no longer has the PR
+ (e.g. it was deleted, or its author's account was suspended)
"""
try:
- github_repo = self._github.get_repo(SUPERSET_REPO)
- self._superset_repo = github_repo
- pull_request = self._github_prs.get(pr_number)
- if not pull_request:
- pull_request = github_repo.get_pull(pr_number)
- self._github_prs[pr_number] = pull_request
+ if self._superset_repo is None:
+ self._superset_repo = self._github.get_repo(SUPERSET_REPO)
+ pull_request = self._superset_repo.get_pull(pr_number)
+ except UnknownObjectException:
+ print(
+ f"\nPR #{pr_number} not found on github, using its commit details",
+ file=sys.stderr,
+ )
+ return None
except BadCredentialsException:
print(
"Bad credentials to github provided"
@@ -103,26 +175,7 @@ class GitChangeLog:
return pull_request
- def _get_github_login(self, git_log: GitLog) -> Optional[str]:
- """
- Tries to fetch a github login (username) from a git author
- """
- author_name = git_log.author
- github_login = self._github_login_cache.get(author_name)
- if github_login:
- return github_login
- if git_log.pr_number:
- pr_info = self._fetch_github_pr(git_log.pr_number)
- if pr_info:
- github_login = pr_info.user.login
- else:
- github_login = author_name
- # set cache
- self._github_login_cache[author_name] = github_login
- return github_login
-
- def _has_commit_migrations(self, git_sha: str) -> bool:
- commit = self._superset_repo.get_commit(sha=git_sha)
+ def _has_commit_migrations(self, commit: Commit) -> bool:
return any(
"superset/migrations/versions/" in file.filename for file in commit.files
)
@@ -135,16 +188,24 @@ class GitChangeLog:
return detail
pr_info = self._fetch_github_pr(pr_number)
- has_migrations = self._has_commit_migrations(git_log.sha)
- title = pr_info.title if pr_info else git_log.message
+ commit = self._superset_repo.get_commit(sha=git_log.sha)
+ has_migrations = self._has_commit_migrations(commit)
+ if pr_info:
+ title, login = pr_info.title, pr_info.user.login
+ else:
+ # github still serves the commit of a PR it no longer serves
+ title = git_log.message.removesuffix(f" (#{pr_number})")
+ login = commit.author.login if commit.author else None
pr_type = re.match(SUPERSET_PULL_REQUEST_TYPES, title)
if pr_type:
pr_type = pr_type.group().strip('"') # type: ignore
- labels = (" | ").join([label.name for label in pr_info.labels])
- is_risky = self._is_risk_pull_request(pr_info.labels)
+ pr_labels = pr_info.labels if pr_info else []
+ labels = (" | ").join([label.name for label in pr_labels])
+ is_risky = self._is_risk_pull_request(pr_labels)
detail = {
"id": pr_number,
+ "login": login,
"has_migrations": has_migrations,
"labels": labels,
"title": title,
@@ -153,7 +214,7 @@ class GitChangeLog:
}
if pr_number:
- self._pr_logs_with_details[pr_number] = detail
+ self._cache_pull_request_details(detail)
return detail
@@ -196,17 +257,14 @@ class GitChangeLog:
def __repr__(self) -> str:
result = f"\n{self._get_changelog_version_head()}\n"
changelog = {
- "Database Migrations": "\n",
- "Features": "\n",
- "Fixes": "\n",
- "Others": "\n",
+ "Database Migrations": "",
+ "Features": "",
+ "Fixes": "",
+ "Others": "",
}
for i, log in enumerate(self._logs):
- github_login = self._get_github_login(log)
pr_info = self._get_pull_request_details(log)
-
- if not github_login:
- github_login = log.author
+ github_login = pr_info["login"] or log.author
if self._show_risk:
if pr_info.get("is_risky"):
@@ -219,14 +277,20 @@ class GitChangeLog:
else:
self._parse_change_log(changelog, pr_info, github_login)
- print(f"\r {i}/{len(self._logs)}", end="", flush=True)
+ print(f"\r {i}/{len(self._logs)}", end="", flush=True, file=sys.stderr)
+ print(file=sys.stderr)
if self._show_risk:
return result
- for key in changelog:
- result += f"**{key}** {changelog[key]}\n"
- return result
+ # the complete CHANGELOG/<version>.md file
+ sections = "\n\n".join(
+ f"**{key}**\n\n{entries}".rstrip() for key, entries in changelog.items()
+ )
+ return (
+ f"{CHANGELOG_HEADER}\n\n## Change Log\n\n"
+ f"{self._get_changelog_version_head()}\n\n{sections}"
+ )
def __iter__(self) -> Iterator[dict[str, Any]]:
for log in self._logs:
@@ -263,8 +327,27 @@ class GitLogs:
def fetch(self) -> None:
self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
+ @property
+ def pull_request_logs(self) -> list[GitLog]:
+ """
+ The latest commit of each PR in effect at this git ref, oldest first. PRs
+ reverted on the branch itself are left out, and PRs merged more than once
+ are listed once
+ """
+ pull_requests: dict[int, GitLog] = {}
+ for log in self._logs:
+ if not log.pr_number:
+ continue
+ pull_requests.pop(log.pr_number, None)
+ if not log.is_revert:
+ pull_requests[log.pr_number] = log
+ return list(pull_requests.values())
+
def diff(self, git_logs: "GitLogs") -> list[GitLog]:
- return [log for log in git_logs.logs if log not in self._logs]
+ pr_numbers = {log.pr_number for log in self.pull_request_logs}
+ return [
+ log for log in git_logs.pull_request_logs if log.pr_number not in pr_numbers
+ ]
def __repr__(self) -> str:
return f"{self._git_ref}, Log count:{len(self._logs)}"
@@ -301,8 +384,13 @@ class GitLogs:
def _parse_log(log_item: str) -> GitLog:
pr_number = None
split_log_item = log_item.split("|")
+ # a revert's subject wraps the reverted commit's, so unwrap it to find the
+ # reverted PR; reverting a revert lands that PR again
+ subject, reverts = split_log_item[4], 0
+ while revert := REVERT_SUBJECT.match(subject):
+ subject, reverts = revert.group(1), reverts + 1
# parse the PR number from the log message
- match = re.match(r".*\(\#(\d*)\)", split_log_item[4])
+ match = re.match(r".*\(\#(\d*)\)", subject)
if match:
pr_number = int(match.group(1))
return GitLog(
@@ -312,6 +400,7 @@ class GitLogs:
time=split_log_item[3],
message=split_log_item[4],
pr_number=pr_number,
+ is_revert=reverts % 2 == 1,
)
@@ -373,19 +462,38 @@ def compare(base_parameters: BaseParameters) -> None:
" if not provided will try to fetch from GITHUB_TOKEN env var",
)
@click.option("--risk", is_flag=True, help="show all pull requests with risky labels")
+@click.option(
+ "--cache_file",
+ default=os.path.join(tempfile.gettempdir(), "superset_changelog_cache.jsonl"),
+ show_default=True,
+ help="File that caches the pull request details fetched from github, so an"
+ " interrupted run resumes where it left off. Delete it to fetch everything again",
+)
+@click.option(
+ "--release_version",
+ envvar="SUPERSET_VERSION",
+ help="The version to name in the changelog heading (e.g. 1.5.0),"
+ " if not provided will try SUPERSET_VERSION env var, then --current_version",
+)
@click.pass_obj
def change_log(
- base_parameters: BaseParameters, csv: str, access_token: str, risk: bool
+ base_parameters: BaseParameters,
+ csv: str,
+ access_token: str,
+ risk: bool,
+ cache_file: str,
+ release_version: Optional[str],
) -> None:
"""Outputs a changelog (by PR)"""
previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs
previous_diff_logs = previous_logs.diff(current_logs)
logs = GitChangeLog(
- current_logs.git_ref,
+ release_version or current_logs.git_ref,
previous_diff_logs[::-1],
access_token=access_token,
risk=risk,
+ cache_file=cache_file,
)
if csv:
with open(csv, "w") as csv_file:
@@ -402,7 +510,7 @@ def change_log(
for log in logs:
writer.writerow(log)
else:
- print("Fetching github usernames, this may take a while:")
+ print("Fetching github usernames, this may take a while:", file=sys.stderr)
print(logs)
diff --git a/RELEASING/requirements.txt b/RELEASING/requirements.txt
index bd3586e04c0..4d80d04bcb7 100644
--- a/RELEASING/requirements.txt
+++ b/RELEASING/requirements.txt
@@ -17,3 +17,5 @@
cherrytree
jinja2
+# changelog.py needs github.Auth and the default rate-limit retry, both in 2.1.1+
+PyGithub>=2.1.1