You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/10/12 01:01:12 UTC
[dolphinscheduler] branch dev updated: [dev] Easier release: cherry-pick, changelog, contributor (#11478)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 2525545a41 [dev] Easier release: cherry-pick, changelog, contributor (#11478)
2525545a41 is described below
commit 2525545a41fd13f1e844a814b4883cb03a75f24d
Author: Jiajie Zhong <zh...@gmail.com>
AuthorDate: Wed Oct 12 09:00:59 2022 +0800
[dev] Easier release: cherry-pick, changelog, contributor (#11478)
Add script for easier release including cherry pick,
generating changelog and contributor list
* Auto cherry-pick: `python release.py cherry-pick`
* Generate changelog: `python release.py changelog`
* Generate contributor: `python release.py contributor`
close: #11289
related: #12222
---
tools/release/README.md | 38 +++++++++
tools/release/github/__init__.py | 16 ++++
tools/release/github/changelog.py | 151 +++++++++++++++++++++++++++++++++++
tools/release/github/git.py | 70 ++++++++++++++++
tools/release/github/pull_request.py | 64 +++++++++++++++
tools/release/github/resp_get.py | 67 ++++++++++++++++
tools/release/github/user.py | 43 ++++++++++
tools/release/release.py | 106 ++++++++++++++++++++++++
tools/release/requirements.txt | 19 +++++
9 files changed, 574 insertions(+)
diff --git a/tools/release/README.md b/tools/release/README.md
new file mode 100644
index 0000000000..b4596f0f88
--- /dev/null
+++ b/tools/release/README.md
@@ -0,0 +1,38 @@
+# Tools Release
+
+A tools for convenient release DolphinScheduler.
+
+## Prepare
+
+* python: python 3.6 or above
+* pip: latest version of pip is better
+
+To install dependence, you should run command
+
+```shell
+python -m pip install -r requirements.txt
+```
+
+## Usage
+
+### Export Environment Variable
+
+You can create new token in [create token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token),
+it is only need all permission under `repo`
+
+```shell
+export GH_ACCESS_TOKEN="<YOUR-GITHUB-TOKEN-WITH-REPO-ACCESS>"
+export GH_REPO_MILESTONE="<YOUR-MILESTONE>"
+```
+
+### Help
+
+```shell
+python release.py -h
+```
+
+### Action
+
+* Auto cherry-pick: `python release.py cherry-pick`, will cause error when your default branch is not up-to-date, or cherry-pick with conflict. But if you fix you can directly re-run this command, it will continue the pick
+* Generate changelog: `python release.py changelog`
+* Generate contributor: `python release.py contributor`
diff --git a/tools/release/github/__init__.py b/tools/release/github/__init__.py
new file mode 100644
index 0000000000..13a83393a9
--- /dev/null
+++ b/tools/release/github/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tools/release/github/changelog.py b/tools/release/github/changelog.py
new file mode 100644
index 0000000000..517d6b0b49
--- /dev/null
+++ b/tools/release/github/changelog.py
@@ -0,0 +1,151 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils for release changelog."""
+
+from typing import Dict, List
+
+
+class Changelog:
+ """Generate changelog according specific pull requests list.
+
+ Each pull requests will only once in final result. If pull requests have more than one label we need,
+ will classify to high priority label type, currently priority is
+ `feature > bug > improvement > document > chore`. pr will into feature section if it with both `feature`,
+ `improvement`, `document` label.
+
+ :param prs: pull requests list.
+ """
+
+ key_number = "number"
+ key_labels = "labels"
+ key_name = "name"
+
+ label_feature = "feature"
+ label_bug = "bug"
+ label_improvement = "improvement"
+ label_document = "document"
+ label_chore = "chore"
+
+ changelog_prefix = "\n\n<details><summary>Click to expand</summary>\n\n"
+ changelog_suffix = "\n\n</details>\n"
+
+ def __init__(self, prs: List[Dict]):
+ self.prs = prs
+ self.features = []
+ self.bugfixs = []
+ self.improvements = []
+ self.documents = []
+ self.chores = []
+
+ def generate(self) -> str:
+ """Generate changelog."""
+ self.classify()
+ final = []
+ if self.features:
+ detail = f"## Feature{self.changelog_prefix}{self._convert(self.features)}{self.changelog_suffix}"
+ final.append(detail)
+ if self.improvements:
+ detail = (
+ f"## Improvement{self.changelog_prefix}"
+ f"{self._convert(self.improvements)}{self.changelog_suffix}"
+ )
+ final.append(detail)
+ if self.bugfixs:
+ detail = f"## Bugfix{self.changelog_prefix}{self._convert(self.bugfixs)}{self.changelog_suffix}"
+ final.append(detail)
+ if self.documents:
+ detail = (
+ f"## Document{self.changelog_prefix}"
+ f"{self._convert(self.documents)}{self.changelog_suffix}"
+ )
+ final.append(detail)
+ if self.chores:
+ detail = f"## Chore{self.changelog_prefix}{self._convert(self.chores)}{self.changelog_suffix}"
+ final.append(detail)
+ return "\n".join(final)
+
+ @staticmethod
+ def _convert(prs: List[Dict]) -> str:
+ """Convert pull requests into changelog item text."""
+ return "\n".join(
+ [f"- {pr['title']} (#{pr['number']}) @{pr['user']['login']}" for pr in prs]
+ )
+
+ def classify(self) -> None:
+ """Classify pull requests different kinds of section in changelog.
+
+ Each pull requests only belongs to one single classification.
+ """
+ for pr in self.prs:
+ if self.key_labels not in pr:
+ raise KeyError("PR %s do not have labels", pr[self.key_number])
+ if self._is_feature(pr):
+ self.features.append(pr)
+ elif self._is_bugfix(pr):
+ self.bugfixs.append(pr)
+ elif self._is_improvement(pr):
+ self.improvements.append(pr)
+ elif self._is_document(pr):
+ self.documents.append(pr)
+ elif self._is_chore(pr):
+ self.chores.append(pr)
+ else:
+ raise KeyError(
+ "There must at least one of labels `feature|bug|improvement|document|chore`"
+ "but it do not, pr: %s",
+ pr["html_url"],
+ )
+
+ def _is_feature(self, pr: Dict) -> bool:
+ """Belong to feature pull requests."""
+ return any(
+ [
+ label[self.key_name] == self.label_feature
+ for label in pr[self.key_labels]
+ ]
+ )
+
+ def _is_bugfix(self, pr: Dict) -> bool:
+ """Belong to bugfix pull requests."""
+ return any(
+ [label[self.key_name] == self.label_bug for label in pr[self.key_labels]]
+ )
+
+ def _is_improvement(self, pr: Dict) -> bool:
+ """Belong to improvement pull requests."""
+ return any(
+ [
+ label[self.key_name] == self.label_improvement
+ for label in pr[self.key_labels]
+ ]
+ )
+
+ def _is_document(self, pr: Dict) -> bool:
+ """Belong to document pull requests."""
+ return any(
+ [
+ label[self.key_name] == self.label_document
+ for label in pr[self.key_labels]
+ ]
+ )
+
+ def _is_chore(self, pr: Dict) -> bool:
+ """Belong to chore pull requests."""
+ return any(
+ [label[self.key_name] == self.label_chore for label in pr[self.key_labels]]
+ )
diff --git a/tools/release/github/git.py b/tools/release/github/git.py
new file mode 100644
index 0000000000..66ef6596e1
--- /dev/null
+++ b/tools/release/github/git.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils for git operations."""
+
+from pathlib import Path
+from typing import Dict, Optional
+
+from git import Repo
+
+git_dir_path: Path = Path(__file__).parent.parent.parent.parent.joinpath(".git")
+
+
+class Git:
+ """Operator to handle git object.
+
+ :param path: git repository path
+ :param branch: branch you want to query
+ """
+
+ def __init__(
+ self, path: Optional[str] = git_dir_path, branch: Optional[str] = None
+ ):
+ self.path = path
+ self.branch = branch
+
+ @property
+ def repo(self) -> Repo:
+ """Get git repo object."""
+ return Repo(self.path)
+
+ def has_commit_current(self, sha: str) -> bool:
+ """Whether SHA in current branch."""
+ branches = self.repo.git.branch("--contains", sha)
+ return f"* {self.repo.active_branch.name}" in branches
+
+ def has_commit_global(self, sha: str) -> bool:
+ """Whether SHA in all branches."""
+ try:
+ self.repo.commit(sha)
+ return True
+ except ValueError:
+ return False
+
+ def cherry_pick_pr(self, pr: Dict) -> None:
+ """Run command `git cherry-pick -x <SHA>`."""
+ sha = pr["merge_commit_sha"]
+ if not self.has_commit_global(sha):
+ raise RuntimeError(
+ "Cherry-pick SHA %s error because SHA not exists,"
+ "please make sure you local default branch is up-to-date",
+ sha,
+ )
+ if self.has_commit_current(sha):
+ print("SHA %s already in current active branch, skip it.", sha)
+ self.repo.git.cherry_pick("-x", sha)
diff --git a/tools/release/github/pull_request.py b/tools/release/github/pull_request.py
new file mode 100644
index 0000000000..74dcd80b32
--- /dev/null
+++ b/tools/release/github/pull_request.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional, List, Dict
+from github.resp_get import RespGet
+
+
+class PullRequest:
+ """Pull request to filter the by specific condition.
+
+ :param token: token to request GitHub API entrypoint.
+ :param repo: GitHub repository identify, use `user/repo` or `org/repo`.
+ """
+ url_search = "https://api.github.com/search/issues"
+ url_pr = "https://api.github.com/repos/{}/pulls/{}"
+
+ def __init__(self, token: str, repo: Optional[str] = "apache/dolphinscheduler"):
+ self.token = token
+ self.repo = repo
+ self.headers = {
+ "Accept": "application/vnd.github+json",
+ "Authorization": f"token {token}",
+ }
+
+ def get_merged_detail(self, number: str) -> Dict:
+ """Get all merged pull requests detail by pr number.
+
+ :param number: pull requests number you want to get detail.
+ """
+ return RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single()
+
+ def get_merged_detail_by_milestone(self, milestone: str) -> List[Dict]:
+ """Get all merged requests pull request detail by specific milestone.
+
+ :param milestone: query by specific milestone.
+ """
+ detail = []
+ numbers = {pr.get("number") for pr in self.search_merged_by_milestone(milestone)}
+ for number in numbers:
+ pr_dict = RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single()
+ detail.append(pr_dict)
+ return detail
+
+ def search_merged_by_milestone(self, milestone: str) -> List[Dict]:
+ """Get all merged requests pull request by specific milestone.
+
+ :param milestone: query by specific milestone.
+ """
+ params = {"q": f"repo:{self.repo} is:pr is:merged milestone:{milestone}"}
+ return RespGet(url=self.url_search, headers=self.headers, param=params).get_total()
diff --git a/tools/release/github/resp_get.py b/tools/release/github/resp_get.py
new file mode 100644
index 0000000000..a249e4c758
--- /dev/null
+++ b/tools/release/github/resp_get.py
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils get HTTP response."""
+
+import copy
+import json
+from typing import Dict, List, Optional
+
+import requests
+
+
+class RespGet:
+ """Get response from GitHub restful API.
+
+ :param url: URL to requests GET method.
+ :param headers: headers for HTTP requests.
+ :param param: param for HTTP requests.
+ """
+
+ def __init__(self, url: str, headers: dict, param: Optional[dict] = None):
+ self.url = url
+ self.headers = headers
+ self.param = param
+
+ @staticmethod
+ def get(url: str, headers: dict, params: Optional[dict] = None) -> Dict:
+ """Get single response dict from HTTP requests by given condition."""
+ resp = requests.get(url=url, headers=headers, params=params)
+ if not resp.ok:
+ raise ValueError("Requests error with", resp.reason)
+ return json.loads(resp.content)
+
+ def get_single(self) -> Dict:
+ """Get single response dict from HTTP requests by given condition."""
+ return self.get(url=self.url, headers=self.headers, params=self.param)
+
+ def get_total(self) -> List[Dict]:
+ """Get all response dict from HTTP requests by given condition.
+
+ Will change page number until no data return.
+ """
+ total = []
+ curr_param = copy.deepcopy(self.param)
+ while True:
+ curr_param["page"] = curr_param.setdefault("page", 0) + 1
+ content_dict = self.get(
+ url=self.url, headers=self.headers, params=curr_param
+ )
+ data = content_dict.get("items")
+ if not data:
+ return total
+ total.extend(data)
diff --git a/tools/release/github/user.py b/tools/release/github/user.py
new file mode 100644
index 0000000000..152169d2ef
--- /dev/null
+++ b/tools/release/github/user.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils for user."""
+
+from typing import Dict, List, Set
+
+
+class User:
+ """Get users according specific pull requests list.
+
+ :param prs: pull requests list.
+ """
+
+ def __init__(self, prs: List[Dict]):
+ self.prs = prs
+
+ def contribution_num(self) -> Dict:
+ """Get unique contributor with name and commit number."""
+ res = dict()
+ for pr in self.prs:
+ user_id = pr["user"]["login"]
+ res[user_id] = res.setdefault(user_id, 0) + 1
+ return res
+
+ def contributors(self) -> Set[str]:
+ """Get unique contributor with name."""
+ cn = self.contribution_num()
+ return {contributor for contributor in cn}
diff --git a/tools/release/release.py b/tools/release/release.py
new file mode 100644
index 0000000000..122e57caaf
--- /dev/null
+++ b/tools/release/release.py
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Main function for releasing."""
+
+import argparse
+import os
+
+from github.changelog import Changelog
+from github.git import Git
+from github.pull_request import PullRequest
+from github.user import User
+
+
+def get_changelog(access_token: str, milestone: str) -> str:
+ """Get changelog in specific milestone from GitHub Restful API."""
+ pr = PullRequest(token=access_token)
+ pr_merged = pr.search_merged_by_milestone(milestone)
+ # Sort according to merged time ascending
+ pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"])
+
+ changelog = Changelog(pr_merged_sort)
+ changelog_text = changelog.generate()
+ return changelog_text
+
+
+def get_contributor(access_token: str, milestone: str) -> str:
+ """Get contributor in specific milestone from GitHub Restful API."""
+ pr = PullRequest(token=access_token)
+ pr_merged = pr.search_merged_by_milestone(milestone)
+
+ users = User(prs=pr_merged)
+ contributor = users.contributors()
+ # Sort according alphabetical
+ return ", ".join(sorted(contributor))
+
+
+def auto_cherry_pick(access_token: str, milestone: str) -> None:
+ """Do git cherry-pick in specific milestone, require update dev branch."""
+ pr = PullRequest(token=access_token)
+ pr_merged = pr.search_merged_by_milestone(milestone)
+ # Sort according to merged time ascending
+ pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"])
+
+ for p in pr_merged_sort:
+ pr_detail = pr.get_merged_detail(p["number"])
+ print(f"git cherry-pick -x {pr_detail['merge_commit_sha']}")
+ Git().cherry_pick_pr(pr_detail)
+
+
+def build_argparse() -> argparse.ArgumentParser:
+ """Build argparse.ArgumentParser with specific configuration."""
+ parser = argparse.ArgumentParser(prog="release")
+
+ subparsers = parser.add_subparsers(
+ title="subcommands",
+ dest="subcommand",
+ help="Choose one of the subcommand you want to run.",
+ )
+ parser_check = subparsers.add_parser(
+ "changelog", help="Generate changelog from specific milestone."
+ )
+ parser_check.set_defaults(func=get_changelog)
+
+ parser_prune = subparsers.add_parser(
+ "contributor", help="List all contributors from specific milestone."
+ )
+ parser_prune.set_defaults(func=get_contributor)
+
+ parser_prune = subparsers.add_parser(
+ "cherry-pick",
+ help="Auto cherry pick pr to current branch from specific milestone.",
+ )
+ parser_prune.set_defaults(func=auto_cherry_pick)
+
+ return parser
+
+
+if __name__ == "__main__":
+ arg_parser = build_argparse()
+ # args = arg_parser.parse_args(["cherry-pick"])
+ args = arg_parser.parse_args()
+
+ ENV_ACCESS_TOKEN = os.environ.get("GH_ACCESS_TOKEN", None)
+ ENV_MILESTONE = os.environ.get("GH_REPO_MILESTONE", None)
+
+ if ENV_ACCESS_TOKEN is None or ENV_MILESTONE is None:
+ raise RuntimeError(
+ "Environment variable `GH_ACCESS_TOKEN` and `GH_REPO_MILESTONE` must provider"
+ )
+
+ print(args.func(ENV_ACCESS_TOKEN, ENV_MILESTONE))
diff --git a/tools/release/requirements.txt b/tools/release/requirements.txt
new file mode 100644
index 0000000000..2cb12bfa09
--- /dev/null
+++ b/tools/release/requirements.txt
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+requests~=2.28
+GitPython~=3.1