You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2018/09/07 11:47:33 UTC
[arrow] branch master updated: ARROW-2948: [Packaging] Generate
changelog with crossbow
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 4007aff ARROW-2948: [Packaging] Generate changelog with crossbow
4007aff is described below
commit 4007aff961d96a6f90364d3b89e1f5e7e0ebed89
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Fri Sep 7 13:47:17 2018 +0200
ARROW-2948: [Packaging] Generate changelog with crossbow
Trying to centralize all the release related scripts. Merge after the current release.
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #2348 from kszucs/ARROW-2948 and squashes the following commits:
014c421a <Krisztián Szűcs> remove accidentally committed tests.yml
f6783643 <Krisztián Szűcs> naive markdown escaping
eecf8299 <Krisztián Szűcs> add toolz to the readme
c6ff83a3 <Krisztián Szűcs> changelog command
---
dev/tasks/README.md | 4 +-
dev/tasks/crossbow.py | 139 +++++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 139 insertions(+), 4 deletions(-)
diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index 666f326..c5975f5 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -97,12 +97,12 @@ submission. The tasks are defined in `tasks.yml`
8. Install the python dependencies for the script:
```bash
- conda install -y jinja2 pygit2 click ruamel.yaml setuptools_scm github3.py python-gnupg
+ conda install -y jinja2 pygit2 click ruamel.yaml setuptools_scm github3.py python-gnupg toolz jira
```
```bash
# pygit2 requires libgit2: http://www.pygit2.org/install.html
- pip install jinja2 pygit2 click ruamel.yaml setuptools_scm github3.py python-gnupg
+ pip install jinja2 pygit2 click ruamel.yaml setuptools_scm github3.py python-gnupg toolz jira
```
9. Try running it:
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index fae5c0a..324b997 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -17,19 +17,22 @@
# specific language governing permissions and limitations
# under the License.
-import hashlib
import os
import re
import sys
import time
import click
+import hashlib
+import gnupg
+import toolz
import pygit2
import github3
-import gnupg
+import jira.client
from io import StringIO
from pathlib import Path
from textwrap import dedent
+from datetime import datetime
from jinja2 import Template, StrictUndefined
from setuptools_scm import get_version
from ruamel.yaml import YAML
@@ -38,6 +41,107 @@ from ruamel.yaml import YAML
CWD = Path(__file__).parent.absolute()
+NEW_FEATURE = 'New Features and Improvements'
+BUGFIX = 'Bug Fixes'
+
+
+def md(template, *args, **kwargs):
+ """Wraps string.format with naive markdown escaping"""
+ def escape(s):
+ for char in ('*', '#', '_', '~', '`', '>'):
+ s = s.replace(char, '\\' + char)
+ return s
+ return template.format(*map(escape, args), **toolz.valmap(escape, kwargs))
+
+
+class JiraChangelog:
+
+ def __init__(self, version, username, password,
+ server='https://issues.apache.org/jira'):
+ self.server = server
+ # clean version to the first numbers
+ self.version = '.'.join(version.split('.')[:3])
+ query = ("project=ARROW "
+ "AND fixVersion='{0}' "
+ "AND status = Resolved "
+ "AND resolution in (Fixed, Done) "
+ "ORDER BY issuetype DESC").format(self.version)
+ self.client = jira.client.JIRA({'server': server},
+ basic_auth=(username, password))
+ self.issues = self.client.search_issues(query, maxResults=9999)
+
+ def format_markdown(self):
+ out = StringIO()
+
+ issues_by_type = toolz.groupby(lambda i: i.fields.issuetype.name,
+ self.issues)
+ for typename, issues in sorted(issues_by_type.items()):
+ issues.sort(key=lambda x: x.key)
+
+ out.write(md('## {}\n\n', typename))
+ for issue in issues:
+ out.write(md('* {} - {}\n', issue.key, issue.fields.summary))
+ out.write('\n')
+
+ return out.getvalue()
+
+ def format_website(self):
+ # jira category => website category mapping
+ categories = {
+ 'New Feature': 'feature',
+ 'Improvement': 'feature',
+ 'Wish': 'feature',
+ 'Task': 'feature',
+ 'Test': 'bug',
+ 'Bug': 'bug',
+ 'Sub-task': 'feature'
+ }
+ titles = {
+ 'feature': 'New Features and Improvements',
+ 'bugfix': 'Bug Fixes'
+ }
+
+ issues_by_category = toolz.groupby(
+ lambda issue: categories[issue.fields.issuetype.name],
+ self.issues
+ )
+
+ out = StringIO()
+
+ for category in ('feature', 'bug'):
+ title = titles[category]
+ issues = issues_by_category[category]
+ issues.sort(key=lambda x: x.key)
+
+ out.write(md('## {}\n\n', title))
+ for issue in issues:
+ link = md('[{0}]({1}/browse/{0})', issue.key, self.server)
+ out.write(md('* {} - {}\n', link, issue.fields.summary))
+ out.write('\n')
+
+ return out.getvalue()
+
+ def render(self, old_changelog, website=False):
+ old_changelog = old_changelog.splitlines()
+ if website:
+ new_changelog = self.format_website()
+ else:
+ new_changelog = self.format_markdown()
+
+ out = StringIO()
+
+ # Apache license header
+ out.write('\n'.join(old_changelog[:18]))
+
+ # Newly generated changelog
+ today = datetime.today().strftime('%d %B %Y')
+ out.write(md('\n\n# Apache Arrow {} ({})\n\n', self.version, today))
+ out.write(new_changelog)
+ out.write('\n'.join(old_changelog[19:]))
+
+ return out.getvalue().strip()
+
+
class GitRemoteCallbacks(pygit2.RemoteCallbacks):
def __init__(self, token):
@@ -414,6 +518,37 @@ def crossbow(ctx, github_token, arrow_path, queue_path):
ctx.obj['queue'] = Queue(Path(queue_path), github_token=github_token)
+@crossbow.command()
+@click.option('--changelog-path', '-c', type=click.Path(exists=True),
+ default=DEFAULT_ARROW_PATH / 'CHANGELOG.md',
+ help='Path of changelog to update')
+@click.option('--arrow-version', '-v', default=None,
+ help='Set target version explicitly')
+@click.option('--is-website', '-w', default=False)
+@click.option('--jira-username', '-u', default=None, help='JIRA username')
+@click.option('--jira-password', '-P', default=None, help='JIRA password')
+@click.option('--dry-run/--write', default=False,
+ help='Just display the new changelog, don\'t write it')
+@click.pass_context
+def changelog(ctx, changelog_path, arrow_version, is_website, jira_username,
+ jira_password, dry_run):
+ changelog_path = Path(changelog_path)
+ target = Target.from_repo(ctx.obj['arrow'])
+ version = arrow_version or target.version
+
+ changelog = JiraChangelog(version, username=jira_username,
+ password=jira_password)
+ new_content = changelog.render(changelog_path.read_text(),
+ website=is_website)
+
+ if dry_run:
+ click.echo(new_content)
+ else:
+ changelog_path.write_text(new_content)
+ click.echo('New changelog successfully generated, see git diff for the'
+ 'changes')
+
+
def load_tasks_from_config(config_path, task_names, group_names):
with Path(config_path).open() as fp:
config = yaml.load(fp)