You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/01/24 07:03:27 UTC
[7/9] impala git commit: IMPALA-6410: Tool to cherrypick changes
across branches.
IMPALA-6410: Tool to cherrypick changes across branches.
This script compares two branches and optionally cherry-picks changes
across. It uses the Gerrit Change-Id as the key, and it supports a
configuration file and a string to ignore commits.
Change-Id: I6120ec2d6e914a1e5fda568178b32aafda8722a9
Reviewed-on: http://gerrit.cloudera.org:8080/9045
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/674d29d7
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/674d29d7
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/674d29d7
Branch: refs/heads/master
Commit: 674d29d71d9c75da4509544ea609bb1a62e6b0fb
Parents: be0f4f9
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Wed Jan 17 09:42:54 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Jan 24 03:34:52 2018 +0000
----------------------------------------------------------------------
bin/compare_branches.py | 273 ++++++++++++++++++++++++++++++++++++++++++
bin/ignored_commits.json | 1 +
2 files changed, 274 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/674d29d7/bin/compare_branches.py
----------------------------------------------------------------------
diff --git a/bin/compare_branches.py b/bin/compare_branches.py
new file mode 100755
index 0000000..6a81901
--- /dev/null
+++ b/bin/compare_branches.py
@@ -0,0 +1,273 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HELP = '''
+Compares two specified branches, using the Gerrit Change-Id as the
+primary identifier. Ignored commits can be added via a JSON
+configuration file or with a special string in the commit message.
+Changes can be cherrypicked with the --cherry_pick argument.
+
+This script can be used to keep two development branches
+(by default, "master" and "2.x", in sync). It is equivalent
+to cherry-picking commits one by one, but automates identifying
+the commits to cherry-pick. Unlike "git cherry", it uses
+the Gerrit Change-Id identifier in the commit message
+as a key.
+
+The ignored_commits.json configuration file is of the following
+form. Note that commits are the full 20-byte git hashes.
+
+[
+ {
+ "source": "master",
+ "target": "2.x",
+ "commits": [
+ { "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "comment": "...",
+ { "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "comment": "..."}
+ ]
+ }
+]
+
+The --target_remote_name is optional. If not specified, the target remote is set to
+the value of the --source_remote_name. Debug logging to stderr can be enabled with
+--verbose.
+
+Example:
+
+$bin/compare_branches.py --source_branch master --target_branch 2.x
+--------------------------------------------------------------------------------
+Commits in asf-gerrit/master but not in asf-gerrit/2.x:
+--------------------------------------------------------------------------------
+35a3e186d61b8f365b0f7d1127be311758437e16 IMPALA-5478: Run TPCDS queries with decimal_v2 enabled (Thu Jan 18 03:28:51 2018 +0000) - Taras Bobrovytsky
+d9b6fd073055b436c7404d49454dc215b2c7a369 IMPALA-6386: Invalidate metadata at table level for dataload (Wed Jan 17 22:52:58 2018 +0000) - Joe McDonnell
+dcc7be0ed483b332dac22d6596f56ff2a6cfdaa3 IMPALA-4315: Allow USE and SHOW TABLES if the user has only column privileges (Wed Jan 17 22:40:13 2018 +0000) - Csaba Ringhofer
+b6e43133e671773d2757612f72cfcdb0ff303226 IMPALA-6399: Increase timeout in test_observability to reduce flakiness (Wed Jan 17 22:31:33 2018 +0000) - Lars Volker
+--------------------------------------------------------------------------------
+Jira keys referenced (Note: not all commit messages will reference a jira key):
+IMPALA-5478,IMPALA-6386,IMPALA-4315,IMPALA-6399
+--------------------------------------------------------------------------------
+'''
+
+import argparse
+import json
+import logging
+import os
+import re
+import sh
+import sys
+
+from collections import defaultdict
+try:
+ from collections import OrderedDict
+except ImportError:
+ from ordereddict import OrderedDict
+from pprint import pformat
+
+def create_parser():
+ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
+ argparse.RawDescriptionHelpFormatter):
+ """
+ Mix-in to leave the description alone, but show
+ defaults.
+ """
+ pass
+
+ parser = argparse.ArgumentParser(
+ formatter_class=CustomFormatter,
+ description=HELP)
+
+ parser.add_argument('--cherry_pick', action='store_true', default=False,
+ help='Cherry-pick mismatched commits to current branch. This ' +
+ 'must match (in the hash sense) the target branch.')
+ parser.add_argument('--source_branch', default='master')
+ parser.add_argument('--target_branch', default='2.x')
+ parser.add_argument('--source_remote_name', default='asf-gerrit',
+ help='Name of the source git remote. If set to empty string, ' +
+ 'this remote is not fetched and branch names are used ' +
+ ' as is; otherwise, the source ref is remote/branch.')
+ parser.add_argument('--target_remote_name', default=None,
+ help='Name of the target git remote; defaults to source remote. ' +
+ 'Empty strings are handled the same way as --source_remote_name.')
+ default_ignored_commits_path = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), 'ignored_commits.json')
+ parser.add_argument('--ignored_commits_file', default=default_ignored_commits_path,
+ help='JSON File that contains ignored commits as specified in the help')
+ parser.add_argument('--skip_commits_matching', default="Cherry-picks: not for {branch}",
+ help='String in commit messages that causes the commit to be ignored. ' +
+ ' {branch} is replaced with target branch; the search is case-insensitive')
+ parser.add_argument('--verbose', '-v', action='store_true', default=False,
+ help='Turn on DEBUG and INFO logging')
+ return parser
+
+def read_ignored_commits(ignored_commits_file):
+ '''Returns a dictionary containing commits that should be ignored.
+
+ ignored_commits_file is a path to a JSON file with schema
+ specified at the top of this file.
+
+ The return structure has dictionary keys are a tuple containing
+ (source_branch, target_branch) and values are a set of git hashes.
+ '''
+ ignored_commits = defaultdict(set)
+ with open(ignored_commits_file) as f:
+ json_data = json.load(f)
+ for result_dict in json_data:
+ logging.debug("Parsing result_dict: {0}".format(result_dict))
+ ignored_commits[(result_dict['source'], result_dict['target'])] =\
+ set([ commit["hash"] for commit in result_dict['commits'] ])
+ return ignored_commits
+
+def build_commit_map(branch, merge_base):
+ '''Creates a map from change id to (hash, subject, author, date, body).'''
+ # Disable git pager in order for the sh.git.log command to work
+ os.environ['GIT_PAGER'] = ''
+
+ fields = ['%H', '%s', '%an', '%cd', '%b']
+ pretty_format = '\x1f'.join(fields) + '\x1e'
+ result = OrderedDict()
+ for line in sh.git.log(
+ branch, "^" + merge_base, pretty=pretty_format, color='never').split('\x1e'):
+ if line == "":
+ # if no changes are identified by the git log, we get an empty string
+ continue
+ if line == "\n":
+ # git log adds a newline to the end; we can skip it
+ continue
+ commit_hash, subject, author, date, body = [t.strip() for t in line.split('\x1f')]
+ change_id_matches = re.findall('Change-Id: (.*)', body)
+ if change_id_matches:
+ if len(change_id_matches) > 1:
+ logging.warning("Commit %s contains multiple change ids; using first one.",
+ commit_hash)
+ change_id = change_id_matches[0]
+ result[change_id] = (commit_hash, subject, author, date, body)
+ else:
+ logging.warning('Commit {0} ({1}...) has no Change-Id.'.format(
+ commit_hash, subject[:40]))
+ logging.debug("Commit map for branch %s has size %d.", branch, len(result))
+ return result
+
+def cherrypick(cherry_pick_hashes, full_target_branch_name):
+ """Cherrypicks the given commits.
+
+ Also, asserts that full_target_branch_name matches the current HEAD.
+
+ cherry_pick_hashes is a list of git hashes, in the order to
+ be cherry-picked.
+
+ Note that this function does not push to the remote.
+ """
+ print "Cherrypicking %d changes." % (len(cherry_pick_hashes),)
+
+ if len(cherry_pick_hashes) == 0:
+ return
+
+ # Cherrypicking only makes sense if we're on the equivalent of the target branch.
+ head_sha = sh.git('rev-parse', 'HEAD').strip()
+ target_branch_sha = sh.git('rev-parse', full_target_branch_name).strip()
+ if head_sha != target_branch_sha:
+ print "Cannot cherrypick because %s (%s) and HEAD (%s) are divergent." % (
+ full_target_branch_name, target_branch_sha, head_sha)
+ sys.exit(1)
+
+ cherry_pick_hashes.reverse()
+ for cherry_pick_hash in cherry_pick_hashes:
+ sh.git('cherry-pick', '--keep-redundant-commits', cherry_pick_hash)
+
+
+def main():
+ parser = create_parser()
+ options = parser.parse_args()
+
+ log_level = logging.WARNING
+ if options.verbose:
+ log_level = logging.DEBUG
+ logging.basicConfig(level=log_level,
+ format='%(asctime)s %(threadName)s %(levelname)s: %(message)s')
+
+ if options.target_remote_name is None:
+ options.target_remote_name = options.source_remote_name
+
+ # Ensure all branches are up to date, unless remotes are disabled
+ # by specifying them with an empty string.
+ if options.source_remote_name != "":
+ sh.git.fetch(options.source_remote_name)
+ full_source_branch_name = options.source_remote_name + '/' + options.source_branch
+ else:
+ full_source_branch_name = options.source_branch
+ if options.target_remote_name != "":
+ if options.source_remote_name != options.target_remote_name:
+ sh.git.fetch(options.target_remote_name)
+ full_target_branch_name = options.target_remote_name + '/' + options.target_branch
+ else:
+ full_target_branch_name = options.target_branch
+
+ merge_base = sh.git("merge-base",
+ full_source_branch_name, full_target_branch_name).strip()
+ source_commits = build_commit_map(full_source_branch_name, merge_base)
+ target_commits = build_commit_map(full_target_branch_name, merge_base)
+
+ ignored_commits = read_ignored_commits(options.ignored_commits_file)
+ logging.debug("ignored commits from {0}:\n{1}"
+ .format(options.ignored_commits_file, pformat(ignored_commits)))
+ commits_ignored = [] # Track commits actually ignored for debug logging
+
+ cherry_pick_hashes = []
+ print '-' * 80
+ print 'Commits in {0} but not in {1}:'.format(
+ full_source_branch_name, full_target_branch_name)
+ print '-' * 80
+ jira_keys = []
+ jira_key_pat = re.compile(r'(IMPALA-\d+)')
+ skip_commits_matching = options.skip_commits_matching.replace(
+ "{branch}", options.target_branch)
+ for change_id, (commit_hash, msg, author, date, body) in source_commits.iteritems():
+ change_in_target = change_id in target_commits
+ ignore_by_config = commit_hash in ignored_commits[
+ (options.source_branch, options.target_branch)]
+ ignore_by_commit_message = skip_commits_matching.lower() in msg.lower() \
+ or skip_commits_matching.lower() in body.lower()
+ # This conditional block just for debug logging of ignored commits
+ if ignore_by_config or ignore_by_commit_message:
+ if change_in_target:
+ logging.debug("Not ignoring commit because change is already in target: {0}"
+ .format(commit_hash))
+ else:
+ if ignore_by_commit_message:
+ logging.debug("Ignoring commit {0} by commit message.".format(commit_hash))
+ else:
+ logging.debug("Ignoring commit {0} by config file.".format(commit_hash))
+ commits_ignored.append(commit_hash)
+ else:
+ logging.debug("NOT ignoring commit {0} since not in ignored commits ({1},{2})"
+ .format(commit_hash, options.source_branch, options.target_branch))
+ if not change_in_target and not ignore_by_config and not ignore_by_commit_message:
+ print u'{0} {1} ({2}) - {3}'.format(commit_hash, msg, date, author)
+ cherry_pick_hashes.append(commit_hash)
+ jira_keys += jira_key_pat.findall(msg)
+
+ print '-' * 80
+
+ print "Jira keys referenced (Note: not all commit messages will reference a jira key):"
+ print ','.join(jira_keys)
+ print '-' * 80
+
+ logging.debug("Commits actually ignored (change was not in target): {0}"
+ .format(pformat(commits_ignored)))
+
+ if options.cherry_pick:
+ cherrypick(cherry_pick_hashes, full_target_branch_name)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/impala/blob/674d29d7/bin/ignored_commits.json
----------------------------------------------------------------------
diff --git a/bin/ignored_commits.json b/bin/ignored_commits.json
new file mode 100644
index 0000000..fe51488
--- /dev/null
+++ b/bin/ignored_commits.json
@@ -0,0 +1 @@
+[]