You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2018/05/24 22:27:23 UTC
[5/6] mesos git commit: Ported all support scripts to Python 3.
Ported all support scripts to Python 3.
The scripts are in a temporary directory, support/python3.
The scripts have been ported using 2to3, the official tool to do so.
Many of these scripts require testing from the community before being
used by default.
The script building the virtual environment and the git hooks have
been updated to use the new scripts if the environment variable
`MESOSSUPPORTPYTHON` is set to `3` by the user.
Review: https://reviews.apache.org/r/67059/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/960df5c4
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/960df5c4
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/960df5c4
Branch: refs/heads/master
Commit: 960df5c48b6b04037aabc1c1f75506d82c629732
Parents: 71315eb
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Thu May 24 14:58:13 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Thu May 24 14:58:13 2018 -0700
----------------------------------------------------------------------
support/README.md | 4 +
support/build-virtualenv | 16 +-
support/hooks/post-rewrite | 14 +-
support/hooks/pre-commit | 14 +-
support/python3/apply-reviews.py | 453 +++++++++++++++++++++
support/python3/generate-endpoint-help.py | 403 +++++++++++++++++++
support/python3/jsonurl.py | 55 +++
support/python3/mesos-gtest-runner.py | 294 ++++++++++++++
support/python3/mesos-split.py | 79 ++++
support/python3/mesos-style.py | 521 +++++++++++++++++++++++++
support/python3/post-reviews.py | 432 ++++++++++++++++++++
support/python3/push-commits.py | 158 ++++++++
support/python3/test-upgrade.py | 254 ++++++++++++
support/python3/verify-reviews.py | 318 +++++++++++++++
14 files changed, 3007 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/README.md
----------------------------------------------------------------------
diff --git a/support/README.md b/support/README.md
index e812e96..0eae4d3 100644
--- a/support/README.md
+++ b/support/README.md
@@ -4,3 +4,7 @@ This directory contains various helper scripts.
The scripts directly in this directory are intended to be used by Mesos
developers on the command-line.
+
+**To use the Python 3 version of these scripts, set the environment
+variable `MESOS_SUPPORT_PYTHON` to `3` and run again the script
+`support/build-virtualenv`.**
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/build-virtualenv
----------------------------------------------------------------------
diff --git a/support/build-virtualenv b/support/build-virtualenv
index 850af89..5fda081 100755
--- a/support/build-virtualenv
+++ b/support/build-virtualenv
@@ -46,10 +46,18 @@ fi
PYTHON_MAJOR=$(${PYTHON} -c 'import sys; print(sys.version_info[0])')
PYTHON_MINOR=$(${PYTHON} -c 'import sys; print(sys.version_info[1])')
-if [ "${PYTHON_MAJOR}" != "2" ] || [ "${PYTHON_MINOR}" -lt "6" ]; then
- echo "You must be running python 2.6 or 2.7 in order to continue."
- echo "Consider running as 'PYTHON=python2 ./bootstrap' or similar."
- exit 1
+if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ if [ "${PYTHON_MAJOR}" != "3" ] || [ "${PYTHON_MINOR}" -lt "6" ]; then
+ echo "You must be running python 3.6 in order to continue."
+ echo "Consider running as 'PYTHON=python3 <previous-command>' or similar."
+ exit 1
+ fi
+else
+ if [ "${PYTHON_MAJOR}" != "2" ] || [ "${PYTHON_MINOR}" -lt "6" ]; then
+ echo "You must be running python 2.6 or 2.7 in order to continue."
+ echo "Consider running as 'PYTHON=python2 ./bootstrap' or similar."
+ exit 1
+ fi
fi
# Set up a virtual environment for the linters.
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/hooks/post-rewrite
----------------------------------------------------------------------
diff --git a/support/hooks/post-rewrite b/support/hooks/post-rewrite
index 1ab14ab..81f3454 100755
--- a/support/hooks/post-rewrite
+++ b/support/hooks/post-rewrite
@@ -30,11 +30,21 @@ if [ "$ADDED_OR_MODIFIED" ]; then
# many implementations do not support the `-r` flag, (which instructs
# `xargs` to not run the script if the arguments are empty), so we also
# cannot use that.
- ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ # TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+ if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ ./support/python3/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ else
+ ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ fi
fi
# Check that the commits are properly split between mesos, libprocess and stout.
## In git, '@' represent current head, '@~' represent previous commit. We check
## the style of changes between current head and previous commit after a commit
## is rewritten.
-git diff --name-only --diff-filter=AMD @~..@ | xargs ./support/mesos-split.py || exit 1
+# TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ git diff --name-only --diff-filter=AMD @~..@ | xargs ./support/python3/mesos-split.py || exit 1
+else
+ git diff --name-only --diff-filter=AMD @~..@ | xargs ./support/mesos-split.py || exit 1
+fi
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/hooks/pre-commit
----------------------------------------------------------------------
diff --git a/support/hooks/pre-commit b/support/hooks/pre-commit
index 6faba98..ac0ff24 100755
--- a/support/hooks/pre-commit
+++ b/support/hooks/pre-commit
@@ -33,8 +33,18 @@ if [ -n "$ADDED_OR_MODIFIED" ]; then
# many implementations do not support the `-r` flag, (which instructs
# `xargs` to not run the script if the arguments are empty), so we also
# cannot use that.
- ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ # TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+ if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ ./support/python3/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ else
+ ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ fi
fi
# Check that the commits are properly split between mesos, libprocess and stout.
-git diff --cached --name-only --diff-filter=AMD | xargs ./support/mesos-split.py || exit 1
+# TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ git diff --cached --name-only --diff-filter=AMD | xargs ./support/python3/mesos-split.py || exit 1
+else
+ git diff --cached --name-only --diff-filter=AMD | xargs ./support/mesos-split.py || exit 1
+fi
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/apply-reviews.py
----------------------------------------------------------------------
diff --git a/support/python3/apply-reviews.py b/support/python3/apply-reviews.py
new file mode 100755
index 0000000..bf72c80
--- /dev/null
+++ b/support/python3/apply-reviews.py
@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script translates reviews on ReviewBoard into commits
+on the current branch.
+"""
+
+import argparse
+import atexit
+import json
+import linecache
+import os
+import pipes
+import platform
+import re
+import ssl
+import subprocess
+import sys
+import urllib.request
+import urllib.error
+import urllib.parse
+
+
+REVIEWBOARD_REVIEW_URL = 'https://reviews.apache.org/r'
+REVIEWBOARD_API_URL =\
+ 'https://reviews.apache.org/api/review-requests'
+REVIEWBOARD_USER_URL = 'https://reviews.apache.org/api/users'
+
+
+GITHUB_URL = 'https://api.github.com/repos/apache/mesos/pulls'
+GITHUB_PATCH_URL =\
+ 'https://patch-diff.githubusercontent.com/raw/apache/mesos/pull'
+
+
+def review_api_url(review_id):
+ """Returns a Review Board API URL given a review ID."""
+ # Reviewboard REST API expects '/' at the end of the URL.
+ return '{base}/{review}/'.format(
+ base=REVIEWBOARD_API_URL,
+ review=review_id)
+
+
+def review_url(review_id):
+ """Returns a Review Board UI URL given a review ID."""
+ return '{base}/{review}/'.format(
+ base=REVIEWBOARD_REVIEW_URL,
+ review=review_id)
+
+
+def pull_request_url(pull_request_number):
+ """Returns a GitHub pull request URL given a PR number."""
+ return '{base}/{pr}'.format(
+ base=GITHUB_URL,
+ pr=pull_request_number)
+
+
+def reviewboard_user_url(username):
+ """Returns a Review Board URL for a user given a username."""
+ # Reviewboard REST API expects '/' at the end of the URL.
+ return '{base}/{user}/'.format(
+ base=REVIEWBOARD_USER_URL,
+ user=username)
+
+
+def patch_url(options):
+ """Returns a Review Board or a GitHub URL for a patch."""
+ if options['review_id']:
+ # Reviewboard REST API expects '/' at the end of the URL.
+ return '{base}/{review}/diff/raw/'.format(
+ base=REVIEWBOARD_REVIEW_URL,
+ review=options['review_id'])
+ elif options['github']:
+ return '{base}/{patch}.patch'.format(
+ base=GITHUB_PATCH_URL,
+ patch=options['github'])
+ return None
+
+
+def url_to_json(url):
+ """Performs HTTP request and returns JSON-ified response."""
+ json_str = urllib.request.urlopen(url)
+ return json.loads(json_str.read())
+
+
+def extract_review_id(url):
+ """Extracts review ID from Review Board URL."""
+ review_id = re.search(REVIEWBOARD_API_URL + r'/(\d+)/', url)
+ if review_id:
+ return review_id.group(1)
+ return ''
+
+def review_chain(review_id):
+ """Returns a parent review chain for a given review ID."""
+ json_obj = url_to_json(review_api_url(review_id))
+
+ # Stop as soon as we stumble upon a submitted request.
+ status = json_obj.get('review_request').get('status')
+ if status == "submitted":
+ sys.stderr.write('Warning: Review {review} has already'
+ ' been applied\n'.format(review=review_id))
+ return []
+
+ # Verify that the review has exactly one parent.
+ parent = json_obj.get('review_request').get('depends_on')
+ if len(parent) > 1:
+ sys.stderr.write('Error: Review {review} has more than'
+ ' one parent'.format(review=review_id))
+ sys.exit(1)
+ elif not parent:
+ return [(review_id, json_obj.get('review_request').get('summary'))]
+ else:
+ # The review has exactly one parent.
+ review_list = review_chain(extract_review_id(parent[0].get('href')))
+
+ review = (review_id, json_obj.get('review_request').get('summary'))
+ if review not in review_list:
+ return review_list + [review]
+ else:
+ sys.stderr.write('Found a circular dependency in the chain starting'
+ ' at {review}\n'.format(review=review_id))
+ sys.exit(1)
+
+
+def shell(command, dry_run):
+ """
+ Runs a command in a shell, unless the dry-run option
+ is set (in which case it just prints the command).
+ """
+ if dry_run:
+ print(command)
+ return
+
+ error_code = subprocess.call(command, stderr=subprocess.STDOUT, shell=True)
+ if error_code != 0:
+ sys.exit(error_code)
+
+
+def apply_review(options):
+ """Applies a review with a given ID locally."""
+ # Make sure we don't leave the patch behind in case of failure.
+ # We store the patch ID in a local variable to ensure the lambda
+ # captures the current patch ID.
+ patch_file = '%s.patch' % patch_id(options)
+ if not options["keep_patches"]:
+ atexit.register(
+ lambda: os.path.exists(patch_file) and os.remove(patch_file))
+
+ fetch_patch(options)
+ apply_patch(options)
+ commit_patch(options)
+
+
+def ssl_create_default_context():
+ """
+ Equivalent to `ssl.create_default_context` with default arguments and
+ certificate/hostname verification disabled.
+ See: https://github.com/python/cpython/blob/2.7/Lib/ssl.py#L410
+ This function requires Python >= 2.7.9.
+ """
+ # pylint: disable=no-member
+ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+
+ # SSLv2 considered harmful.
+ context.options |= ssl.OP_NO_SSLv2
+
+ # SSLv3 has problematic security and is only required for really old
+ # clients such as IE6 on Windows XP.
+ context.options |= ssl.OP_NO_SSLv3
+
+ # Disable compression to prevent CRIME attacks (OpenSSL 1.0+).
+ context.options |= getattr(ssl, "OP_NO_COMPRESSION", 0)
+
+ # Disable certificate and hostname verification.
+ context.verify_mode = ssl.CERT_NONE
+ context.check_hostname = False
+
+
+def fetch_patch(options):
+ """Fetches a patch from Review Board or GitHub."""
+ # pylint: disable=unexpected-keyword-arg
+ if platform.system() == 'Windows':
+ # This call requires Python >= 2.7.9.
+ response = urllib.request.urlopen(
+ patch_url(options),
+ context=ssl_create_default_context())
+
+ with open('%s.patch' % patch_id(options), 'wb') as patch:
+ patch.write(response.read())
+ else:
+ # NOTE: SSL contexts are only supported in Python 2.7.9+. The version
+ # of Python running on the non-Windows ASF CI machines is sometimes
+ # older. Hence, we fall back to `wget` on non-Windows machines.
+ cmd = ' '.join([
+ 'wget',
+ '--no-check-certificate',
+ '--no-verbose',
+ '-O '
+ '{review_id}.patch',
+ '{url}']).format(
+ review_id=patch_id(options),
+ url=patch_url(options))
+
+ # In case of GitHub we always need to fetch the patch to extract
+ # username and email, so we ignore the dry_run option by setting the
+ # second parameter to False.
+ if options['github']:
+ shell(cmd, False)
+ else:
+ shell(cmd, options['dry_run'])
+
+
+def patch_id(options):
+ """Returns the review ID or the GitHub pull request number."""
+ return options['review_id'] or options['github']
+
+
+def apply_patch(options):
+ """Applies patch locally."""
+ cmd = 'git apply --index {review_id}.patch'.format(
+ review_id=patch_id(options))
+
+ if options['3way']:
+ cmd += ' --3way'
+
+ if platform.system() == 'Windows':
+ # NOTE: Depending on the Git settings, there may or may not be
+ # carriage returns in files and in the downloaded patch.
+ # We ignore these errors on Windows.
+ cmd += ' --ignore-whitespace'
+
+ shell(cmd, options['dry_run'])
+
+
+def quote(string):
+ """Quote a variable so it can be safely used in shell."""
+ return string.replace("'", "'\\''")
+
+
+def commit_patch(options):
+ """Commits patch locally."""
+ data = patch_data(options)
+
+ # Check whether we need to amend the commit message.
+ if options['no_amend']:
+ amend = ''
+ else:
+ amend = '-e'
+
+ # Check whether we should skip the commit hooks.
+ if options['skip_hooks']:
+ verify = '-n'
+ else:
+ verify = ''
+
+ # NOTE: Windows does not support multi-line commit messages via the shell.
+ message_file = '%s.message' % patch_id(options)
+ atexit.register(
+ lambda: os.path.exists(message_file) and os.remove(message_file))
+
+ with open(message_file, 'w') as message:
+ # Add a shell command creating the message file for dry-run mode.
+ if options["dry_run"]:
+ shell(
+ "printf {msg} > {file}".format(
+ msg=pipes.quote(data['message']).replace('\n', '\\n'),
+ file=message_file),
+ True)
+ message.write(data['message'])
+
+ cmd = 'git commit' \
+ ' --author \"{author}\"' \
+ ' {amend} -aF \"{message}\"' \
+ ' {verify}'.format(
+ author=quote(data['author']),
+ amend=amend,
+ message=message_file,
+ verify=verify)
+
+ shell(cmd, options['dry_run'])
+
+
+def patch_data(options):
+ """
+ Populates and returns a dictionary with data necessary for
+ committing the patch (such as the message, the author, etc.).
+ """
+ if options['review_id']:
+ return reviewboard_data(options)
+ elif options['github']:
+ return github_data(options)
+ return None
+
+
+def get_author(patch):
+ """Reads the author name and email from the .patch file"""
+ author = linecache.getline(patch, 2)
+ return author.replace('From: ', '').rstrip()
+
+
+def github_data(options):
+ """Fetches pull request data and populates internal data structure."""
+ pull_request_number = options['github']
+ pull_request = url_to_json(pull_request_url(pull_request_number))
+
+ title = pull_request.get('title')
+ description = pull_request.get('body')
+ url = '{url}/{pr}'.format(url=GITHUB_URL, pr=pull_request_number)
+ author = get_author('{pr}.patch'.format(pr=pull_request_number))
+ message = '\n\n'.join([
+ title,
+ description,
+ 'This closes #{pr}'.format(pr=pull_request_number)])
+
+ review_data = {
+ "summary": title,
+ "description": description,
+ "url": url,
+ "author": author,
+ "message": message
+ }
+
+ return review_data
+
+
+def reviewboard_data(options):
+ """Fetches review data and populates internal data structure."""
+ review_id = options['review_id']
+
+ # Populate review object.
+ review = url_to_json(review_api_url(review_id)).get('review_request')
+
+ url = review_url(review_id)
+
+ # Populate user object.
+ user = url_to_json(reviewboard_user_url(
+ review.get('links').get('submitter').get('title'))).get('user')
+
+ # Only include a description if it is not identical to the summary.
+ message_data = [review.get('summary')]
+ if review.get('description') != review.get('summary'):
+ message_data.append(review.get('description'))
+ message_data.append('Review: {review_url}'.format(review_url=url))
+
+ author = '{author} <{email}>'.format(
+ author=user.get('fullname'),
+ email=user.get('email'))
+ message = '\n\n'.join(message_data)
+
+ review_data = {
+ "summary": review.get('summary'),
+ "description": review.get('description'),
+ "url": url,
+ "author": author,
+ "message": message
+ }
+
+ return review_data
+
+
+def parse_options():
+ """Parses command line options and returns an option dictionary."""
+ options = {}
+
+ parser = argparse.ArgumentParser(
+ description='Recursively apply Review Board reviews'
+ ' and GitHub pull requests.')
+
+ parser.add_argument('-d', '--dry-run',
+ action='store_true',
+ help='Perform a dry run.')
+ parser.add_argument('-k', '--keep-patches',
+ action='store_true',
+ help="Do not delete downloaded patch files.")
+ parser.add_argument('-n', '--no-amend',
+ action='store_true',
+ help='Do not amend commit message.')
+ parser.add_argument('-c', '--chain',
+ action='store_true',
+ help='Recursively apply parent review chain.')
+ parser.add_argument('-s', '--skip-hooks',
+ action='store_true',
+ help='Skip the commit hooks (e.g., Mesos style check).')
+ parser.add_argument('-3', '--3way',
+ dest='three_way',
+ action='store_true',
+ help='Use 3 way merge in git apply.')
+
+ # Add -g and -r and make them mutually exclusive.
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('-g', '--github',
+ metavar='PULL_REQUEST',
+ help='Pull request number')
+ group.add_argument('-r', '--review-id',
+ metavar='REVIEW_ID',
+ help='Numeric review ID')
+
+ args = parser.parse_args()
+
+ options['review_id'] = args.review_id
+ options['dry_run'] = args.dry_run
+ options['keep_patches'] = args.keep_patches
+ options['no_amend'] = args.no_amend
+ options['github'] = args.github
+ options['chain'] = args.chain
+ options['skip_hooks'] = args.skip_hooks
+ options['3way'] = args.three_way
+
+ return options
+
+
+def reviewboard(options):
+ """Applies either a chain of reviewboard patches or a single patch."""
+ if options['chain']:
+ # Retrieve the list of reviews to apply.
+ applied = set()
+ for review_id, _ in review_chain(options['review_id']):
+ if review_id not in applied:
+ applied.add(review_id)
+ options['review_id'] = review_id
+ apply_review(options)
+ else:
+ apply_review(options)
+
+
+def main():
+ """
+ Main function to apply reviews.
+ """
+ options = parse_options()
+
+ if options['review_id']:
+ reviewboard(options)
+ else:
+ apply_review(options)
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/generate-endpoint-help.py
----------------------------------------------------------------------
diff --git a/support/python3/generate-endpoint-help.py b/support/python3/generate-endpoint-help.py
new file mode 100755
index 0000000..711d3d3
--- /dev/null
+++ b/support/python3/generate-endpoint-help.py
@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Autogenerate documentation for all process endpoints spawned by a
+Mesos master and agent.
+"""
+
+import argparse
+import atexit
+import json
+import os
+import posixpath
+import re
+import shutil
+import subprocess
+import sys
+import time
+import urllib.request
+import urllib.error
+import urllib.parse
+
+
+# The host ip and master and agent ports.
+HOST_IP = "127.0.0.1"
+MASTER_PORT = 5050
+AGENT_PORT = 5051
+
+# The master and agent programs to launch.
+# We considered making the parameters to these commands something that
+# a user could specify on the command line, but ultimately chose to
+# hard code them. Different parameters may cause different endpoints
+# to become available, and we should modify this script to ensure that
+# we cover all of them instead of leaving that up to the user.
+MASTER_COMMAND = [
+ 'mesos-master.sh',
+ '--ip=%s' % (HOST_IP),
+ '--registry=in_memory',
+ '--work_dir=/tmp/mesos'
+]
+
+# NOTE: The agent flags here ensure that this script can run inside docker.
+AGENT_COMMAND = [
+ 'mesos-agent.sh',
+ '--master=%s:%s' % (HOST_IP, MASTER_PORT),
+ '--work_dir=/tmp/mesos',
+ '--systemd_enable_support=false',
+ '--launcher=posix'
+]
+
+
+# A header to add onto all generated markdown files.
+MARKDOWN_HEADER = """---
+title: %s
+layout: documentation
+---
+<!--- This is an automatically generated file. DO NOT EDIT! --->
+"""
+
+# A template of the title to add onto all generated markdown files.
+MARKDOWN_TITLE = "Apache Mesos - HTTP Endpoints%s"
+
+# A global timeout as well as a retry interval when hitting any http
+# endpoints on the master or agent (in seconds).
+RECEIVE_TIMEOUT = 10
+RETRY_INTERVAL = 0.10
+
+
+class Subprocess(object):
+ """The process running using this script."""
+ def __init__(self):
+ self.current = None
+
+ def cleanup(self):
+ """Kill the process running once the script is done."""
+ if self.current:
+ self.current.kill()
+
+
+# A pointer to the top level directory of the mesos project.
+GIT_TOP_DIR = subprocess.check_output(
+ ['git', 'rev-parse', '--show-cdup']).strip()
+
+with open(os.path.join(GIT_TOP_DIR, 'CHANGELOG'), 'r') as f:
+ if 'mesos' not in f.readline().lower():
+ print(('You must run this command from within'
+ ' the Mesos source repository!'), file=sys.stderr)
+ sys.exit(1)
+
+
+def parse_options():
+ """Parses command line options and populates the dictionary."""
+
+ options = {}
+
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawTextHelpFormatter,
+ description='Generate markdown files from all installed HTTP '
+ 'endpoints on a Mesos master and agent process.')
+
+ parser.add_argument(
+ '-c', '--command-path',
+ metavar='COMMAND_PATH',
+ default=os.path.join(GIT_TOP_DIR, "build/bin"),
+ help='Path to the Mesos master and agent commands.\n'
+ '(default: %(default)s)')
+
+ parser.add_argument(
+ '-o', '--output-path',
+ metavar='OUTPUT_PATH',
+ default=os.path.join(GIT_TOP_DIR, "docs/endpoints"),
+ help='Path to the top level directory where all\n'
+ 'generated markdown files will be placed.\n'
+ '(default: %(default)s)')
+
+ args = parser.parse_args()
+
+ options['command_path'] = args.command_path
+ options['output_path'] = args.output_path
+
+ return options
+
+
+def get_url_until_success(url):
+ """Continuously tries to open a url until it succeeds or times out."""
+ time_spent = 0
+ while time_spent < RECEIVE_TIMEOUT:
+ try:
+ helps = urllib.request.urlopen(url)
+ break
+ except Exception:
+ time.sleep(RETRY_INTERVAL)
+ time_spent += RETRY_INTERVAL
+
+ if time_spent >= RECEIVE_TIMEOUT:
+ print('Timeout attempting to hit url: %s' % (url), file=sys.stderr)
+ sys.exit(1)
+
+ return helps.read()
+
+
+def get_help(ip, port):
+ """
+ Grabs the help strings for all endpoints at http://ip:port as a JSON object.
+ """
+ url = 'http://%s:%d/help?format=json' % (ip, port)
+ return json.loads(get_url_until_success(url))
+
+
+def generalize_endpoint_id(p_id):
+ """Generalizes the id of the form e.g. process(1) to process(id)."""
+ return re.sub(r'\([0-9]+\)', '(id)', p_id)
+
+
+def normalize_endpoint_id(p_id):
+ """Normalizes the id of the form e.g. process(id) to process."""
+ return re.sub(r'\([0-9]+\)', '', p_id)
+
+
+def get_endpoint_path(p_id, name):
+ """
+ Generates the canonical endpoint path, given id and name.
+
+ Examples: ('process', '/') -> '/process'
+ ('process(id)', '/') -> '/process(id)'
+ ('process', '/endpoint') -> '/process/endpoint'
+ """
+ # Tokenize the endpoint by '/' (filtering
+ # out any empty strings between '/'s)
+ path_parts = [_f for _f in name.split('/') if _f]
+
+ # Conditionally prepend the 'id' to the list of path parts.
+ # Following the notion of a 'delegate' in Mesos, we want our
+ # preferred endpoint paths for the delegate process to be
+ # '/endpoint' instead of '/process/endpoint'. Since this script only
+ # starts 1 master and 1 agent, our only delegate processes are
+ # "master" and "slave(id)". If the id matches one of these, we don't
+ # prepend it, otherwise we do.
+ p_id = generalize_endpoint_id(p_id)
+ delegates = ["master", "slave(id)"]
+ if p_id not in delegates:
+ path_parts = [p_id] + path_parts
+
+ return posixpath.join('/', *path_parts)
+
+
+def get_relative_md_path(p_id, name):
+ """
+ Generates the relative path of the generated .md file from id and name.
+
+ This path is relative to the options['output_path'] directory.
+
+ Examples: master/health.md
+ master/maintenance/schedule.md
+ registrar/registry.md
+ version.md
+ """
+ new_id = normalize_endpoint_id(p_id)
+ # Strip the leading slash
+ new_name = name[1:]
+
+ if new_name:
+ return os.path.join(new_id, new_name + '.md')
+ return os.path.join(new_id + '.md')
+
+
+def write_markdown(path, output, title):
+ """Writes 'output' to the file at 'path'."""
+ print('generating: %s' % (path))
+
+ dirname = os.path.dirname(path)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+ outfile = open(path, 'w+')
+
+ # Add our header and remove all '\n's at the end of the output if
+ # there are any.
+ output = (MARKDOWN_HEADER % title) + '\n' + output.rstrip()
+
+ outfile.write(output)
+ outfile.close()
+
+
+def dump_index_markdown(master_help, agent_help, options):
+ """
+ Dumps an index for linking to the master and agent help files.
+
+ This file is dumped into a directory rooted at
+ options['output_path'].
+ """
+
+ # The output template for the HTTP endpoints index.
+ # We use a helper function below to insert text into the '%s' format
+ # strings contained in the "Master Endpoints" and "Agent Endpoints"
+ # sections of this template.
+ output = """# HTTP Endpoints #
+
+Below is a list of HTTP endpoints available for a given Mesos process.
+
+Depending on your configuration, some subset of these endpoints will be
+available on your Mesos master or agent. Additionally, a `/help`
+endpoint will be available that displays help similar to what you see
+below.
+
+** NOTE: ** If you are using Mesos 1.1 or later, we recommend using the
+new [v1 Operator HTTP API](../operator-http-api.md) instead of the
+unversioned REST endpoints listed below. These endpoints will be
+deprecated in the future.
+
+
+** NOTE: ** The documentation for these endpoints is auto-generated from
+the Mesos source code. See `support/python3/generate-endpoint-help.py`.
+
+## Master Endpoints ##
+
+Below are the endpoints that are available on a Mesos master. These
+endpoints are reachable at the address `http://ip:port/endpoint`.
+
+For example, `http://master.com:5050/files/browse`.
+
+%s
+
+## Agent Endpoints ##
+
+Below are the endpoints that are available on a Mesos agent. These
+endpoints are reachable at the address `http://ip:port/endpoint`.
+
+For example, `http://agent.com:5051/files/browse`.
+
+%s
+"""
+
+ def generate_links(master_or_agent_help):
+ """
+ Iterates over the input JSON and creates a list of links to
+ to the markdown files generated by this script. These links
+ are grouped per process, with the process's name serving as a
+ header for each group. All links are relative to
+ options['output_path'].
+
+ For example:
+ ### profiler ###
+ * [/profiler/start] (profiler/start.md)
+ * [/profiler/stop] (profiler/stop.md)
+
+ ### version ###
+ * [/version] (version.md)
+ """
+ output = ""
+ for process in master_or_agent_help['processes']:
+ p_id = process['id']
+ output += '### %s ###\n' % (generalize_endpoint_id(p_id))
+ for endpoint in process['endpoints']:
+ name = endpoint['name']
+ output += '* [%s](%s)\n' % (get_endpoint_path(p_id, name),
+ get_relative_md_path(p_id, name))
+ output += '\n'
+
+ # Remove any trailing newlines
+ return output.rstrip()
+
+ output = output % (generate_links(master_help),
+ generate_links(agent_help))
+
+ path = os.path.join(options['output_path'], 'index.md')
+ write_markdown(path, output, MARKDOWN_TITLE % "")
+
+
+def dump_markdown(master_or_agent_help, options):
+ """
+ Dumps JSON encoded help strings into markdown files.
+
+ These files are dumped into a directory rooted at
+ options['output_path'].
+ """
+ for process in master_or_agent_help['processes']:
+ p_id = process['id']
+ for endpoint in process['endpoints']:
+ name = endpoint['name']
+ text = endpoint['text']
+ title = get_endpoint_path(p_id, name)
+
+ relative_path = get_relative_md_path(p_id, name)
+ path = os.path.join(options['output_path'], relative_path)
+ write_markdown(path, text, MARKDOWN_TITLE % (" - " + title))
+
+
+def start_master(options):
+ """
+ Starts the Mesos master using the specified command.
+
+ This method returns the Popen object used to start it so it can
+ be killed later on.
+ """
+ cmd = os.path.join('.', options['command_path'], MASTER_COMMAND[0])
+ master = subprocess.Popen([cmd] + MASTER_COMMAND[1:])
+
+ # Wait for the master to become responsive
+ get_url_until_success("http://%s:%d/health" % (HOST_IP, MASTER_PORT))
+ return master
+
+
+def start_agent(options):
+ """
+ Starts the Mesos agent using the specified command.
+
+ This method returns the Popen object used to start it so it can
+ be killed later on.
+ """
+ cmd = os.path.join('.', options['command_path'], AGENT_COMMAND[0])
+ agent = subprocess.Popen([cmd] + AGENT_COMMAND[1:])
+
+ # Wait for the agent to become responsive.
+ get_url_until_success('http://%s:%d/health' % (HOST_IP, AGENT_PORT))
+ return agent
+
+
+def main():
+ """
+ Called when the Python script is used, we do not directly write code
+ after 'if __name__ == "__main__"' as we cannot set variables in that case.
+ """
+ # A dictionary of the command line options passed in.
+ options = parse_options()
+
+ # A pointer to the current subprocess for the master or agent.
+ # This is useful for tracking the master or agent subprocesses so
+ # that we can kill them if the script exits prematurely.
+ subproc = Subprocess()
+ atexit.register(subproc.cleanup())
+ subproc.current = start_master(options)
+ master_help = get_help(HOST_IP, MASTER_PORT)
+ subproc.current.kill()
+
+ subproc.current = start_agent(options)
+ agent_help = get_help(HOST_IP, AGENT_PORT)
+ subproc.current.kill()
+
+ shutil.rmtree(options['output_path'], ignore_errors=True)
+ os.makedirs(options['output_path'])
+
+ dump_index_markdown(master_help, agent_help, options)
+ dump_markdown(master_help, options)
+ dump_markdown(agent_help, options)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/jsonurl.py
----------------------------------------------------------------------
diff --git a/support/python3/jsonurl.py b/support/python3/jsonurl.py
new file mode 100755
index 0000000..ffa3e36
--- /dev/null
+++ b/support/python3/jsonurl.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A utility that expects JSON data at a particular URL and lets you
+recursively extract keys from the JSON object as specified on the
+command line (each argument on the command line after the first will
+be used to recursively index into the JSON object). The name is a
+play off of 'curl'.
+"""
+
+import json
+import sys
+import urllib.request
+import urllib.error
+import urllib.parse
+
+
+def main():
+ """Expects at least one argument on the command line."""
+ if len(sys.argv) < 2:
+ print("USAGE: {} URL [KEY...]".format(sys.argv[0]), file=sys.stderr)
+ sys.exit(1)
+
+ url = sys.argv[1]
+
+ data = json.loads(urllib.request.urlopen(url).read())
+
+ for arg in sys.argv[2:]:
+ try:
+ temp = data[arg]
+ data = temp
+ except KeyError:
+ print("'" + arg + "' was not found", file=sys.stderr)
+ sys.exit(1)
+
+ print(data.encode("utf-8"))
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/mesos-gtest-runner.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-gtest-runner.py b/support/python3/mesos-gtest-runner.py
new file mode 100755
index 0000000..153ded2
--- /dev/null
+++ b/support/python3/mesos-gtest-runner.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+Parallel test runner for GoogleTest programs.
+
+This script allows one to execute GoogleTest tests in parallel.
+GoogleTest programs come with built-in support for running in parallel.
+Here tests can automatically be partitioned across a number of test
+program invocations ("shards"). This script provides a convenient
+wrapper around that functionality and stream-lined output.
+"""
+
+import argparse
+import multiprocessing
+import os
+import shlex
+import signal
+import subprocess
+import sys
+
+
+DEFAULT_NUM_JOBS = int(multiprocessing.cpu_count() * 1.5)
+
+
+class Bcolors(object):
+ """
+ A collection of tty output modifiers.
+
+ To switch the output of a string, prefix it with the desired
+ modifier, and terminate it with 'ENDC'.
+ """
+
+ HEADER = '\033[95m' if sys.stdout.isatty() else ''
+ OKBLUE = '\033[94m' if sys.stdout.isatty() else ''
+ OKGREEN = '\033[92m' if sys.stdout.isatty() else ''
+ WARNING = '\033[93m' if sys.stdout.isatty() else ''
+ FAIL = '\033[91m'if sys.stdout.isatty() else ''
+ ENDC = '\033[0m' if sys.stdout.isatty() else ''
+ BOLD = '\033[1m' if sys.stdout.isatty() else ''
+ UNDERLINE = '\033[4m' if sys.stdout.isatty() else ''
+
+ @staticmethod
+ def colorize(string, *color_codes):
+ """Decorate a string with a number of color codes."""
+ colors = ''.join(color_codes)
+ return '{begin}{string}{end}'.format(
+ begin=colors if sys.stdout.isatty() else '',
+ string=string,
+ end=Bcolors.ENDC if sys.stdout.isatty() else '')
+
+
+def run_test(opts):
+ """
+ Perform an actual run of the test executable.
+
+ Expects a list of parameters giving the number of the current
+ shard, the total number of shards, and the executable to run.
+ """
+ shard, nshards, executable = opts
+
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+ env = os.environ.copy()
+ env['GTEST_TOTAL_SHARDS'] = str(nshards)
+ env['GTEST_SHARD_INDEX'] = str(shard)
+
+ try:
+ output = subprocess.check_output(
+ executable.split(),
+ stderr=subprocess.STDOUT,
+ env=env,
+ universal_newlines=True)
+ print(Bcolors.colorize('.', Bcolors.OKGREEN), end='')
+ sys.stdout.flush()
+ return True, output
+ except subprocess.CalledProcessError as error:
+ print(Bcolors.colorize('.', Bcolors.FAIL), end='')
+ sys.stdout.flush()
+ return False, error.output
+
+
+def parse_arguments():
+ """Return the executable to work on, and a list of options."""
+ parser = argparse.ArgumentParser(
+ usage='Usage: %prog [options] <test> [-- <test_options>]')
+
+ parser.add_argument(
+ '-j', '--jobs', type='int',
+ default=DEFAULT_NUM_JOBS,
+ help='number of parallel jobs to spawn. DEFAULT: {default_}'
+ .format(default_=DEFAULT_NUM_JOBS))
+
+ parser.add_argument(
+ '-s', '--sequential', type='string',
+ default='',
+ help='gtest filter for tests to run sequentially')
+
+ parser.add_argument(
+ '-v', '--verbosity', type='int',
+ default=1,
+ help='output verbosity:'
+ ' 0 only shows summarized information,'
+ ' 1 also shows full logs of failed shards, and anything'
+ ' >1 shows all output. DEFAULT: 1')
+
+ parser.epilog = (
+ 'The environment variable MESOS_GTEST_RUNNER_FLAGS '
+ 'can be used to set a default set of flags. Flags passed on the '
+ 'command line always have precedence over these defaults.')
+
+ # If the environment variable `MESOS_GTEST_RUNNER_FLAGS` is set we
+ # use it to set a default set of flags to pass. Flags passed on
+ # the command line always have precedence over these defaults.
+ #
+ # We manually construct `args` here and make use of the fact that
+ # in `optparser`'s implementation flags passed later on the
+ # command line overrule identical flags passed earlier.
+ args = []
+ if 'MESOS_GTEST_RUNNER_FLAGS' in os.environ:
+ args.extend(shlex.split(os.environ['MESOS_GTEST_RUNNER_FLAGS']))
+ args.extend(sys.argv[1:])
+
+ (options, executable) = parser.parse_args(args)
+
+ if not executable:
+ parser.print_usage()
+ sys.exit(1)
+
+ if not os.path.isfile(executable[0]):
+ print(
+ Bcolors.colorize(
+ "ERROR: File '{file}' does not exists"
+ .format(file=executable[0]), Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ if not os.access(executable[0], os.X_OK):
+ print(
+ Bcolors.colorize(
+ "ERROR: File '{file}' is not executable"
+ .format(file=executable[0]), Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ if options.sequential and options.sequential.count(':-'):
+ print(
+ Bcolors.colorize(
+ "ERROR: Cannot use negative filters in "
+ "'sequential' parameter: '{filter}'"
+ .format(filter=options.sequential), Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ if options.sequential and os.environ.get('GTEST_FILTER') and \
+ os.environ['GTEST_FILTER'].count(':-'):
+ print(
+ Bcolors.colorize(
+ "ERROR: Cannot specify both 'sequential' ""option "
+ "and environment variable 'GTEST_FILTER' "
+ "containing negative filters",
+ Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ # Since empty strings are falsy, directly compare against `None`
+ # to preserve an empty string passed via `GTEST_FILTER`.
+ if os.environ.get('GTEST_FILTER') != None:
+ options.parallel = '{env_filter}:-{sequential_filter}'\
+ .format(env_filter=os.environ['GTEST_FILTER'],
+ sequential_filter=options.sequential)
+ else:
+ options.parallel = '*:-{sequential_filter}'\
+ .format(sequential_filter=options.sequential)
+
+ return executable, options
+
+
+if __name__ == '__main__':
+ EXECUTABLE, OPTIONS = parse_arguments()
+
+ def options_gen(executable, filter_, jobs):
+ """Generator for options for a certain shard.
+
+ Here we set up GoogleTest specific flags, and generate
+ distinct shard indices.
+ """
+ opts = list(range(jobs))
+
+ # If we run in a terminal, enable colored test output. We
+ # still allow users to disable this themselves via extra args.
+ if sys.stdout.isatty():
+ args = executable[1:]
+ executable = '{exe} --gtest_color=yes {args}'\
+ .format(exe=executable[0], args=args if args else '')
+
+ if filter_:
+ executable = '{exe} --gtest_filter={filter}'\
+ .format(exe=executable, filter=filter_)
+
+ for opt in opts:
+ yield opt, jobs, executable
+
+ try:
+ RESULTS = []
+
+ POOL = multiprocessing.Pool(processes=OPTIONS.jobs)
+
+ # Run parallel tests.
+ #
+ # Multiprocessing's `map` cannot properly handle `KeyboardInterrupt` in
+ # some python versions. Use `map_async` with an explicit timeout
+ # instead. See http://stackoverflow.com/a/1408476.
+ RESULTS.extend(
+ POOL.map_async(
+ run_test,
+ options_gen(
+ EXECUTABLE, OPTIONS.parallel, OPTIONS.jobs)).get(
+ timeout=sys.maxsize))
+
+ # Now run sequential tests.
+ if OPTIONS.sequential:
+ RESULTS.extend(
+ POOL.map_async(
+ run_test,
+ options_gen(
+ EXECUTABLE, OPTIONS.sequential, 1)).get(
+ timeout=sys.maxsize))
+
+ # Count the number of failed shards and print results from
+ # failed shards.
+ #
+ # NOTE: The `RESULTS` array stores the result for each
+ # `run_test` invocation returning a tuple (success, output).
+ NFAILED = len([success for success, __ in RESULTS if not success])
+
+ # TODO(bbannier): Introduce a verbosity which prints results
+ # as they arrive; this likely requires some output parsing to
+ # ensure results from different tests do not interleave.
+ for result in RESULTS:
+ if not result[0]:
+ if OPTIONS.verbosity > 0:
+ print(result[1], file=sys.stderr)
+ else:
+ if OPTIONS.verbosity > 1:
+ print(result[1], file=sys.stdout)
+
+ if NFAILED > 0:
+ print(Bcolors.colorize(
+ '\n[FAIL]: {nfailed} shard(s) have failed tests'.format(
+ nfailed=NFAILED),
+ Bcolors.FAIL, Bcolors.BOLD),
+ file=sys.stderr)
+ else:
+ print(Bcolors.colorize('\n[PASS]', Bcolors.OKGREEN, Bcolors.BOLD))
+
+ sys.exit(NFAILED)
+
+ except KeyboardInterrupt:
+ # Force a newline after intermediate test reports.
+ print()
+
+ print('Caught KeyboardInterrupt, terminating workers')
+
+ POOL.terminate()
+ POOL.join()
+
+ sys.exit(1)
+
+ except OSError as error:
+ print(Bcolors.colorize(
+ '\nERROR: {err}'.format(err=error),
+ Bcolors.FAIL, Bcolors.BOLD))
+
+ POOL.terminate()
+ POOL.join()
+
+ sys.exit(1)
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/mesos-split.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-split.py b/support/python3/mesos-split.py
new file mode 100755
index 0000000..0a77c25
--- /dev/null
+++ b/support/python3/mesos-split.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Errors if a list of files spans across
+the projects which make up mesos.
+"""
+
+from collections import defaultdict
+import sys
+
+if len(sys.argv) < 2:
+ print("Usage: ./mesos-split.py <filename>...")
+
+BASE_PROJECT = "mesos"
+
+SUBPROJECTS = {
+ "libprocess": "3rdparty/libprocess",
+ "stout": "3rdparty/stout"
+}
+
+ERROR = """ERROR: Commit spanning multiple projects.
+
+Please use separate commits for mesos, libprocess and stout.
+
+Paths grouped by project:"""
+
+
+def find_project(filename):
+ """Find a project using its filename."""
+
+ # Find longest prefix match.
+ found_path_len = 0
+ found_project = BASE_PROJECT
+ for project, path in SUBPROJECTS.items():
+ if filename.startswith(path) and len(path) > found_path_len:
+ found_path_len = len(path)
+ found_project = project
+
+ return found_project
+
+
+def main():
+ """
+ Expects a list of filenames on the command line.
+
+ See `support/hooks/pre-commit` for the canonical usage of this method.
+ """
+ touched_projects = defaultdict(list)
+ for filename in sys.argv[1:]:
+ touched_projects[find_project(filename)].append(filename)
+
+ if len(touched_projects) > 1:
+ print(ERROR)
+ for project in touched_projects.keys():
+ print("%s:" % project)
+ for filename in touched_projects[project]:
+ print(" %s" % filename)
+ sys.exit(1)
+
+ sys.exit(0)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-style.py b/support/python3/mesos-style.py
new file mode 100755
index 0000000..c0e918c
--- /dev/null
+++ b/support/python3/mesos-style.py
@@ -0,0 +1,521 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Runs checks for mesos style."""
+
+import os
+import re
+import string
+import subprocess
+import sys
+
+
+class LinterBase(object):
+ """
+ This is an abstract class that provides the base functionality for
+ linting files in the mesos project. Its 'main()' function
+ walks through the set of files passed to it and runs some
+ standard linting over them. This includes checking for license headers
+ and checking for non-supported characters. From there it calls a
+ 'run_lint()' function that can be overridden to provide
+ customizable style checks for a specific class of files (e.g. C++,
+ Python, etc.).
+
+ Any class that extends from 'LinterBase' should override the
+ following class variables / functions:
+
+ linter_type
+ source_dirs
+ exclude_files
+ source_files
+ comment_prefix
+
+ run_lint()
+
+ Please see the comments below for details on how to override each
+ variable.
+ """
+ # The name of the linter to help with printing which linter files
+ # are currently being processed by.
+ linter_type = ''
+
+ # Root source paths (will be traversed recursively).
+ source_dirs = []
+
+ # Add file paths and patterns which should not be checked
+ # This should include 3rdparty libraries, includes and machine generated
+ # source.
+ exclude_files = ''
+
+ # A regex of possible matches for your source files.
+ source_files = ''
+
+ # A prefix at the beginning of the line to demark comments (e.g. '//')
+ comment_prefix = ''
+
+ def check_encoding(self, source_paths):
+ """
+ Checks for encoding errors in the given files. Source
+ code files must contain only printable ascii characters.
+ This excludes the extended ascii characters 128-255.
+ http://www.asciitable.com/
+ """
+ error_count = 0
+ for path in source_paths:
+ with open(path) as source_file:
+ for line_number, line in enumerate(source_file):
+ # If we find an error, add 1 to both the character and
+ # the line offset to give them 1-based indexing
+ # instead of 0 (as is common in most editors).
+ char_errors = [offset for offset, char in enumerate(line)
+ if char not in string.printable]
+ if char_errors:
+ sys.stderr.write(
+ "{path}:{line_number}: Non-printable characters"
+ " found at [{offsets}]: \"{line}\"\n".format(
+ path=path,
+ line_number=line_number + 1,
+ offsets=', '.join([str(offset + 1) for offset
+ in char_errors]),
+ line=line.rstrip()))
+ error_count += 1
+
+ return error_count
+
+ def check_license_header(self, source_paths):
+ """Checks the license headers of the given files."""
+ error_count = 0
+ for path in source_paths:
+ with open(path) as source_file:
+ # We read the three first lines of the file as the
+ # first line could be a shebang and the second line empty.
+ head = "".join([next(source_file) for _ in range(3)])
+
+ # TODO(bbannier) We allow `Copyright` for
+ # currently deviating files. This should be
+ # removed one we have a uniform license format.
+ regex = r'^{comment_prefix} [Licensed|Copyright]'.format(
+ comment_prefix=self.comment_prefix)
+ # pylint: disable=no-member
+ regex = re.compile(regex, re.MULTILINE)
+
+ if not regex.search(head):
+ sys.stderr.write(
+ "{path}:1: A license header should appear's on one of"
+ " the first line of the file starting with"
+ " '{comment_prefix} Licensed'.: {head}".format(
+ path=path,
+ head=head,
+ comment_prefix=self.comment_prefix))
+ error_count += 1
+
+ return error_count
+
+ def find_candidates(self, root_dir):
+ """
+ Search through the all files rooted at 'root_dir' and compare
+ them against 'self.exclude_files' and 'self.source_files' to
+ come up with a set of candidate files to lint.
+ """
+ exclude_file_regex = re.compile(self.exclude_files)
+ source_criteria_regex = re.compile(self.source_files)
+ for root, _, files in os.walk(root_dir):
+ for name in files:
+ path = os.path.join(root, name)
+ if exclude_file_regex.search(path) is not None:
+ continue
+
+ if source_criteria_regex.search(name) is not None:
+ yield path
+
+ def run_command_in_virtualenv(self, command):
+ """
+ Activate the virtual environment, run the
+ given command and return its output.
+ """
+ virtualenv = os.path.join('support', '.virtualenv')
+ command = '. {virtualenv_path}/bin/activate; {cmd}'.format(
+ virtualenv_path=virtualenv, cmd=command)
+ process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+
+ return process
+
+ def run_lint(self, source_paths):
+ """
+ A custom function to provide linting for 'linter_type'.
+ It takes a list of source files to lint and returns the number
+ of errors found during the linting process.
+
+ It should print any errors as it encounters them to provide
+ feedback to the caller.
+ """
+ pass
+
+ def main(self, modified_files):
+ """
+ This function takes a list of files and lints them for the
+ class of files defined by 'linter_type'.
+ """
+
+ # Verify that source roots are accessible from current
+ # working directory. A common error could be to call
+ # the style checker from other (possibly nested) paths.
+ for source_dir in self.source_dirs:
+ if not os.path.exists(source_dir):
+ print("Could not find '{dir}'".format(dir=source_dir))
+ print('Please run from the root of the mesos source directory')
+ exit(1)
+
+ # Add all source file candidates to candidates list.
+ candidates = []
+ for source_dir in self.source_dirs:
+ for candidate in self.find_candidates(source_dir):
+ candidates.append(candidate)
+
+ # If file paths are specified, check all file paths that are
+ # candidates; else check all candidates.
+ file_paths = modified_files if modified_files else candidates
+
+ # Compute the set intersect of the input file paths and candidates.
+ # This represents the reduced set of candidates to run lint on.
+ candidates_set = set(candidates)
+ clean_file_paths_set = set(path.rstrip() for path in file_paths)
+ filtered_candidates_set = clean_file_paths_set.intersection(
+ candidates_set)
+
+ if filtered_candidates_set:
+ plural = '' if len(filtered_candidates_set) == 1 else 's'
+ print('Checking {num_files} {linter} file{plural}'.format(
+ num_files=len(filtered_candidates_set),
+ linter=self.linter_type,
+ plural=plural))
+
+ license_errors = self.check_license_header(filtered_candidates_set)
+ encoding_errors = self.check_encoding(list(filtered_candidates_set))
+ lint_errors = self.run_lint(list(filtered_candidates_set))
+ total_errors = license_errors + encoding_errors + lint_errors
+
+ sys.stderr.write('Total errors found: {num_errors}\n'.format(
+ num_errors=total_errors))
+
+ return total_errors
+
+ print("No {linter} files to lint".format(linter=self.linter_type))
+ return 0
+
+
+class CppLinter(LinterBase):
+ """The linter for C++ files, uses cpplint."""
+ linter_type = 'C++'
+
+ source_dirs = ['src',
+ 'include',
+ os.path.join('3rdparty', 'libprocess'),
+ os.path.join('3rdparty', 'stout')]
+
+ exclude_files = '(' \
+ r'elfio\-3\.2|' \
+ r'protobuf\-2\.4\.1|' \
+ r'googletest\-release\-1\.8\.0|' \
+ r'glog\-0\.3\.3|' \
+ r'boost\-1\.53\.0|' \
+ r'libev\-4\.15|' \
+ r'java/jni|' \
+ r'\.pb\.cc|\.pb\.h|\.md|\.virtualenv' \
+ ')'
+
+ source_files = r'\.(cpp|hpp|cc|h)$'
+
+ comment_prefix = r'\/\/'
+
+ def run_lint(self, source_paths):
+ """
+ Runs cpplint over given files.
+
+ http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
+ """
+
+ # See cpplint.py for full list of rules.
+ active_rules = [
+ 'build/class',
+ 'build/deprecated',
+ 'build/endif_comment',
+ 'build/nullptr',
+ 'readability/todo',
+ 'readability/namespace',
+ 'runtime/vlog',
+ 'whitespace/blank_line',
+ 'whitespace/comma',
+ 'whitespace/end_of_line',
+ 'whitespace/ending_newline',
+ 'whitespace/forcolon',
+ 'whitespace/indent',
+ 'whitespace/line_length',
+ 'whitespace/operators',
+ 'whitespace/semicolon',
+ 'whitespace/tab',
+ 'whitespace/comments',
+ 'whitespace/todo']
+
+ rules_filter = '--filter=-,+' + ',+'.join(active_rules)
+
+ # We do not use a version of cpplint available through pip as
+ # we use a custom version (see cpplint.path) to lint C++ files.
+ process = subprocess.Popen(
+ ['python3', 'support/python3/cpplint.py',
+ rules_filter] + source_paths,
+ stderr=subprocess.PIPE,
+ close_fds=True)
+
+ # Lines are stored and filtered, only showing found errors instead
+ # of e.g., 'Done processing XXX.' which tends to be dominant output.
+ for line in process.stderr:
+ if re.match('^(Done processing |Total errors found: )',
+ line.decode(sys.stdout.encoding)):
+ continue
+ sys.stderr.write(line)
+
+ process.wait()
+ return process.returncode
+
+
+class JsLinter(LinterBase):
+ """The linter for JavaScript files, uses eslint."""
+ linter_type = 'JavaScript'
+
+ source_dirs = [os.path.join('src', 'webui')]
+
+ exclude_files = '(' \
+ r'angular\-1\.2\.32|' \
+ r'angular\-route\-1\.2\.32|' \
+ r'bootstrap\-table\-1\.11\.1|' \
+ r'clipboard\-1\.5\.16|' \
+ r'jquery\-3\.2\.1|' \
+ r'relative\-date|' \
+ r'ui\-bootstrap\-tpls\-0\.9\.0|' \
+ r'angular\-route\-1\.2\.32|' \
+ r'underscore\-1\.4\.3' \
+ ')'
+
+ source_files = r'\.(js)$'
+
+ comment_prefix = '//'
+
+ def run_lint(self, source_paths):
+ """
+ Runs eslint over given files.
+
+ https://eslint.org/docs/user-guide/configuring
+ """
+
+ num_errors = 0
+
+ source_files = ' '.join(source_paths)
+ config_path = os.path.join('support', '.eslintrc.js')
+
+ process = self.run_command_in_virtualenv(
+ 'eslint {files} -c {config} -f compact'.format(
+ files=source_files,
+ config=config_path
+ )
+ )
+
+ for line in process.stdout:
+ line = line.decode(sys.stdout.encoding)
+ if "Error -" in line or "Warning -" in line:
+ sys.stderr.write(line)
+ if "Error -" in line:
+ num_errors += 1
+
+ return num_errors
+
+ def main(self, modified_files):
+ return super(JsLinter, self).main(modified_files)
+
+
+class PyLinter(LinterBase):
+ """The linter for Python files, uses pylint."""
+ linter_type = 'Python'
+
+ cli_dir = os.path.join('src', 'python', 'cli_new')
+ lib_dir = os.path.join('src', 'python', 'lib')
+ support_dir = os.path.join('support', 'python3')
+ source_dirs_to_lint_with_venv = [support_dir]
+ source_dirs_to_lint_with_tox = [cli_dir, lib_dir]
+ source_dirs = source_dirs_to_lint_with_tox + source_dirs_to_lint_with_venv
+
+ exclude_files = '(' \
+ r'protobuf\-2\.4\.1|' \
+ r'googletest\-release\-1\.8\.0|' \
+ r'glog\-0\.3\.3|' \
+ r'boost\-1\.53\.0|' \
+ r'libev\-4\.15|' \
+ r'java/jni|' \
+ r'\.virtualenv|' \
+ r'\.tox' \
+ ')'
+
+ source_files = r'\.(py)$'
+
+ comment_prefix = '#'
+
+ pylint_config = os.path.abspath(os.path.join('support', 'pylint.config'))
+
+ def run_tox(self, configfile, args, tox_env=None, recreate=False):
+ """
+ Runs tox with given configfile and args. Optionally set tox env
+ and/or recreate the tox-managed virtualenv.
+ """
+ cmd = ['tox']
+ cmd += ['-qq']
+ cmd += ['-c', configfile]
+ if tox_env is not None:
+ cmd += ['-e', tox_env]
+ if recreate:
+ cmd += ['--recreate']
+ cmd += ['--']
+ cmd += args
+
+ return subprocess.Popen(cmd, stdout=subprocess.PIPE)
+
+ def filter_source_files(self, source_dir, source_files):
+ """
+ Filters out files starting with source_dir.
+ """
+ return [f for f in source_files if f.startswith(source_dir)]
+
+ def lint_source_files_under_source_dir(self, source_dir, source_files):
+ """
+ Runs pylint directly or indirectly throgh tox on source_files which
+ are under source_dir. If tox is to be used, it must be configured
+ in source_dir, i.e. a tox.ini must be present.
+ """
+ filtered_source_files = self.filter_source_files(
+ source_dir, source_files)
+
+ if not filtered_source_files:
+ return 0
+
+ if source_dir in self.source_dirs_to_lint_with_tox:
+ process = self.run_tox(
+ configfile=os.path.join(source_dir, 'tox.ini'),
+ args=['--rcfile='+self.pylint_config] + filtered_source_files,
+ tox_env='py27-lint')
+ else:
+ process = self.run_command_in_virtualenv(
+ 'pylint --rcfile={rcfile} {files}'.format(
+ rcfile=self.pylint_config,
+ files=' '.join(filtered_source_files)))
+
+ num_errors = 0
+ for line in process.stdout:
+ line = line.decode(sys.stdout.encoding)
+ if re.match(r'^[RCWEF]: *[\d]+', line):
+ num_errors += 1
+ sys.stderr.write(line)
+
+ return num_errors
+
+ def run_lint(self, source_paths):
+ """
+ Runs pylint over given files.
+
+ https://google.github.io/styleguide/pyguide.html
+ """
+ num_errors = 0
+
+ for source_dir in self.source_dirs:
+ num_errors += self.lint_source_files_under_source_dir(
+ source_dir, source_paths)
+
+ return num_errors
+
+ def main(self, modified_files):
+ return super(PyLinter, self).main(modified_files)
+
+
+def should_build_virtualenv(modified_files):
+ """
+ Check if we should build the virtual environment required.
+ This is the case if the requirements of the environment
+ have changed or if the support script is run with no
+ arguments (meaning that the entire codebase should be linted).
+ """
+ # NOTE: If the file list is empty, we are linting the entire
+ # codebase. We should always rebuild the virtualenv in this case.
+ if not modified_files:
+ return True
+
+ basenames = [os.path.basename(path) for path in modified_files]
+
+ if 'pip-requirements.txt' in basenames:
+ print('The "pip-requirements.txt" file has changed.')
+ return True
+
+ if 'build-virtualenv' in basenames:
+ print('The "build-virtualenv" file has changed.')
+ return True
+
+ # The JS and Python linters require a virtual environment.
+ # If all the files modified are not JS or Python files,
+ # we do not need to build the virtual environment.
+ # TODO(ArmandGrillet): There should be no duplicated logic to know
+ # which linters to instantiate depending on the files to analyze.
+ if not os.path.isdir(os.path.join('support', '.virtualenv')):
+ js_and_python_files = [JsLinter().source_files, PyLinter().source_files]
+ js_and_python_files_regex = re.compile('|'.join(js_and_python_files))
+
+ for basename in basenames:
+ if js_and_python_files_regex.search(basename) is not None:
+ print('Virtualenv not detected and required... building')
+ return True
+
+ return False
+
+
+def build_virtualenv():
+ """
+ Rebuild the virtualenv by running a bootstrap script.
+ This will exit the program if there is a failure.
+ """
+ print('Rebuilding virtualenv...')
+
+ process = subprocess.Popen(
+ [os.path.join('support', 'build-virtualenv')],
+ stdout=subprocess.PIPE)
+
+ output = ''
+ for line in process.stdout:
+ output += line.decode(sys.stdout.encoding)
+
+ process.wait()
+
+ if process.returncode != 0:
+ sys.stderr.write(output)
+ sys.exit(1)
+
+if __name__ == '__main__':
+ # TODO(ArmandGrillet): We should only instantiate the linters
+ # required to lint the files to analyze. See MESOS-8351.
+ CPP_LINTER = CppLinter()
+ CPP_ERRORS = CPP_LINTER.main(sys.argv[1:])
+ JS_LINTER = JsLinter()
+ JS_ERRORS = JS_LINTER.main(sys.argv[1:])
+ PY_LINTER = PyLinter()
+ PY_ERRORS = PY_LINTER.main(sys.argv[1:])
+ sys.exit(CPP_ERRORS + JS_ERRORS + PY_ERRORS)
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/post-reviews.py
----------------------------------------------------------------------
diff --git a/support/python3/post-reviews.py b/support/python3/post-reviews.py
new file mode 100755
index 0000000..9621898
--- /dev/null
+++ b/support/python3/post-reviews.py
@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Wrapper around the post-review/rbt tool provided by Review Board.
+
+This script provides the ability to send one review for each
+commit on the current branch, instead of squashing all changes
+into a single large review. We encourage contributors to create
+logical commits which can be reviewed independently.
+
+Options to pass onto 'rbt' can be placed in a '.reviewboardrc'
+file at the top of the Mesos source directory. A default
+'.reviewboardrc' can be found at 'support/reviewboardrc'.
+Running './bootstrap' will populate this file for you.
+
+To use this script, first install 'RBTools' from Review Board:
+http://www.reviewboard.org/downloads/rbtools/
+
+$ cd /path/to/mesos
+$ [ do some work on your branch off of master, make commit(s) ]
+$ ./support/post-reviews.py
+"""
+# pylint: skip-file
+
+import argparse
+import atexit
+import imp
+import os
+import platform
+import re
+import sys
+import urllib.parse
+
+from distutils.version import LooseVersion
+
+from subprocess import check_output, Popen, PIPE, STDOUT
+
+
+def execute(command, ignore_errors=False):
+ """Execute a process and leave."""
+ process = None
+ try:
+ process = Popen(command,
+ stdin=PIPE,
+ stdout=PIPE,
+ stderr=STDOUT,
+ shell=False)
+ except Exception:
+ if not ignore_errors:
+ raise
+ return None
+
+ data, _ = process.communicate()
+ data = data.decode(sys.stdout.encoding)
+ status = process.wait()
+ if status != 0 and not ignore_errors:
+ cmdline = ' '.join(command) if isinstance(command, list) else command
+ need_login = 'Please log in to the Review Board' \
+ ' server at reviews.apache.org.'
+ if need_login in data:
+ print(need_login, '\n')
+ print("You can either:")
+ print(" (1) Run 'rbt login', or")
+ print(" (2) Set the default USERNAME/PASSWORD in '.reviewboardrc'")
+ else:
+ print('Failed to execute: \'' + cmdline + '\':')
+ print(data)
+ sys.exit(1)
+ elif status != 0:
+ return None
+ return data
+
+
+def main():
+ """Main function, post commits added to this branch as review requests."""
+ # TODO(benh): Make sure this is a git repository, apologize if not.
+
+ # Choose 'rbt' if available, otherwise choose 'post-review'.
+ post_review = None
+
+ rbt_command = 'rbt'
+ # Windows command name must have `cmd` extension.
+ if platform.system() == 'Windows':
+ rbt_command = 'rbt.cmd'
+
+ rbt_version = execute([rbt_command, '--version'], ignore_errors=True)
+ if rbt_version:
+ rbt_version = LooseVersion(rbt_version)
+ post_review = [rbt_command, 'post']
+ elif execute(['post-review', '--version'], ignore_errors=True):
+ post_review = ['post-review']
+ else:
+ print('Please install RBTools before proceeding')
+ sys.exit(1)
+
+ # Warn if people have unstaged changes.
+ diff_stat = execute(['git', 'diff', '--shortstat']).strip()
+
+ if diff_stat:
+ print('WARNING: Worktree contains unstaged changes, continuing anyway.', file=sys.stderr)
+
+ # Warn if people have uncommitted changes.
+ diff_stat = execute(['git', 'diff', '--shortstat', '--staged']).strip()
+
+ if diff_stat:
+ print('WARNING: Worktree contains staged but uncommitted changes, ' \
+ 'continuing anyway.', file=sys.stderr)
+
+ # Grab a reference to the repo's git directory. Usually this is simply
+ # .git in the repo's top level directory. However, when submodules are
+ # used, it may appear elsewhere. The most up-to-date way of finding this
+ # directory is to use `git rev-parse --git-common-dir`. This is necessary
+ # to support things like git worktree in addition to git submodules.
+ # However, as of January 2016, support for the '--git-common-dir' flag is
+ # fairly new, forcing us to fall back to the '--git-dir' flag if
+ # '--git-common-dir' is not supported. We do this by checking the output of
+ # git rev-parse --git-common-dir` and check if it gives a valid directory.
+ # If not, we set the git directory using the '--git-dir' flag instead.
+ git_dir = execute(['git', 'rev-parse', '--git-common-dir']).strip()
+ if not os.path.isdir(git_dir):
+ git_dir = execute(['git', 'rev-parse', '--git-dir']).strip()
+
+ # Grab a reference to the top level directory of this repo.
+ top_level_dir = execute(['git', 'rev-parse', '--show-toplevel']).strip()
+
+ # Use the tracking_branch specified by the user if exists.
+ parser = argparse.ArgumentParser(add_help=True)
+ parser.add_argument(
+ '--server',
+ help='Specifies the Review Board server to use.')
+ parser.add_argument(
+ '--no-markdown',
+ action='store_true',
+ help='Specifies if the commit text should not be treated as Markdown.')
+ parser.add_argument(
+ '--bugs-closed',
+ help='The comma-separated list of bug IDs closed.')
+ parser.add_argument(
+ '--target-people',
+ help='The usernames of the people who should perform the review.')
+ parser.add_argument(
+ '--tracking-branch',
+ help='The remote tracking branch from which your local branch is derived.')
+ args, _ = parser.parse_known_args()
+
+ # Try to read the .reviewboardrc in the top-level directory.
+ reviewboardrc_filepath = os.path.join(top_level_dir, '.reviewboardrc')
+ if os.path.exists(reviewboardrc_filepath):
+ # Prevent generation of '.reviewboardrcc'.
+ sys.dont_write_bytecode = True
+ reviewboardrc = imp.load_source('reviewboardrc', reviewboardrc_filepath)
+
+ if args.server:
+ reviewboard_url = args.server
+ elif 'REVIEWBOARD_URL' in dir(reviewboardrc):
+ reviewboard_url = reviewboardrc.REVIEWBOARD_URL
+ else:
+ reviewboard_url = 'https://reviews.apache.org'
+
+ if args.tracking_branch:
+ tracking_branch = args.tracking_branch
+ elif 'TRACKING_BRANCH' in dir(reviewboardrc):
+ tracking_branch = reviewboardrc.TRACKING_BRANCH
+ else:
+ tracking_branch = 'master'
+
+ branch_ref = execute(['git', 'symbolic-ref', 'HEAD']).strip()
+ branch = branch_ref.replace('refs/heads/', '', 1)
+
+ # Do not work on the tracking branch.
+ if branch == tracking_branch:
+ print("We're expecting you to be working on another branch" \
+ " from {}!".format(tracking_branch))
+ sys.exit(1)
+
+ temporary_branch = '_post-reviews_' + branch
+
+ # Always delete the temporary branch.
+ atexit.register(
+ lambda: execute(['git', 'branch', '-D', temporary_branch], True))
+
+ # Always put us back on the original branch.
+ atexit.register(lambda: execute(['git', 'checkout', branch]))
+
+ # Warn if the tracking branch is no direct ancestor of this review chain.
+ if execute([
+ 'git', 'merge-base', '--is-ancestor', tracking_branch, branch_ref],
+ ignore_errors=True) is None:
+ print("WARNING: Tracking branch '%s' is no direct ancestor of HEAD." \
+ " Did you forget to rebase?" % tracking_branch, file=sys.stderr)
+
+ try:
+ input("Press enter to continue or 'Ctrl-C' to abort.\n")
+ except KeyboardInterrupt:
+ sys.exit(0)
+
+ merge_base = execute(
+ ['git', 'merge-base', tracking_branch, branch_ref]).strip()
+
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C'
+ '(yellow)%d%Creset %s %Cgreen(%cr)%Creset',
+ merge_base + '..HEAD'])
+
+ print('Running \'%s\' across all of ...' % " ".join(post_review))
+ sys.stdout.buffer.write(output)
+
+ log = execute(['git',
+ '--no-pager',
+ 'log',
+ '--no-color',
+ '--pretty=oneline',
+ '--reverse',
+ merge_base + '..HEAD']).strip()
+
+ if len(log) <= 0:
+ print("No new changes compared with master branch!")
+ sys.exit(1)
+
+ shas = []
+
+ for line in log.split('\n'):
+ sha = line.split()[0]
+ shas.append(sha)
+
+ previous = merge_base
+ parent_review_request_id = None
+ for i, sha in enumerate(shas):
+ execute(['git', 'branch', '-D', temporary_branch], True)
+
+ message = execute(['git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%s%n%n%b',
+ previous + '..' + sha])
+
+ review_request_id = None
+
+ pos = message.find('Review:')
+ if pos != -1:
+ regex = 'Review: ({url})$'.format(
+ url=urllib.parse.urljoin(reviewboard_url, 'r/[0-9]+'))
+ pattern = re.compile(regex)
+ match = pattern.search(message[pos:].strip().strip('/'))
+ if match is None:
+ print("\nInvalid ReviewBoard URL: '{}'".format(message[pos:]))
+ sys.exit(1)
+
+ url = match.group(1)
+ review_request_id = url.split('/')[-1]
+
+ # Show the commit.
+ if review_request_id is None:
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C(yellow)%d%Creset %s',
+ previous + '..' + sha])
+ print('\nCreating diff of:')
+ sys.stdout.buffer.write(output)
+ else:
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C'
+ '(yellow)%d%Creset %s %Cgreen(%cr)%Creset',
+ previous + '..' + sha])
+ print('\nUpdating diff of:')
+ sys.stdout.buffer.write(output)
+
+ # Show the "parent" commit(s).
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C'
+ '(yellow)%d%Creset %s %Cgreen(%cr)%Creset',
+ tracking_branch + '..' + previous])
+
+ if output:
+ print('\n... with parent diff created from:')
+ sys.stdout.buffer.write(output)
+
+ try:
+ input('\nPress enter to continue or \'Ctrl-C\' to skip.\n')
+ except KeyboardInterrupt:
+ i = i + 1
+ previous = sha
+ parent_review_request_id = review_request_id
+ continue
+
+ # Strip the review url from the commit message, so that
+ # it is not included in the summary message when GUESS_FIELDS
+ # is set in .reviewboardc. Update the SHA appropriately.
+ if review_request_id:
+ stripped_message = message[:pos]
+ execute(['git', 'checkout', sha])
+ execute(['git', 'commit', '--amend', '-m', stripped_message])
+ sha = execute(['git', 'rev-parse', 'HEAD']).strip()
+ execute(['git', 'checkout', branch])
+
+ revision_range = previous + ':' + sha
+
+ # Build the post-review/rbt command up
+ # to the point where they are common.
+ command = post_review
+
+ if not args.no_markdown:
+ command = command + ['--markdown']
+
+ if args.bugs_closed:
+ command = command + ['--bugs-closed=' + args.bugs_closed]
+
+ if args.target_people:
+ command = command + ['--target-people=' + args.target_people]
+
+ if args.tracking_branch is None:
+ command = command + ['--tracking-branch=' + tracking_branch]
+
+ if review_request_id:
+ command = command + ['--review-request-id=' + review_request_id]
+
+ # Determine how to specify the revision range.
+ if rbt_command in post_review and \
+ rbt_version >= LooseVersion('RBTools 0.6'):
+ # rbt >= 0.6.1 supports '--depends-on' argument.
+ # Only set the "depends on" if this
+ # is not the first review in the chain.
+ if rbt_version >= LooseVersion('RBTools 0.6.1') and \
+ parent_review_request_id:
+ command = command + ['--depends-on=' + parent_review_request_id]
+
+ # rbt >= 0.6 revisions are passed in as args.
+ command = command + sys.argv[1:] + [previous, sha]
+ else:
+ # post-review and rbt < 0.6 revisions are
+ # passed in using the revision range option.
+ command = command + \
+ ['--revision-range=' + revision_range] + \
+ sys.argv[1:]
+
+ output = execute(command).strip()
+
+ sys.stdout.buffer.write(output)
+
+ # If we already have a request_id, continue on to the next commit in the
+ # chain. We update 'previous' from the shas[] array because we have
+ # overwritten the temporary sha variable above.
+ if review_request_id is not None:
+ previous = shas[i]
+ parent_review_request_id = review_request_id
+ i = i + 1
+ continue
+
+ # Otherwise, get the request_id from the output of post-review, append
+ # it to the commit message and rebase all other commits on top of it.
+ lines = output.split('\n')
+
+ # The last line of output in post-review is the review url.
+ # The second to the last line of output in rbt is the review url.
+ url = lines[len(lines) - 2] if rbt_command in post_review \
+ else lines[len(lines) - 1]
+
+ # Using rbt >= 0.6.3 on Linux prints out two URLs where the second
+ # one has /diff/ at the end. We want to remove this so that a
+ # subsequent call to post-reviews does not fail when looking up
+ # the reviewboard entry to edit.
+ url = url.replace('diff/', '')
+ url = url.strip('/')
+ review_request_id = os.path.basename(url)
+
+ # Construct new commit message.
+ message = message + '\n' + 'Review: ' + url + '\n'
+
+ execute(['git', 'checkout', '-b', temporary_branch])
+ execute(['git', 'reset', '--hard', sha])
+ execute(['git', 'commit', '--amend', '-m', message])
+
+ # Now rebase all remaining shas on top of this amended commit.
+ j = i + 1
+ old_sha = execute(
+ ['git', 'rev-parse', '--verify', temporary_branch]).strip()
+ previous = old_sha
+ while j < len(shas):
+ execute(['git', 'checkout', shas[j]])
+ execute(['git', 'rebase', temporary_branch])
+ # Get the sha for our detached HEAD.
+ new_sha = execute([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%H', '-n', '1', 'HEAD']).strip()
+ execute(['git',
+ 'update-ref',
+ 'refs/heads/' + temporary_branch,
+ new_sha,
+ old_sha])
+ old_sha = new_sha
+ shas[j] = new_sha
+ j = j + 1
+
+ # Okay, now update the actual branch to our temporary branch.
+ new_sha = old_sha
+ old_sha = execute(['git', 'rev-parse', '--verify', branch]).strip()
+ execute(['git', 'update-ref', 'refs/heads/' + branch, new_sha, old_sha])
+
+ i = i + 1
+ parent_review_request_id = review_request_id
+
+if __name__ == '__main__':
+ main()