You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2018/05/24 22:27:19 UTC
[1/6] mesos git commit: Added support script to check if Python >=
3.6 is available.
Repository: mesos
Updated Branches:
refs/heads/master 05594d69d -> ae4e7956a
Added support script to check if Python >= 3.6 is available.
Review: https://reviews.apache.org/r/67247/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7ecb2fef
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7ecb2fef
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7ecb2fef
Branch: refs/heads/master
Commit: 7ecb2fef2edf99a03188a871cfaf0f16ec71508a
Parents: 05594d6
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Thu May 24 14:58:00 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Thu May 24 14:58:00 2018 -0700
----------------------------------------------------------------------
support/check-python3.py | 56 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7ecb2fef/support/check-python3.py
----------------------------------------------------------------------
diff --git a/support/check-python3.py b/support/check-python3.py
new file mode 100644
index 0000000..4cb5365
--- /dev/null
+++ b/support/check-python3.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks if Python 3.6 is available on the machine.
+"""
+# pylint: disable=superfluous-parens
+
+import os
+import subprocess
+import sys
+
+
+def print_error():
+ """Prints a warning requesting to install Python 3.6."""
+ print("The support scripts will be upgraded to Python 3 by July 1st.")
+ print("Make sure to install Python 3.6 on your machine before.")
+
+if sys.version_info[0] < 3:
+ # On Windows, system-wide installations of Python 3.6 gives a tools called
+ # py and that we can use to know if Python 3 is installed.
+ if os.name == "nt":
+ PY = subprocess.call(["WHERE", "py"], stdout=open(os.devnull, "wb"))
+ else:
+ # We are not using Python 3 as python, let's check if python3 exists.
+ PY = subprocess.call(["which", "python3"],
+ stdout=open(os.devnull, "wb"))
+ if PY != 0:
+ print_error()
+ else:
+ # It does exist, let's check its version.
+ if os.name == "nt":
+ VERSION = subprocess.check_output("py -3 --version", shell=True)
+ else:
+ VERSION = subprocess.check_output("python3 --version", shell=True)
+ # x goes from 0 to 5 so that we can check for Python < 3.6.
+ for x in range(0, 6):
+ if "3.%d." % (x) in VERSION:
+ print_error()
+elif sys.version_info[1] < 6:
+ # python is by default Python 3 but it's < 3.6.
+ print_error()
[4/6] mesos git commit: Ported all support scripts to Python 3.
Posted by an...@apache.org.
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/push-commits.py
----------------------------------------------------------------------
diff --git a/support/python3/push-commits.py b/support/python3/push-commits.py
new file mode 100755
index 0000000..82a7004
--- /dev/null
+++ b/support/python3/push-commits.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is typically used by Mesos committers to push a locally applied
+review chain to ASF git repo and mark the reviews as submitted on ASF
+ReviewBoard.
+
+Example Usage:
+
+> git checkout master
+> git pull origin
+> ./support/python3/apply-reviews.py -c -r 1234
+> ./support/python3/push-commits.py
+"""
+
+# TODO(vinod): Also post the commit message to the corresponding ASF JIRA
+# tickets and resolve them if necessary.
+
+import argparse
+import os
+import re
+import sys
+
+from subprocess import check_output
+
+REVIEWBOARD_URL = 'https://reviews.apache.org'
+
+
+def get_reviews(revision_range):
+ """Return the list of reviews found in the commits in the revision range."""
+ reviews = [] # List of (review id, commit log) tuples
+
+ rev_list = check_output(['git',
+ 'rev-list',
+ '--reverse',
+ revision_range]).strip().split('\n')
+ for rev in rev_list:
+ commit_log = check_output(['git',
+ '--no-pager',
+ 'show',
+ '--no-color',
+ '--no-patch',
+ rev]).strip()
+
+ pos = commit_log.find('Review: ')
+ if pos != -1:
+ pattern = re.compile('Review: ({url})$'.format(
+ url=os.path.join(REVIEWBOARD_URL, 'r', '[0-9]+')))
+ match = pattern.search(commit_log.strip().strip('/'))
+ if match is None:
+ print("\nInvalid ReviewBoard URL: '{}'".format(
+ commit_log[pos:]))
+ sys.exit(1)
+
+ url = match.group(1)
+ reviews.append((os.path.basename(url), commit_log))
+
+ return reviews
+
+
+def close_reviews(reviews, options):
+ """Mark the given reviews as submitted on ReviewBoard."""
+ for review_id, commit_log in reviews:
+ print('Closing review', review_id)
+ if not options['dry_run']:
+ check_output(['rbt',
+ 'close',
+ '--description',
+ commit_log,
+ review_id])
+
+
+def parse_options():
+ """Return a dictionary of options parsed from command line arguments."""
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('-n',
+ '--dry-run',
+ action='store_true',
+ help='Perform a dry run.')
+
+ args = parser.parse_args()
+
+ options = {}
+ options['dry_run'] = args.dry_run
+
+ return options
+
+
+def main():
+ """Main function to push the commits in this branch as review requests."""
+ options = parse_options()
+
+ current_branch_ref = check_output(['git', 'symbolic-ref', 'HEAD']).strip()
+ current_branch = current_branch_ref.replace('refs/heads/', '', 1)
+
+ if current_branch != 'master':
+ print('Please run this script from master branch')
+ sys.exit(1)
+
+ remote_tracking_branch = check_output(['git',
+ 'rev-parse',
+ '--abbrev-ref',
+ 'master@{upstream}']).strip()
+
+ merge_base = check_output([
+ 'git',
+ 'merge-base',
+ remote_tracking_branch,
+ 'master']).strip()
+
+ if merge_base == current_branch_ref:
+ print('No new commits found to push')
+ sys.exit(1)
+
+ reviews = get_reviews(merge_base + ".." + current_branch_ref)
+
+ # Push the current branch to remote master.
+ remote = check_output(['git',
+ 'config',
+ '--get',
+ 'branch.master.remote']).strip()
+
+ print('Pushing commits to', remote)
+
+ if options['dry_run']:
+ check_output(['git',
+ 'push',
+ '--dry-run',
+ remote,
+ 'master:master'])
+ else:
+ check_output(['git',
+ 'push',
+ remote,
+ 'master:master'])
+
+ # Now mark the reviews as submitted.
+ close_reviews(reviews, options)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/test-upgrade.py
----------------------------------------------------------------------
diff --git a/support/python3/test-upgrade.py b/support/python3/test-upgrade.py
new file mode 100755
index 0000000..a1745bd
--- /dev/null
+++ b/support/python3/test-upgrade.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to test the upgrade path between two versions of Mesos."""
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+import time
+
+DEFAULT_PRINCIPAL = 'foo'
+DEFAULT_SECRET = 'bar'
+
+
+class Process(object):
+ """
+ Helper class to keep track of process lifecycles.
+
+ This class allows to start processes, capture their
+ output, and check their liveness during delays/sleep.
+ """
+
+ def __init__(self, args, environment=None):
+ """Initialize the Process."""
+ outfile = tempfile.mktemp()
+ fout = open(outfile, 'w')
+ print('Run %s, output: %s' % (args, outfile))
+
+ # TODO(nnielsen): Enable glog verbose logging.
+ self.process = subprocess.Popen(args,
+ stdout=fout,
+ stderr=subprocess.STDOUT,
+ env=environment)
+
+ def sleep(self, seconds):
+ """
+ Poll the process for the specified number of seconds.
+
+ If the process ends during that time, this method returns the process's
+ return value. If the process is still running after that time period,
+ this method returns `True`.
+ """
+ poll_time = 0.1
+ while seconds > 0:
+ seconds -= poll_time
+ time.sleep(poll_time)
+ poll = self.process.poll()
+ if poll != None:
+ return poll
+ return True
+
+ def __del__(self):
+ """Kill the Process."""
+ if self.process.poll() is None:
+ self.process.kill()
+
+
+class Agent(Process):
+ """Class representing an agent process."""
+
+ def __init__(self, path, work_dir, credfile):
+ """Initialize a Mesos agent by running mesos-slave.sh."""
+ Process.__init__(self, [os.path.join(path, 'bin', 'mesos-slave.sh'),
+ '--master=127.0.0.1:5050',
+ '--credential=' + credfile,
+ '--work_dir=' + work_dir,
+ '--resources=disk:2048;mem:2048;cpus:2'])
+
+
+class Master(Process):
+ """Class representing a master process."""
+
+ def __init__(self, path, work_dir, credfile):
+ """Initialize a Mesos master by running mesos-master.sh."""
+ Process.__init__(self, [os.path.join(path, 'bin', 'mesos-master.sh'),
+ '--ip=127.0.0.1',
+ '--work_dir=' + work_dir,
+ '--authenticate',
+ '--credentials=' + credfile,
+ '--roles=test'])
+
+
+# TODO(greggomann): Add support for multiple frameworks.
+class Framework(Process):
+ """Class representing a framework instance (the test-framework for now)."""
+
+ def __init__(self, path):
+ """Initialize a framework."""
+ # The test-framework can take these parameters as environment variables,
+ # but not as command-line parameters.
+ environment = {
+ # In Mesos 0.28.0, the `MESOS_BUILD_DIR` environment variable in the
+ # test framework was changed to `MESOS_HELPER_DIR`, and the '/src'
+ # subdirectory was added to the variable's path. Both are included
+ # here for backwards compatibility.
+ 'MESOS_BUILD_DIR': path,
+ 'MESOS_HELPER_DIR': os.path.join(path, 'src'),
+ # MESOS_AUTHENTICATE is deprecated in favor of
+ # MESOS_AUTHENTICATE_FRAMEWORKS, although 0.28.x still expects
+ # previous one, therefore adding both.
+ 'MESOS_AUTHENTICATE': '1',
+ 'MESOS_AUTHENTICATE_FRAMEWORKS': '1',
+ 'DEFAULT_PRINCIPAL': DEFAULT_PRINCIPAL,
+ 'DEFAULT_SECRET': DEFAULT_SECRET
+ }
+
+ Process.__init__(self, [os.path.join(path, 'src', 'test-framework'),
+ '--master=127.0.0.1:5050'], environment)
+
+
+def version(path):
+ """Get the Mesos version from the built executables."""
+ mesos_master_path = os.path.join(path, 'bin', 'mesos-master.sh')
+ process = subprocess.Popen([mesos_master_path, '--version'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ output, _ = process.communicate()
+ return_code = process.returncode
+ if return_code != 0:
+ return False
+
+ return output[:-1]
+
+
+def create_master(master_version, build_path, work_dir, credfile):
+ """Create a master using a specific version."""
+ print('##### Starting %s master #####' % master_version)
+ master = Master(build_path, work_dir, credfile)
+ if not master.sleep(0.5):
+ print('%s master exited prematurely' % master_version)
+ sys.exit(1)
+ return master
+
+
+def create_agent(agent_version, build_path, work_dir, credfile):
+ """Create an agent using a specific version."""
+ print('##### Starting %s agent #####' % agent_version)
+ agent = Agent(build_path, work_dir, credfile)
+ if not agent.sleep(0.5):
+ print('%s agent exited prematurely' % agent_version)
+ sys.exit(1)
+ return agent
+
+
+def test_framework(framework_version, build_path):
+ """Run a version of the test framework on a specified version of Mesos."""
+ print('##### Starting %s framework #####' % framework_version)
+ print('Waiting for %s framework to complete (10 sec max)...' % (
+ framework_version))
+ framework = Framework(build_path)
+ if framework.sleep(10) != 0:
+ print('%s framework failed' % framework_version)
+ sys.exit(1)
+
+
+# TODO(nnielsen): Add support for zookeeper and failover of master.
+# TODO(nnielsen): Add support for testing scheduler live upgrade/failover.
+def main():
+ """Main function to test the upgrade between two Mesos builds."""
+ parser = argparse.ArgumentParser(
+ description='Test upgrade path between two mesos builds')
+ parser.add_argument('--prev',
+ type=str,
+ help='Build path to mesos version to upgrade from',
+ required=True)
+
+ parser.add_argument('--next',
+ type=str,
+ help='Build path to mesos version to upgrade to',
+ required=True)
+ args = parser.parse_args()
+
+ # Get the version strings from the built executables.
+ prev_version = version(args.prev)
+ next_version = version(args.__next__)
+
+ if not prev_version or not next_version:
+ print('Could not get mesos version numbers')
+ sys.exit(1)
+
+ # Write credentials to temporary file.
+ credfile = tempfile.mktemp()
+ with open(credfile, 'w') as fout:
+ fout.write(DEFAULT_PRINCIPAL + ' ' + DEFAULT_SECRET)
+
+ # Create a work directory for the master.
+ master_work_dir = tempfile.mkdtemp()
+
+ # Create a work directory for the agent.
+ agent_work_dir = tempfile.mkdtemp()
+
+ print('Running upgrade test from %s to %s' % (prev_version, next_version))
+
+ print("""\
++--------------+----------------+----------------+---------------+
+| Test case | Framework | Master | Agent |
++--------------+----------------+----------------+---------------+
+| #1 | %s\t| %s\t | %s\t |
+| #2 | %s\t| %s\t | %s\t |
+| #3 | %s\t| %s\t | %s\t |
+| #4 | %s\t| %s\t | %s\t |
++--------------+----------------+----------------+---------------+
+
+NOTE: live denotes that master process keeps running from previous case.
+ """ % (prev_version, prev_version, prev_version,
+ prev_version, next_version, prev_version,
+ prev_version, next_version, next_version,
+ next_version, next_version, next_version))
+
+ # Test case 1.
+ master = create_master(prev_version, args.prev, master_work_dir, credfile)
+ agent = create_agent(prev_version, args.prev, agent_work_dir, credfile)
+ test_framework(prev_version, args.prev)
+
+ # Test case 2.
+ # NOTE: Need to stop and start the agent because standalone detector does
+ # not detect master failover.
+ agent.process.kill()
+ master.process.kill()
+ master = create_master(next_version, args.__next__, master_work_dir,
+ credfile)
+ agent = create_agent(prev_version, args.prev, agent_work_dir, credfile)
+ test_framework(prev_version, args.prev)
+
+ # Test case 3.
+ agent.process.kill()
+ agent = create_agent(next_version, args.__next__, agent_work_dir, credfile)
+ test_framework(prev_version, args.prev)
+
+ # Test case 4.
+ test_framework(next_version, args.__next__)
+
+ # Tests passed.
+ sys.exit(0)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/verify-reviews.py
----------------------------------------------------------------------
diff --git a/support/python3/verify-reviews.py b/support/python3/verify-reviews.py
new file mode 100755
index 0000000..2e92590
--- /dev/null
+++ b/support/python3/verify-reviews.py
@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to build and test (verify) reviews that are posted
+to ReviewBoard. The script is intended for use by automated "ReviewBots"
+that are run on ASF infrastructure (or by anyone that wishes to donate
+some compute power). For example, see 'support/jenkins/reviewbot.sh'.
+
+The script performs the following sequence:
+* A query grabs review IDs from Reviewboard.
+* In reverse order (most recent first), the script determines if the
+ review needs verification (if the review has been updated or changed
+ since the last run through this script).
+* For each review that needs verification:
+ * The review is applied (via 'support/python3/apply-reviews.py').
+ * Mesos is built and unit tests are run.
+ * The result is posted to ReviewBoard.
+"""
+
+import atexit
+import json
+import os
+import platform
+import subprocess
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from datetime import datetime
+
+REVIEWBOARD_URL = "https://reviews.apache.org"
+REVIEW_SIZE = 1000000 # 1 MB in bytes.
+
+# TODO(vinod): Use 'argparse' module.
+# Get the user and password from command line.
+if len(sys.argv) < 3:
+ print("Usage: ./verify-reviews.py <user>"
+ "<password> [num-reviews] [query-params]")
+ sys.exit(1)
+
+USER = sys.argv[1]
+PASSWORD = sys.argv[2]
+
+# Number of reviews to verify.
+NUM_REVIEWS = -1 # All possible reviews.
+if len(sys.argv) >= 4:
+ NUM_REVIEWS = int(sys.argv[3])
+
+# Unless otherwise specified consider pending review requests to Mesos updated
+# since 03/01/2014.
+GROUP = "mesos"
+LAST_UPDATED = "2014-03-01T00:00:00"
+QUERY_PARAMS = "?to-groups=%s&status=pending&last-updated-from=%s" \
+ % (GROUP, LAST_UPDATED)
+if len(sys.argv) >= 5:
+ QUERY_PARAMS = sys.argv[4]
+
+
+class ReviewError(Exception):
+ """Exception returned by post_review()."""
+ pass
+
+
+def shell(command):
+ """Run a shell command."""
+ print(command)
+ return subprocess.check_output(
+ command, stderr=subprocess.STDOUT, shell=True)
+
+
+HEAD = shell("git rev-parse HEAD")
+
+
+def api(url, data=None):
+ """Call the ReviewBoard API."""
+ try:
+ auth_handler = urllib.request.HTTPBasicAuthHandler()
+ auth_handler.add_password(
+ realm="Web API",
+ uri="reviews.apache.org",
+ user=USER,
+ passwd=PASSWORD)
+
+ opener = urllib.request.build_opener(auth_handler)
+ urllib.request.install_opener(opener)
+
+ return json.loads(urllib.request.urlopen(url, data=data).read())
+ except urllib.error.HTTPError as err:
+ print("Error handling URL %s: %s (%s)" % (url, err.reason, err.read()))
+ exit(1)
+ except urllib.error.URLError as err:
+ print("Error handling URL %s: %s" % (url, err.reason))
+ exit(1)
+
+
+def apply_review(review_id):
+ """Apply a review using the script apply-reviews.py."""
+ print("Applying review %s" % review_id)
+ shell("python support/python3/apply-reviews.py -n -r %s" % review_id)
+
+
+def apply_reviews(review_request, reviews):
+ """Apply multiple reviews at once."""
+ # If there are no reviewers specified throw an error.
+ if not review_request["target_people"]:
+ raise ReviewError("No reviewers specified. Please find a reviewer by"
+ " asking on JIRA or the mailing list.")
+
+ # If there is a circular dependency throw an error.`
+ if review_request["id"] in reviews:
+ raise ReviewError("Circular dependency detected for review %s."
+ "Please fix the 'depends_on' field."
+ % review_request["id"])
+ else:
+ reviews.append(review_request["id"])
+
+ # First recursively apply the dependent reviews.
+ for review in review_request["depends_on"]:
+ review_url = review["href"]
+ print("Dependent review: %s " % review_url)
+ apply_reviews(api(review_url)["review_request"], reviews)
+
+ # Now apply this review if not yet submitted.
+ if review_request["status"] != "submitted":
+ apply_review(review_request["id"])
+
+
+def post_review(review_request, message):
+ """Post a review on the review board."""
+ print("Posting review: %s" % message)
+
+ review_url = review_request["links"]["reviews"]["href"]
+ data = urllib.parse.urlencode({'body_top': message, 'public': 'true'})
+ api(review_url, data)
+
+
+@atexit.register
+def cleanup():
+ """Clean the git repository."""
+ try:
+ shell("git clean -fd")
+ shell("git reset --hard %s" % HEAD)
+ except subprocess.CalledProcessError as err:
+ print("Failed command: %s\n\nError: %s" % (err.cmd, err.output))
+
+
+def verify_review(review_request):
+ """Verify a review."""
+ print("Verifying review %s" % review_request["id"])
+ build_output = "build_" + str(review_request["id"])
+
+ try:
+ # Recursively apply the review and its dependents.
+ reviews = []
+ apply_reviews(review_request, reviews)
+
+ reviews.reverse() # Reviews are applied in the reverse order.
+
+ command = ""
+ if platform.system() == 'Windows':
+ command = "support\\windows-build.bat"
+
+ # There is no equivalent to `tee` on Windows.
+ subprocess.check_call(
+ ['cmd', '/c', '%s 2>&1 > %s' % (command, build_output)])
+ else:
+ # Launch docker build script.
+
+ # TODO(jojy): Launch 'docker_build.sh' in subprocess so that
+ # verifications can be run in parallel for various configurations.
+ configuration = ("export "
+ "OS='ubuntu:14.04' "
+ "BUILDTOOL='autotools' "
+ "COMPILER='gcc' "
+ "CONFIGURATION='--verbose "
+ "--disable-libtool-wrappers' "
+ "ENVIRONMENT='GLOG_v=1 MESOS_VERBOSE=1'")
+
+ command = "%s; ./support/docker-build.sh" % configuration
+
+ # `tee` the output so that the console can log the whole build
+ # output. `pipefail` ensures that the exit status of the build
+ # command ispreserved even after tee'ing.
+ subprocess.check_call(['bash', '-c',
+ ('set -o pipefail; %s 2>&1 | tee %s')
+ % (command, build_output)])
+
+ # Success!
+ post_review(
+ review_request,
+ "Patch looks great!\n\n" \
+ "Reviews applied: %s\n\n" \
+ "Passed command: %s" % (reviews, command))
+ except subprocess.CalledProcessError as err:
+ # If we are here because the docker build command failed, read the
+ # output from `build_output` file. For all other command failures read
+ # the output from `e.output`.
+ if os.path.exists(build_output):
+ output = open(build_output).read()
+ else:
+ output = err.output
+
+ if platform.system() == 'Windows':
+ # We didn't output anything during the build (because `tee`
+ # doesn't exist), so we print the output to stdout upon error.
+
+ # Pylint raises a no-member error on that line due to a bug
+ # fixed in pylint 1.7.
+ # TODO(ArmandGrillet): Remove this once pylint updated to >= 1.7.
+ # pylint: disable=no-member
+ sys.stdout.buffer.write(output)
+
+ # Truncate the output when posting the review as it can be very large.
+ if len(output) > REVIEW_SIZE:
+ output = "...<truncated>...\n" + output[-REVIEW_SIZE:]
+
+ output += "\nFull log: "
+ output += urllib.parse.urljoin(os.environ['BUILD_URL'], 'console')
+
+ post_review(
+ review_request,
+ "Bad patch!\n\n" \
+ "Reviews applied: %s\n\n" \
+ "Failed command: %s\n\n" \
+ "Error:\n%s" % (reviews, err.cmd, output))
+ except ReviewError as err:
+ post_review(
+ review_request,
+ "Bad review!\n\n" \
+ "Reviews applied: %s\n\n" \
+ "Error:\n%s" % (reviews, err.args[0]))
+
+ # Clean up.
+ cleanup()
+
+
+def needs_verification(review_request):
+ """Return True if this review request needs to be verified."""
+ print("Checking if review: %s needs verification" % review_request["id"])
+
+ # Skip if the review blocks another review.
+ if review_request["blocks"]:
+ print("Skipping blocking review %s" % review_request["id"])
+ return False
+
+ diffs_url = review_request["links"]["diffs"]["href"]
+ diffs = api(diffs_url)
+
+ if not diffs["diffs"]: # No diffs attached!
+ print("Skipping review %s as it has no diffs" % review_request["id"])
+ return False
+
+ # Get the timestamp of the latest diff.
+ timestamp = diffs["diffs"][-1]["timestamp"]
+ rb_date_format = "%Y-%m-%dT%H:%M:%SZ"
+ diff_time = datetime.strptime(timestamp, rb_date_format)
+ print("Latest diff timestamp: %s" % diff_time)
+
+ # Get the timestamp of the latest review from this script.
+ reviews_url = review_request["links"]["reviews"]["href"]
+ reviews = api(reviews_url + "?max-results=200")
+ review_time = None
+ for review in reversed(reviews["reviews"]):
+ if review["links"]["user"]["title"] == USER:
+ timestamp = review["timestamp"]
+ review_time = datetime.strptime(timestamp, rb_date_format)
+ print("Latest review timestamp: %s" % review_time)
+ break
+
+ # TODO: Apply this check recursively up the dependency chain.
+ changes_url = review_request["links"]["changes"]["href"]
+ changes = api(changes_url)
+ dependency_time = None
+ for change in changes["changes"]:
+ if "depends_on" in change["fields_changed"]:
+ timestamp = change["timestamp"]
+ dependency_time = datetime.strptime(timestamp, rb_date_format)
+ print("Latest dependency change timestamp: %s" % dependency_time)
+ break
+
+ # Needs verification if there is a new diff, or if the dependencies changed,
+ # after the last time it was verified.
+ return not review_time or review_time < diff_time or \
+ (dependency_time and review_time < dependency_time)
+
+
+def main():
+ """Main function to verify the submitted reviews."""
+ review_requests_url = \
+ "%s/api/review-requests/%s" % (REVIEWBOARD_URL, QUERY_PARAMS)
+
+ review_requests = api(review_requests_url)
+ num_reviews = 0
+ for review_request in reversed(review_requests["review_requests"]):
+ if (NUM_REVIEWS == -1 or num_reviews < NUM_REVIEWS) and \
+ needs_verification(review_request):
+ verify_review(review_request)
+ num_reviews += 1
+
+if __name__ == '__main__':
+ main()
[5/6] mesos git commit: Ported all support scripts to Python 3.
Posted by an...@apache.org.
Ported all support scripts to Python 3.
The scripts are in a temporary directory, support/python3.
The scripts have been ported using 2to3, the official tool to do so.
Many of these scripts require testing from the community before being
used by default.
The script building the virtual environment and the git hooks have
been updated to use the new scripts if the environment variable
`MESOSSUPPORTPYTHON` is set to `3` by the user.
Review: https://reviews.apache.org/r/67059/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/960df5c4
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/960df5c4
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/960df5c4
Branch: refs/heads/master
Commit: 960df5c48b6b04037aabc1c1f75506d82c629732
Parents: 71315eb
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Thu May 24 14:58:13 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Thu May 24 14:58:13 2018 -0700
----------------------------------------------------------------------
support/README.md | 4 +
support/build-virtualenv | 16 +-
support/hooks/post-rewrite | 14 +-
support/hooks/pre-commit | 14 +-
support/python3/apply-reviews.py | 453 +++++++++++++++++++++
support/python3/generate-endpoint-help.py | 403 +++++++++++++++++++
support/python3/jsonurl.py | 55 +++
support/python3/mesos-gtest-runner.py | 294 ++++++++++++++
support/python3/mesos-split.py | 79 ++++
support/python3/mesos-style.py | 521 +++++++++++++++++++++++++
support/python3/post-reviews.py | 432 ++++++++++++++++++++
support/python3/push-commits.py | 158 ++++++++
support/python3/test-upgrade.py | 254 ++++++++++++
support/python3/verify-reviews.py | 318 +++++++++++++++
14 files changed, 3007 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/README.md
----------------------------------------------------------------------
diff --git a/support/README.md b/support/README.md
index e812e96..0eae4d3 100644
--- a/support/README.md
+++ b/support/README.md
@@ -4,3 +4,7 @@ This directory contains various helper scripts.
The scripts directly in this directory are intended to be used by Mesos
developers on the command-line.
+
+**To use the Python 3 version of these scripts, set the environment
+variable `MESOS_SUPPORT_PYTHON` to `3` and run again the script
+`support/build-virtualenv`.**
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/build-virtualenv
----------------------------------------------------------------------
diff --git a/support/build-virtualenv b/support/build-virtualenv
index 850af89..5fda081 100755
--- a/support/build-virtualenv
+++ b/support/build-virtualenv
@@ -46,10 +46,18 @@ fi
PYTHON_MAJOR=$(${PYTHON} -c 'import sys; print(sys.version_info[0])')
PYTHON_MINOR=$(${PYTHON} -c 'import sys; print(sys.version_info[1])')
-if [ "${PYTHON_MAJOR}" != "2" ] || [ "${PYTHON_MINOR}" -lt "6" ]; then
- echo "You must be running python 2.6 or 2.7 in order to continue."
- echo "Consider running as 'PYTHON=python2 ./bootstrap' or similar."
- exit 1
+if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ if [ "${PYTHON_MAJOR}" != "3" ] || [ "${PYTHON_MINOR}" -lt "6" ]; then
+ echo "You must be running python 3.6 in order to continue."
+ echo "Consider running as 'PYTHON=python3 <previous-command>' or similar."
+ exit 1
+ fi
+else
+ if [ "${PYTHON_MAJOR}" != "2" ] || [ "${PYTHON_MINOR}" -lt "6" ]; then
+ echo "You must be running python 2.6 or 2.7 in order to continue."
+ echo "Consider running as 'PYTHON=python2 ./bootstrap' or similar."
+ exit 1
+ fi
fi
# Set up a virtual environment for the linters.
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/hooks/post-rewrite
----------------------------------------------------------------------
diff --git a/support/hooks/post-rewrite b/support/hooks/post-rewrite
index 1ab14ab..81f3454 100755
--- a/support/hooks/post-rewrite
+++ b/support/hooks/post-rewrite
@@ -30,11 +30,21 @@ if [ "$ADDED_OR_MODIFIED" ]; then
# many implementations do not support the `-r` flag, (which instructs
# `xargs` to not run the script if the arguments are empty), so we also
# cannot use that.
- ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ # TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+ if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ ./support/python3/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ else
+ ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ fi
fi
# Check that the commits are properly split between mesos, libprocess and stout.
## In git, '@' represent current head, '@~' represent previous commit. We check
## the style of changes between current head and previous commit after a commit
## is rewritten.
-git diff --name-only --diff-filter=AMD @~..@ | xargs ./support/mesos-split.py || exit 1
+# TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ git diff --name-only --diff-filter=AMD @~..@ | xargs ./support/python3/mesos-split.py || exit 1
+else
+ git diff --name-only --diff-filter=AMD @~..@ | xargs ./support/mesos-split.py || exit 1
+fi
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/hooks/pre-commit
----------------------------------------------------------------------
diff --git a/support/hooks/pre-commit b/support/hooks/pre-commit
index 6faba98..ac0ff24 100755
--- a/support/hooks/pre-commit
+++ b/support/hooks/pre-commit
@@ -33,8 +33,18 @@ if [ -n "$ADDED_OR_MODIFIED" ]; then
# many implementations do not support the `-r` flag, (which instructs
# `xargs` to not run the script if the arguments are empty), so we also
# cannot use that.
- ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ # TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+ if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ ./support/python3/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ else
+ ./support/mesos-style.py $ADDED_OR_MODIFIED || exit 1
+ fi
fi
# Check that the commits are properly split between mesos, libprocess and stout.
-git diff --cached --name-only --diff-filter=AMD | xargs ./support/mesos-split.py || exit 1
+# TODO(ArmandGrillet): Remove the if to really switch to Python 3.
+if [ "$MESOS_SUPPORT_PYTHON" = "3" ]; then
+ git diff --cached --name-only --diff-filter=AMD | xargs ./support/python3/mesos-split.py || exit 1
+else
+ git diff --cached --name-only --diff-filter=AMD | xargs ./support/mesos-split.py || exit 1
+fi
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/apply-reviews.py
----------------------------------------------------------------------
diff --git a/support/python3/apply-reviews.py b/support/python3/apply-reviews.py
new file mode 100755
index 0000000..bf72c80
--- /dev/null
+++ b/support/python3/apply-reviews.py
@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script translates reviews on ReviewBoard into commits
+on the current branch.
+"""
+
+import argparse
+import atexit
+import json
+import linecache
+import os
+import pipes
+import platform
+import re
+import ssl
+import subprocess
+import sys
+import urllib.request
+import urllib.error
+import urllib.parse
+
+
+REVIEWBOARD_REVIEW_URL = 'https://reviews.apache.org/r'
+REVIEWBOARD_API_URL =\
+ 'https://reviews.apache.org/api/review-requests'
+REVIEWBOARD_USER_URL = 'https://reviews.apache.org/api/users'
+
+
+GITHUB_URL = 'https://api.github.com/repos/apache/mesos/pulls'
+GITHUB_PATCH_URL =\
+ 'https://patch-diff.githubusercontent.com/raw/apache/mesos/pull'
+
+
+def review_api_url(review_id):
+ """Returns a Review Board API URL given a review ID."""
+ # Reviewboard REST API expects '/' at the end of the URL.
+ return '{base}/{review}/'.format(
+ base=REVIEWBOARD_API_URL,
+ review=review_id)
+
+
+def review_url(review_id):
+ """Returns a Review Board UI URL given a review ID."""
+ return '{base}/{review}/'.format(
+ base=REVIEWBOARD_REVIEW_URL,
+ review=review_id)
+
+
+def pull_request_url(pull_request_number):
+ """Returns a GitHub pull request URL given a PR number."""
+ return '{base}/{pr}'.format(
+ base=GITHUB_URL,
+ pr=pull_request_number)
+
+
+def reviewboard_user_url(username):
+ """Returns a Review Board URL for a user given a username."""
+ # Reviewboard REST API expects '/' at the end of the URL.
+ return '{base}/{user}/'.format(
+ base=REVIEWBOARD_USER_URL,
+ user=username)
+
+
+def patch_url(options):
+ """Returns a Review Board or a GitHub URL for a patch."""
+ if options['review_id']:
+ # Reviewboard REST API expects '/' at the end of the URL.
+ return '{base}/{review}/diff/raw/'.format(
+ base=REVIEWBOARD_REVIEW_URL,
+ review=options['review_id'])
+ elif options['github']:
+ return '{base}/{patch}.patch'.format(
+ base=GITHUB_PATCH_URL,
+ patch=options['github'])
+ return None
+
+
+def url_to_json(url):
+ """Performs HTTP request and returns JSON-ified response."""
+ json_str = urllib.request.urlopen(url)
+ return json.loads(json_str.read())
+
+
+def extract_review_id(url):
+ """Extracts review ID from Review Board URL."""
+ review_id = re.search(REVIEWBOARD_API_URL + r'/(\d+)/', url)
+ if review_id:
+ return review_id.group(1)
+ return ''
+
+def review_chain(review_id):
+ """Returns a parent review chain for a given review ID."""
+ json_obj = url_to_json(review_api_url(review_id))
+
+ # Stop as soon as we stumble upon a submitted request.
+ status = json_obj.get('review_request').get('status')
+ if status == "submitted":
+ sys.stderr.write('Warning: Review {review} has already'
+ ' been applied\n'.format(review=review_id))
+ return []
+
+ # Verify that the review has exactly one parent.
+ parent = json_obj.get('review_request').get('depends_on')
+ if len(parent) > 1:
+ sys.stderr.write('Error: Review {review} has more than'
+ ' one parent'.format(review=review_id))
+ sys.exit(1)
+ elif not parent:
+ return [(review_id, json_obj.get('review_request').get('summary'))]
+ else:
+ # The review has exactly one parent.
+ review_list = review_chain(extract_review_id(parent[0].get('href')))
+
+ review = (review_id, json_obj.get('review_request').get('summary'))
+ if review not in review_list:
+ return review_list + [review]
+ else:
+ sys.stderr.write('Found a circular dependency in the chain starting'
+ ' at {review}\n'.format(review=review_id))
+ sys.exit(1)
+
+
+def shell(command, dry_run):
+ """
+ Runs a command in a shell, unless the dry-run option
+ is set (in which case it just prints the command).
+ """
+ if dry_run:
+ print(command)
+ return
+
+ error_code = subprocess.call(command, stderr=subprocess.STDOUT, shell=True)
+ if error_code != 0:
+ sys.exit(error_code)
+
+
+def apply_review(options):
+ """Applies a review with a given ID locally."""
+ # Make sure we don't leave the patch behind in case of failure.
+ # We store the patch ID in a local variable to ensure the lambda
+ # captures the current patch ID.
+ patch_file = '%s.patch' % patch_id(options)
+ if not options["keep_patches"]:
+ atexit.register(
+ lambda: os.path.exists(patch_file) and os.remove(patch_file))
+
+ fetch_patch(options)
+ apply_patch(options)
+ commit_patch(options)
+
+
+def ssl_create_default_context():
+ """
+ Equivalent to `ssl.create_default_context` with default arguments and
+ certificate/hostname verification disabled.
+ See: https://github.com/python/cpython/blob/2.7/Lib/ssl.py#L410
+ This function requires Python >= 2.7.9.
+ """
+ # pylint: disable=no-member
+ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+
+ # SSLv2 considered harmful.
+ context.options |= ssl.OP_NO_SSLv2
+
+ # SSLv3 has problematic security and is only required for really old
+ # clients such as IE6 on Windows XP.
+ context.options |= ssl.OP_NO_SSLv3
+
+ # Disable compression to prevent CRIME attacks (OpenSSL 1.0+).
+ context.options |= getattr(ssl, "OP_NO_COMPRESSION", 0)
+
+ # Disable certificate and hostname verification.
+ context.verify_mode = ssl.CERT_NONE
+ context.check_hostname = False
+
+
+def fetch_patch(options):
+ """Fetches a patch from Review Board or GitHub."""
+ # pylint: disable=unexpected-keyword-arg
+ if platform.system() == 'Windows':
+ # This call requires Python >= 2.7.9.
+ response = urllib.request.urlopen(
+ patch_url(options),
+ context=ssl_create_default_context())
+
+ with open('%s.patch' % patch_id(options), 'wb') as patch:
+ patch.write(response.read())
+ else:
+ # NOTE: SSL contexts are only supported in Python 2.7.9+. The version
+ # of Python running on the non-Windows ASF CI machines is sometimes
+ # older. Hence, we fall back to `wget` on non-Windows machines.
+ cmd = ' '.join([
+ 'wget',
+ '--no-check-certificate',
+ '--no-verbose',
+ '-O '
+ '{review_id}.patch',
+ '{url}']).format(
+ review_id=patch_id(options),
+ url=patch_url(options))
+
+ # In case of GitHub we always need to fetch the patch to extract
+ # username and email, so we ignore the dry_run option by setting the
+ # second parameter to False.
+ if options['github']:
+ shell(cmd, False)
+ else:
+ shell(cmd, options['dry_run'])
+
+
+def patch_id(options):
+ """Returns the review ID or the GitHub pull request number."""
+ return options['review_id'] or options['github']
+
+
+def apply_patch(options):
+ """Applies patch locally."""
+ cmd = 'git apply --index {review_id}.patch'.format(
+ review_id=patch_id(options))
+
+ if options['3way']:
+ cmd += ' --3way'
+
+ if platform.system() == 'Windows':
+ # NOTE: Depending on the Git settings, there may or may not be
+ # carriage returns in files and in the downloaded patch.
+ # We ignore these errors on Windows.
+ cmd += ' --ignore-whitespace'
+
+ shell(cmd, options['dry_run'])
+
+
+def quote(string):
+ """Quote a variable so it can be safely used in shell."""
+ return string.replace("'", "'\\''")
+
+
+def commit_patch(options):
+ """Commits patch locally."""
+ data = patch_data(options)
+
+ # Check whether we need to amend the commit message.
+ if options['no_amend']:
+ amend = ''
+ else:
+ amend = '-e'
+
+ # Check whether we should skip the commit hooks.
+ if options['skip_hooks']:
+ verify = '-n'
+ else:
+ verify = ''
+
+ # NOTE: Windows does not support multi-line commit messages via the shell.
+ message_file = '%s.message' % patch_id(options)
+ atexit.register(
+ lambda: os.path.exists(message_file) and os.remove(message_file))
+
+ with open(message_file, 'w') as message:
+ # Add a shell command creating the message file for dry-run mode.
+ if options["dry_run"]:
+ shell(
+ "printf {msg} > {file}".format(
+ msg=pipes.quote(data['message']).replace('\n', '\\n'),
+ file=message_file),
+ True)
+ message.write(data['message'])
+
+ cmd = 'git commit' \
+ ' --author \"{author}\"' \
+ ' {amend} -aF \"{message}\"' \
+ ' {verify}'.format(
+ author=quote(data['author']),
+ amend=amend,
+ message=message_file,
+ verify=verify)
+
+ shell(cmd, options['dry_run'])
+
+
+def patch_data(options):
+ """
+ Populates and returns a dictionary with data necessary for
+ committing the patch (such as the message, the author, etc.).
+ """
+ if options['review_id']:
+ return reviewboard_data(options)
+ elif options['github']:
+ return github_data(options)
+ return None
+
+
+def get_author(patch):
+ """Reads the author name and email from the .patch file"""
+ author = linecache.getline(patch, 2)
+ return author.replace('From: ', '').rstrip()
+
+
+def github_data(options):
+ """Fetches pull request data and populates internal data structure."""
+ pull_request_number = options['github']
+ pull_request = url_to_json(pull_request_url(pull_request_number))
+
+ title = pull_request.get('title')
+ description = pull_request.get('body')
+ url = '{url}/{pr}'.format(url=GITHUB_URL, pr=pull_request_number)
+ author = get_author('{pr}.patch'.format(pr=pull_request_number))
+ message = '\n\n'.join([
+ title,
+ description,
+ 'This closes #{pr}'.format(pr=pull_request_number)])
+
+ review_data = {
+ "summary": title,
+ "description": description,
+ "url": url,
+ "author": author,
+ "message": message
+ }
+
+ return review_data
+
+
+def reviewboard_data(options):
+ """Fetches review data and populates internal data structure."""
+ review_id = options['review_id']
+
+ # Populate review object.
+ review = url_to_json(review_api_url(review_id)).get('review_request')
+
+ url = review_url(review_id)
+
+ # Populate user object.
+ user = url_to_json(reviewboard_user_url(
+ review.get('links').get('submitter').get('title'))).get('user')
+
+ # Only include a description if it is not identical to the summary.
+ message_data = [review.get('summary')]
+ if review.get('description') != review.get('summary'):
+ message_data.append(review.get('description'))
+ message_data.append('Review: {review_url}'.format(review_url=url))
+
+ author = '{author} <{email}>'.format(
+ author=user.get('fullname'),
+ email=user.get('email'))
+ message = '\n\n'.join(message_data)
+
+ review_data = {
+ "summary": review.get('summary'),
+ "description": review.get('description'),
+ "url": url,
+ "author": author,
+ "message": message
+ }
+
+ return review_data
+
+
+def parse_options():
+ """Parses command line options and returns an option dictionary."""
+ options = {}
+
+ parser = argparse.ArgumentParser(
+ description='Recursively apply Review Board reviews'
+ ' and GitHub pull requests.')
+
+ parser.add_argument('-d', '--dry-run',
+ action='store_true',
+ help='Perform a dry run.')
+ parser.add_argument('-k', '--keep-patches',
+ action='store_true',
+ help="Do not delete downloaded patch files.")
+ parser.add_argument('-n', '--no-amend',
+ action='store_true',
+ help='Do not amend commit message.')
+ parser.add_argument('-c', '--chain',
+ action='store_true',
+ help='Recursively apply parent review chain.')
+ parser.add_argument('-s', '--skip-hooks',
+ action='store_true',
+ help='Skip the commit hooks (e.g., Mesos style check).')
+ parser.add_argument('-3', '--3way',
+ dest='three_way',
+ action='store_true',
+ help='Use 3 way merge in git apply.')
+
+ # Add -g and -r and make them mutually exclusive.
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('-g', '--github',
+ metavar='PULL_REQUEST',
+ help='Pull request number')
+ group.add_argument('-r', '--review-id',
+ metavar='REVIEW_ID',
+ help='Numeric review ID')
+
+ args = parser.parse_args()
+
+ options['review_id'] = args.review_id
+ options['dry_run'] = args.dry_run
+ options['keep_patches'] = args.keep_patches
+ options['no_amend'] = args.no_amend
+ options['github'] = args.github
+ options['chain'] = args.chain
+ options['skip_hooks'] = args.skip_hooks
+ options['3way'] = args.three_way
+
+ return options
+
+
+def reviewboard(options):
+ """Applies either a chain of reviewboard patches or a single patch."""
+ if options['chain']:
+ # Retrieve the list of reviews to apply.
+ applied = set()
+ for review_id, _ in review_chain(options['review_id']):
+ if review_id not in applied:
+ applied.add(review_id)
+ options['review_id'] = review_id
+ apply_review(options)
+ else:
+ apply_review(options)
+
+
+def main():
+ """
+ Main function to apply reviews.
+ """
+ options = parse_options()
+
+ if options['review_id']:
+ reviewboard(options)
+ else:
+ apply_review(options)
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/generate-endpoint-help.py
----------------------------------------------------------------------
diff --git a/support/python3/generate-endpoint-help.py b/support/python3/generate-endpoint-help.py
new file mode 100755
index 0000000..711d3d3
--- /dev/null
+++ b/support/python3/generate-endpoint-help.py
@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Autogenerate documentation for all process endpoints spawned by a
+Mesos master and agent.
+"""
+
+import argparse
+import atexit
+import json
+import os
+import posixpath
+import re
+import shutil
+import subprocess
+import sys
+import time
+import urllib.request
+import urllib.error
+import urllib.parse
+
+
+# The host ip and master and agent ports.
+HOST_IP = "127.0.0.1"
+MASTER_PORT = 5050
+AGENT_PORT = 5051
+
+# The master and agent programs to launch.
+# We considered making the parameters to these commands something that
+# a user could specify on the command line, but ultimately chose to
+# hard code them. Different parameters may cause different endpoints
+# to become available, and we should modify this script to ensure that
+# we cover all of them instead of leaving that up to the user.
+MASTER_COMMAND = [
+ 'mesos-master.sh',
+ '--ip=%s' % (HOST_IP),
+ '--registry=in_memory',
+ '--work_dir=/tmp/mesos'
+]
+
+# NOTE: The agent flags here ensure that this script can run inside docker.
+AGENT_COMMAND = [
+ 'mesos-agent.sh',
+ '--master=%s:%s' % (HOST_IP, MASTER_PORT),
+ '--work_dir=/tmp/mesos',
+ '--systemd_enable_support=false',
+ '--launcher=posix'
+]
+
+
+# A header to add onto all generated markdown files.
+MARKDOWN_HEADER = """---
+title: %s
+layout: documentation
+---
+<!--- This is an automatically generated file. DO NOT EDIT! --->
+"""
+
+# A template of the title to add onto all generated markdown files.
+MARKDOWN_TITLE = "Apache Mesos - HTTP Endpoints%s"
+
+# A global timeout as well as a retry interval when hitting any http
+# endpoints on the master or agent (in seconds).
+RECEIVE_TIMEOUT = 10
+RETRY_INTERVAL = 0.10
+
+
+class Subprocess(object):
+ """The process running using this script."""
+ def __init__(self):
+ self.current = None
+
+ def cleanup(self):
+ """Kill the process running once the script is done."""
+ if self.current:
+ self.current.kill()
+
+
+# A pointer to the top level directory of the mesos project.
+GIT_TOP_DIR = subprocess.check_output(
+ ['git', 'rev-parse', '--show-cdup']).strip()
+
+with open(os.path.join(GIT_TOP_DIR, 'CHANGELOG'), 'r') as f:
+ if 'mesos' not in f.readline().lower():
+ print(('You must run this command from within'
+ ' the Mesos source repository!'), file=sys.stderr)
+ sys.exit(1)
+
+
+def parse_options():
+ """Parses command line options and populates the dictionary."""
+
+ options = {}
+
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawTextHelpFormatter,
+ description='Generate markdown files from all installed HTTP '
+ 'endpoints on a Mesos master and agent process.')
+
+ parser.add_argument(
+ '-c', '--command-path',
+ metavar='COMMAND_PATH',
+ default=os.path.join(GIT_TOP_DIR, "build/bin"),
+ help='Path to the Mesos master and agent commands.\n'
+ '(default: %(default)s)')
+
+ parser.add_argument(
+ '-o', '--output-path',
+ metavar='OUTPUT_PATH',
+ default=os.path.join(GIT_TOP_DIR, "docs/endpoints"),
+ help='Path to the top level directory where all\n'
+ 'generated markdown files will be placed.\n'
+ '(default: %(default)s)')
+
+ args = parser.parse_args()
+
+ options['command_path'] = args.command_path
+ options['output_path'] = args.output_path
+
+ return options
+
+
+def get_url_until_success(url):
+ """Continuously tries to open a url until it succeeds or times out."""
+ time_spent = 0
+ while time_spent < RECEIVE_TIMEOUT:
+ try:
+ helps = urllib.request.urlopen(url)
+ break
+ except Exception:
+ time.sleep(RETRY_INTERVAL)
+ time_spent += RETRY_INTERVAL
+
+ if time_spent >= RECEIVE_TIMEOUT:
+ print('Timeout attempting to hit url: %s' % (url), file=sys.stderr)
+ sys.exit(1)
+
+ return helps.read()
+
+
+def get_help(ip, port):
+ """
+ Grabs the help strings for all endpoints at http://ip:port as a JSON object.
+ """
+ url = 'http://%s:%d/help?format=json' % (ip, port)
+ return json.loads(get_url_until_success(url))
+
+
+def generalize_endpoint_id(p_id):
+ """Generalizes the id of the form e.g. process(1) to process(id)."""
+ return re.sub(r'\([0-9]+\)', '(id)', p_id)
+
+
+def normalize_endpoint_id(p_id):
+ """Normalizes the id of the form e.g. process(id) to process."""
+ return re.sub(r'\([0-9]+\)', '', p_id)
+
+
+def get_endpoint_path(p_id, name):
+ """
+ Generates the canonical endpoint path, given id and name.
+
+ Examples: ('process', '/') -> '/process'
+ ('process(id)', '/') -> '/process(id)'
+ ('process', '/endpoint') -> '/process/endpoint'
+ """
+ # Tokenize the endpoint by '/' (filtering
+ # out any empty strings between '/'s)
+ path_parts = [_f for _f in name.split('/') if _f]
+
+ # Conditionally prepend the 'id' to the list of path parts.
+ # Following the notion of a 'delegate' in Mesos, we want our
+ # preferred endpoint paths for the delegate process to be
+ # '/endpoint' instead of '/process/endpoint'. Since this script only
+ # starts 1 master and 1 agent, our only delegate processes are
+ # "master" and "slave(id)". If the id matches one of these, we don't
+ # prepend it, otherwise we do.
+ p_id = generalize_endpoint_id(p_id)
+ delegates = ["master", "slave(id)"]
+ if p_id not in delegates:
+ path_parts = [p_id] + path_parts
+
+ return posixpath.join('/', *path_parts)
+
+
+def get_relative_md_path(p_id, name):
+ """
+ Generates the relative path of the generated .md file from id and name.
+
+ This path is relative to the options['output_path'] directory.
+
+ Examples: master/health.md
+ master/maintenance/schedule.md
+ registrar/registry.md
+ version.md
+ """
+ new_id = normalize_endpoint_id(p_id)
+ # Strip the leading slash
+ new_name = name[1:]
+
+ if new_name:
+ return os.path.join(new_id, new_name + '.md')
+ return os.path.join(new_id + '.md')
+
+
+def write_markdown(path, output, title):
+ """Writes 'output' to the file at 'path'."""
+ print('generating: %s' % (path))
+
+ dirname = os.path.dirname(path)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+ outfile = open(path, 'w+')
+
+ # Add our header and remove all '\n's at the end of the output if
+ # there are any.
+ output = (MARKDOWN_HEADER % title) + '\n' + output.rstrip()
+
+ outfile.write(output)
+ outfile.close()
+
+
+def dump_index_markdown(master_help, agent_help, options):
+ """
+ Dumps an index for linking to the master and agent help files.
+
+ This file is dumped into a directory rooted at
+ options['output_path'].
+ """
+
+ # The output template for the HTTP endpoints index.
+ # We use a helper function below to insert text into the '%s' format
+ # strings contained in the "Master Endpoints" and "Agent Endpoints"
+ # sections of this template.
+ output = """# HTTP Endpoints #
+
+Below is a list of HTTP endpoints available for a given Mesos process.
+
+Depending on your configuration, some subset of these endpoints will be
+available on your Mesos master or agent. Additionally, a `/help`
+endpoint will be available that displays help similar to what you see
+below.
+
+** NOTE: ** If you are using Mesos 1.1 or later, we recommend using the
+new [v1 Operator HTTP API](../operator-http-api.md) instead of the
+unversioned REST endpoints listed below. These endpoints will be
+deprecated in the future.
+
+
+** NOTE: ** The documentation for these endpoints is auto-generated from
+the Mesos source code. See `support/python3/generate-endpoint-help.py`.
+
+## Master Endpoints ##
+
+Below are the endpoints that are available on a Mesos master. These
+endpoints are reachable at the address `http://ip:port/endpoint`.
+
+For example, `http://master.com:5050/files/browse`.
+
+%s
+
+## Agent Endpoints ##
+
+Below are the endpoints that are available on a Mesos agent. These
+endpoints are reachable at the address `http://ip:port/endpoint`.
+
+For example, `http://agent.com:5051/files/browse`.
+
+%s
+"""
+
+ def generate_links(master_or_agent_help):
+ """
+ Iterates over the input JSON and creates a list of links to
+ to the markdown files generated by this script. These links
+ are grouped per process, with the process's name serving as a
+ header for each group. All links are relative to
+ options['output_path'].
+
+ For example:
+ ### profiler ###
+ * [/profiler/start] (profiler/start.md)
+ * [/profiler/stop] (profiler/stop.md)
+
+ ### version ###
+ * [/version] (version.md)
+ """
+ output = ""
+ for process in master_or_agent_help['processes']:
+ p_id = process['id']
+ output += '### %s ###\n' % (generalize_endpoint_id(p_id))
+ for endpoint in process['endpoints']:
+ name = endpoint['name']
+ output += '* [%s](%s)\n' % (get_endpoint_path(p_id, name),
+ get_relative_md_path(p_id, name))
+ output += '\n'
+
+ # Remove any trailing newlines
+ return output.rstrip()
+
+ output = output % (generate_links(master_help),
+ generate_links(agent_help))
+
+ path = os.path.join(options['output_path'], 'index.md')
+ write_markdown(path, output, MARKDOWN_TITLE % "")
+
+
+def dump_markdown(master_or_agent_help, options):
+ """
+ Dumps JSON encoded help strings into markdown files.
+
+ These files are dumped into a directory rooted at
+ options['output_path'].
+ """
+ for process in master_or_agent_help['processes']:
+ p_id = process['id']
+ for endpoint in process['endpoints']:
+ name = endpoint['name']
+ text = endpoint['text']
+ title = get_endpoint_path(p_id, name)
+
+ relative_path = get_relative_md_path(p_id, name)
+ path = os.path.join(options['output_path'], relative_path)
+ write_markdown(path, text, MARKDOWN_TITLE % (" - " + title))
+
+
+def start_master(options):
+ """
+ Starts the Mesos master using the specified command.
+
+ This method returns the Popen object used to start it so it can
+ be killed later on.
+ """
+ cmd = os.path.join('.', options['command_path'], MASTER_COMMAND[0])
+ master = subprocess.Popen([cmd] + MASTER_COMMAND[1:])
+
+ # Wait for the master to become responsive
+ get_url_until_success("http://%s:%d/health" % (HOST_IP, MASTER_PORT))
+ return master
+
+
+def start_agent(options):
+ """
+ Starts the Mesos agent using the specified command.
+
+ This method returns the Popen object used to start it so it can
+ be killed later on.
+ """
+ cmd = os.path.join('.', options['command_path'], AGENT_COMMAND[0])
+ agent = subprocess.Popen([cmd] + AGENT_COMMAND[1:])
+
+ # Wait for the agent to become responsive.
+ get_url_until_success('http://%s:%d/health' % (HOST_IP, AGENT_PORT))
+ return agent
+
+
+def main():
+ """
+ Called when the Python script is used, we do not directly write code
+ after 'if __name__ == "__main__"' as we cannot set variables in that case.
+ """
+ # A dictionary of the command line options passed in.
+ options = parse_options()
+
+ # A pointer to the current subprocess for the master or agent.
+ # This is useful for tracking the master or agent subprocesses so
+ # that we can kill them if the script exits prematurely.
+ subproc = Subprocess()
+ atexit.register(subproc.cleanup())
+ subproc.current = start_master(options)
+ master_help = get_help(HOST_IP, MASTER_PORT)
+ subproc.current.kill()
+
+ subproc.current = start_agent(options)
+ agent_help = get_help(HOST_IP, AGENT_PORT)
+ subproc.current.kill()
+
+ shutil.rmtree(options['output_path'], ignore_errors=True)
+ os.makedirs(options['output_path'])
+
+ dump_index_markdown(master_help, agent_help, options)
+ dump_markdown(master_help, options)
+ dump_markdown(agent_help, options)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/jsonurl.py
----------------------------------------------------------------------
diff --git a/support/python3/jsonurl.py b/support/python3/jsonurl.py
new file mode 100755
index 0000000..ffa3e36
--- /dev/null
+++ b/support/python3/jsonurl.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A utility that expects JSON data at a particular URL and lets you
+recursively extract keys from the JSON object as specified on the
+command line (each argument on the command line after the first will
+be used to recursively index into the JSON object). The name is a
+play off of 'curl'.
+"""
+
+import json
+import sys
+import urllib.request
+import urllib.error
+import urllib.parse
+
+
+def main():
+ """Expects at least one argument on the command line."""
+ if len(sys.argv) < 2:
+ print("USAGE: {} URL [KEY...]".format(sys.argv[0]), file=sys.stderr)
+ sys.exit(1)
+
+ url = sys.argv[1]
+
+ data = json.loads(urllib.request.urlopen(url).read())
+
+ for arg in sys.argv[2:]:
+ try:
+ temp = data[arg]
+ data = temp
+ except KeyError:
+ print("'" + arg + "' was not found", file=sys.stderr)
+ sys.exit(1)
+
+ print(data.encode("utf-8"))
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/mesos-gtest-runner.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-gtest-runner.py b/support/python3/mesos-gtest-runner.py
new file mode 100755
index 0000000..153ded2
--- /dev/null
+++ b/support/python3/mesos-gtest-runner.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+Parallel test runner for GoogleTest programs.
+
+This script allows one to execute GoogleTest tests in parallel.
+GoogleTest programs come with built-in support for running in parallel.
+Here tests can automatically be partitioned across a number of test
+program invocations ("shards"). This script provides a convenient
+wrapper around that functionality and stream-lined output.
+"""
+
+import argparse
+import multiprocessing
+import os
+import shlex
+import signal
+import subprocess
+import sys
+
+
+DEFAULT_NUM_JOBS = int(multiprocessing.cpu_count() * 1.5)
+
+
+class Bcolors(object):
+ """
+ A collection of tty output modifiers.
+
+ To switch the output of a string, prefix it with the desired
+ modifier, and terminate it with 'ENDC'.
+ """
+
+ HEADER = '\033[95m' if sys.stdout.isatty() else ''
+ OKBLUE = '\033[94m' if sys.stdout.isatty() else ''
+ OKGREEN = '\033[92m' if sys.stdout.isatty() else ''
+ WARNING = '\033[93m' if sys.stdout.isatty() else ''
+ FAIL = '\033[91m'if sys.stdout.isatty() else ''
+ ENDC = '\033[0m' if sys.stdout.isatty() else ''
+ BOLD = '\033[1m' if sys.stdout.isatty() else ''
+ UNDERLINE = '\033[4m' if sys.stdout.isatty() else ''
+
+ @staticmethod
+ def colorize(string, *color_codes):
+ """Decorate a string with a number of color codes."""
+ colors = ''.join(color_codes)
+ return '{begin}{string}{end}'.format(
+ begin=colors if sys.stdout.isatty() else '',
+ string=string,
+ end=Bcolors.ENDC if sys.stdout.isatty() else '')
+
+
+def run_test(opts):
+ """
+ Perform an actual run of the test executable.
+
+ Expects a list of parameters giving the number of the current
+ shard, the total number of shards, and the executable to run.
+ """
+ shard, nshards, executable = opts
+
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+ env = os.environ.copy()
+ env['GTEST_TOTAL_SHARDS'] = str(nshards)
+ env['GTEST_SHARD_INDEX'] = str(shard)
+
+ try:
+ output = subprocess.check_output(
+ executable.split(),
+ stderr=subprocess.STDOUT,
+ env=env,
+ universal_newlines=True)
+ print(Bcolors.colorize('.', Bcolors.OKGREEN), end='')
+ sys.stdout.flush()
+ return True, output
+ except subprocess.CalledProcessError as error:
+ print(Bcolors.colorize('.', Bcolors.FAIL), end='')
+ sys.stdout.flush()
+ return False, error.output
+
+
+def parse_arguments():
+ """Return the executable to work on, and a list of options."""
+ parser = argparse.ArgumentParser(
+ usage='Usage: %prog [options] <test> [-- <test_options>]')
+
+ parser.add_argument(
+ '-j', '--jobs', type='int',
+ default=DEFAULT_NUM_JOBS,
+ help='number of parallel jobs to spawn. DEFAULT: {default_}'
+ .format(default_=DEFAULT_NUM_JOBS))
+
+ parser.add_argument(
+ '-s', '--sequential', type='string',
+ default='',
+ help='gtest filter for tests to run sequentially')
+
+ parser.add_argument(
+ '-v', '--verbosity', type='int',
+ default=1,
+ help='output verbosity:'
+ ' 0 only shows summarized information,'
+ ' 1 also shows full logs of failed shards, and anything'
+ ' >1 shows all output. DEFAULT: 1')
+
+ parser.epilog = (
+ 'The environment variable MESOS_GTEST_RUNNER_FLAGS '
+ 'can be used to set a default set of flags. Flags passed on the '
+ 'command line always have precedence over these defaults.')
+
+ # If the environment variable `MESOS_GTEST_RUNNER_FLAGS` is set we
+ # use it to set a default set of flags to pass. Flags passed on
+ # the command line always have precedence over these defaults.
+ #
+ # We manually construct `args` here and make use of the fact that
+ # in `optparser`'s implementation flags passed later on the
+ # command line overrule identical flags passed earlier.
+ args = []
+ if 'MESOS_GTEST_RUNNER_FLAGS' in os.environ:
+ args.extend(shlex.split(os.environ['MESOS_GTEST_RUNNER_FLAGS']))
+ args.extend(sys.argv[1:])
+
+ (options, executable) = parser.parse_args(args)
+
+ if not executable:
+ parser.print_usage()
+ sys.exit(1)
+
+ if not os.path.isfile(executable[0]):
+ print(
+ Bcolors.colorize(
+ "ERROR: File '{file}' does not exists"
+ .format(file=executable[0]), Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ if not os.access(executable[0], os.X_OK):
+ print(
+ Bcolors.colorize(
+ "ERROR: File '{file}' is not executable"
+ .format(file=executable[0]), Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ if options.sequential and options.sequential.count(':-'):
+ print(
+ Bcolors.colorize(
+ "ERROR: Cannot use negative filters in "
+ "'sequential' parameter: '{filter}'"
+ .format(filter=options.sequential), Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ if options.sequential and os.environ.get('GTEST_FILTER') and \
+ os.environ['GTEST_FILTER'].count(':-'):
+ print(
+ Bcolors.colorize(
+ "ERROR: Cannot specify both 'sequential' ""option "
+ "and environment variable 'GTEST_FILTER' "
+ "containing negative filters",
+ Bcolors.FAIL),
+ file=sys.stderr)
+ sys.exit(1)
+
+ # Since empty strings are falsy, directly compare against `None`
+ # to preserve an empty string passed via `GTEST_FILTER`.
+ if os.environ.get('GTEST_FILTER') != None:
+ options.parallel = '{env_filter}:-{sequential_filter}'\
+ .format(env_filter=os.environ['GTEST_FILTER'],
+ sequential_filter=options.sequential)
+ else:
+ options.parallel = '*:-{sequential_filter}'\
+ .format(sequential_filter=options.sequential)
+
+ return executable, options
+
+
+if __name__ == '__main__':
+ EXECUTABLE, OPTIONS = parse_arguments()
+
+ def options_gen(executable, filter_, jobs):
+ """Generator for options for a certain shard.
+
+ Here we set up GoogleTest specific flags, and generate
+ distinct shard indices.
+ """
+ opts = list(range(jobs))
+
+ # If we run in a terminal, enable colored test output. We
+ # still allow users to disable this themselves via extra args.
+ if sys.stdout.isatty():
+ args = executable[1:]
+ executable = '{exe} --gtest_color=yes {args}'\
+ .format(exe=executable[0], args=args if args else '')
+
+ if filter_:
+ executable = '{exe} --gtest_filter={filter}'\
+ .format(exe=executable, filter=filter_)
+
+ for opt in opts:
+ yield opt, jobs, executable
+
+ try:
+ RESULTS = []
+
+ POOL = multiprocessing.Pool(processes=OPTIONS.jobs)
+
+ # Run parallel tests.
+ #
+ # Multiprocessing's `map` cannot properly handle `KeyboardInterrupt` in
+ # some python versions. Use `map_async` with an explicit timeout
+ # instead. See http://stackoverflow.com/a/1408476.
+ RESULTS.extend(
+ POOL.map_async(
+ run_test,
+ options_gen(
+ EXECUTABLE, OPTIONS.parallel, OPTIONS.jobs)).get(
+ timeout=sys.maxsize))
+
+ # Now run sequential tests.
+ if OPTIONS.sequential:
+ RESULTS.extend(
+ POOL.map_async(
+ run_test,
+ options_gen(
+ EXECUTABLE, OPTIONS.sequential, 1)).get(
+ timeout=sys.maxsize))
+
+ # Count the number of failed shards and print results from
+ # failed shards.
+ #
+ # NOTE: The `RESULTS` array stores the result for each
+ # `run_test` invocation returning a tuple (success, output).
+ NFAILED = len([success for success, __ in RESULTS if not success])
+
+ # TODO(bbannier): Introduce a verbosity which prints results
+ # as they arrive; this likely requires some output parsing to
+ # ensure results from different tests do not interleave.
+ for result in RESULTS:
+ if not result[0]:
+ if OPTIONS.verbosity > 0:
+ print(result[1], file=sys.stderr)
+ else:
+ if OPTIONS.verbosity > 1:
+ print(result[1], file=sys.stdout)
+
+ if NFAILED > 0:
+ print(Bcolors.colorize(
+ '\n[FAIL]: {nfailed} shard(s) have failed tests'.format(
+ nfailed=NFAILED),
+ Bcolors.FAIL, Bcolors.BOLD),
+ file=sys.stderr)
+ else:
+ print(Bcolors.colorize('\n[PASS]', Bcolors.OKGREEN, Bcolors.BOLD))
+
+ sys.exit(NFAILED)
+
+ except KeyboardInterrupt:
+ # Force a newline after intermediate test reports.
+ print()
+
+ print('Caught KeyboardInterrupt, terminating workers')
+
+ POOL.terminate()
+ POOL.join()
+
+ sys.exit(1)
+
+ except OSError as error:
+ print(Bcolors.colorize(
+ '\nERROR: {err}'.format(err=error),
+ Bcolors.FAIL, Bcolors.BOLD))
+
+ POOL.terminate()
+ POOL.join()
+
+ sys.exit(1)
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/mesos-split.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-split.py b/support/python3/mesos-split.py
new file mode 100755
index 0000000..0a77c25
--- /dev/null
+++ b/support/python3/mesos-split.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Errors if a list of files spans across
+the projects which make up mesos.
+"""
+
+from collections import defaultdict
+import sys
+
+if len(sys.argv) < 2:
+ print("Usage: ./mesos-split.py <filename>...")
+
+BASE_PROJECT = "mesos"
+
+SUBPROJECTS = {
+ "libprocess": "3rdparty/libprocess",
+ "stout": "3rdparty/stout"
+}
+
+ERROR = """ERROR: Commit spanning multiple projects.
+
+Please use separate commits for mesos, libprocess and stout.
+
+Paths grouped by project:"""
+
+
+def find_project(filename):
+ """Find a project using its filename."""
+
+ # Find longest prefix match.
+ found_path_len = 0
+ found_project = BASE_PROJECT
+ for project, path in SUBPROJECTS.items():
+ if filename.startswith(path) and len(path) > found_path_len:
+ found_path_len = len(path)
+ found_project = project
+
+ return found_project
+
+
+def main():
+ """
+ Expects a list of filenames on the command line.
+
+ See `support/hooks/pre-commit` for the canonical usage of this method.
+ """
+ touched_projects = defaultdict(list)
+ for filename in sys.argv[1:]:
+ touched_projects[find_project(filename)].append(filename)
+
+ if len(touched_projects) > 1:
+ print(ERROR)
+ for project in touched_projects.keys():
+ print("%s:" % project)
+ for filename in touched_projects[project]:
+ print(" %s" % filename)
+ sys.exit(1)
+
+ sys.exit(0)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-style.py b/support/python3/mesos-style.py
new file mode 100755
index 0000000..c0e918c
--- /dev/null
+++ b/support/python3/mesos-style.py
@@ -0,0 +1,521 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Runs checks for mesos style."""
+
+import os
+import re
+import string
+import subprocess
+import sys
+
+
+class LinterBase(object):
+ """
+ This is an abstract class that provides the base functionality for
+ linting files in the mesos project. Its 'main()' function
+ walks through the set of files passed to it and runs some
+ standard linting over them. This includes checking for license headers
+ and checking for non-supported characters. From there it calls a
+ 'run_lint()' function that can be overridden to provide
+ customizable style checks for a specific class of files (e.g. C++,
+ Python, etc.).
+
+ Any class that extends from 'LinterBase' should override the
+ following class variables / functions:
+
+ linter_type
+ source_dirs
+ exclude_files
+ source_files
+ comment_prefix
+
+ run_lint()
+
+ Please see the comments below for details on how to override each
+ variable.
+ """
+ # The name of the linter to help with printing which linter files
+ # are currently being processed by.
+ linter_type = ''
+
+ # Root source paths (will be traversed recursively).
+ source_dirs = []
+
+ # Add file paths and patterns which should not be checked
+ # This should include 3rdparty libraries, includes and machine generated
+ # source.
+ exclude_files = ''
+
+ # A regex of possible matches for your source files.
+ source_files = ''
+
+ # A prefix at the beginning of the line to demark comments (e.g. '//')
+ comment_prefix = ''
+
+ def check_encoding(self, source_paths):
+ """
+ Checks for encoding errors in the given files. Source
+ code files must contain only printable ascii characters.
+ This excludes the extended ascii characters 128-255.
+ http://www.asciitable.com/
+ """
+ error_count = 0
+ for path in source_paths:
+ with open(path) as source_file:
+ for line_number, line in enumerate(source_file):
+ # If we find an error, add 1 to both the character and
+ # the line offset to give them 1-based indexing
+ # instead of 0 (as is common in most editors).
+ char_errors = [offset for offset, char in enumerate(line)
+ if char not in string.printable]
+ if char_errors:
+ sys.stderr.write(
+ "{path}:{line_number}: Non-printable characters"
+ " found at [{offsets}]: \"{line}\"\n".format(
+ path=path,
+ line_number=line_number + 1,
+ offsets=', '.join([str(offset + 1) for offset
+ in char_errors]),
+ line=line.rstrip()))
+ error_count += 1
+
+ return error_count
+
+ def check_license_header(self, source_paths):
+ """Checks the license headers of the given files."""
+ error_count = 0
+ for path in source_paths:
+ with open(path) as source_file:
+ # We read the three first lines of the file as the
+ # first line could be a shebang and the second line empty.
+ head = "".join([next(source_file) for _ in range(3)])
+
+ # TODO(bbannier) We allow `Copyright` for
+ # currently deviating files. This should be
+ # removed one we have a uniform license format.
+ regex = r'^{comment_prefix} [Licensed|Copyright]'.format(
+ comment_prefix=self.comment_prefix)
+ # pylint: disable=no-member
+ regex = re.compile(regex, re.MULTILINE)
+
+ if not regex.search(head):
+ sys.stderr.write(
+ "{path}:1: A license header should appear's on one of"
+ " the first line of the file starting with"
+ " '{comment_prefix} Licensed'.: {head}".format(
+ path=path,
+ head=head,
+ comment_prefix=self.comment_prefix))
+ error_count += 1
+
+ return error_count
+
+ def find_candidates(self, root_dir):
+ """
+ Search through the all files rooted at 'root_dir' and compare
+ them against 'self.exclude_files' and 'self.source_files' to
+ come up with a set of candidate files to lint.
+ """
+ exclude_file_regex = re.compile(self.exclude_files)
+ source_criteria_regex = re.compile(self.source_files)
+ for root, _, files in os.walk(root_dir):
+ for name in files:
+ path = os.path.join(root, name)
+ if exclude_file_regex.search(path) is not None:
+ continue
+
+ if source_criteria_regex.search(name) is not None:
+ yield path
+
+ def run_command_in_virtualenv(self, command):
+ """
+ Activate the virtual environment, run the
+ given command and return its output.
+ """
+ virtualenv = os.path.join('support', '.virtualenv')
+ command = '. {virtualenv_path}/bin/activate; {cmd}'.format(
+ virtualenv_path=virtualenv, cmd=command)
+ process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+
+ return process
+
+ def run_lint(self, source_paths):
+ """
+ A custom function to provide linting for 'linter_type'.
+ It takes a list of source files to lint and returns the number
+ of errors found during the linting process.
+
+ It should print any errors as it encounters them to provide
+ feedback to the caller.
+ """
+ pass
+
+ def main(self, modified_files):
+ """
+ This function takes a list of files and lints them for the
+ class of files defined by 'linter_type'.
+ """
+
+ # Verify that source roots are accessible from current
+ # working directory. A common error could be to call
+ # the style checker from other (possibly nested) paths.
+ for source_dir in self.source_dirs:
+ if not os.path.exists(source_dir):
+ print("Could not find '{dir}'".format(dir=source_dir))
+ print('Please run from the root of the mesos source directory')
+ exit(1)
+
+ # Add all source file candidates to candidates list.
+ candidates = []
+ for source_dir in self.source_dirs:
+ for candidate in self.find_candidates(source_dir):
+ candidates.append(candidate)
+
+ # If file paths are specified, check all file paths that are
+ # candidates; else check all candidates.
+ file_paths = modified_files if modified_files else candidates
+
+ # Compute the set intersect of the input file paths and candidates.
+ # This represents the reduced set of candidates to run lint on.
+ candidates_set = set(candidates)
+ clean_file_paths_set = set(path.rstrip() for path in file_paths)
+ filtered_candidates_set = clean_file_paths_set.intersection(
+ candidates_set)
+
+ if filtered_candidates_set:
+ plural = '' if len(filtered_candidates_set) == 1 else 's'
+ print('Checking {num_files} {linter} file{plural}'.format(
+ num_files=len(filtered_candidates_set),
+ linter=self.linter_type,
+ plural=plural))
+
+ license_errors = self.check_license_header(filtered_candidates_set)
+ encoding_errors = self.check_encoding(list(filtered_candidates_set))
+ lint_errors = self.run_lint(list(filtered_candidates_set))
+ total_errors = license_errors + encoding_errors + lint_errors
+
+ sys.stderr.write('Total errors found: {num_errors}\n'.format(
+ num_errors=total_errors))
+
+ return total_errors
+
+ print("No {linter} files to lint".format(linter=self.linter_type))
+ return 0
+
+
+class CppLinter(LinterBase):
+ """The linter for C++ files, uses cpplint."""
+ linter_type = 'C++'
+
+ source_dirs = ['src',
+ 'include',
+ os.path.join('3rdparty', 'libprocess'),
+ os.path.join('3rdparty', 'stout')]
+
+ exclude_files = '(' \
+ r'elfio\-3\.2|' \
+ r'protobuf\-2\.4\.1|' \
+ r'googletest\-release\-1\.8\.0|' \
+ r'glog\-0\.3\.3|' \
+ r'boost\-1\.53\.0|' \
+ r'libev\-4\.15|' \
+ r'java/jni|' \
+ r'\.pb\.cc|\.pb\.h|\.md|\.virtualenv' \
+ ')'
+
+ source_files = r'\.(cpp|hpp|cc|h)$'
+
+ comment_prefix = r'\/\/'
+
+ def run_lint(self, source_paths):
+ """
+ Runs cpplint over given files.
+
+ http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
+ """
+
+ # See cpplint.py for full list of rules.
+ active_rules = [
+ 'build/class',
+ 'build/deprecated',
+ 'build/endif_comment',
+ 'build/nullptr',
+ 'readability/todo',
+ 'readability/namespace',
+ 'runtime/vlog',
+ 'whitespace/blank_line',
+ 'whitespace/comma',
+ 'whitespace/end_of_line',
+ 'whitespace/ending_newline',
+ 'whitespace/forcolon',
+ 'whitespace/indent',
+ 'whitespace/line_length',
+ 'whitespace/operators',
+ 'whitespace/semicolon',
+ 'whitespace/tab',
+ 'whitespace/comments',
+ 'whitespace/todo']
+
+ rules_filter = '--filter=-,+' + ',+'.join(active_rules)
+
+ # We do not use a version of cpplint available through pip as
+ # we use a custom version (see cpplint.path) to lint C++ files.
+ process = subprocess.Popen(
+ ['python3', 'support/python3/cpplint.py',
+ rules_filter] + source_paths,
+ stderr=subprocess.PIPE,
+ close_fds=True)
+
+ # Lines are stored and filtered, only showing found errors instead
+ # of e.g., 'Done processing XXX.' which tends to be dominant output.
+ for line in process.stderr:
+ if re.match('^(Done processing |Total errors found: )',
+ line.decode(sys.stdout.encoding)):
+ continue
+ sys.stderr.write(line)
+
+ process.wait()
+ return process.returncode
+
+
+class JsLinter(LinterBase):
+ """The linter for JavaScript files, uses eslint."""
+ linter_type = 'JavaScript'
+
+ source_dirs = [os.path.join('src', 'webui')]
+
+ exclude_files = '(' \
+ r'angular\-1\.2\.32|' \
+ r'angular\-route\-1\.2\.32|' \
+ r'bootstrap\-table\-1\.11\.1|' \
+ r'clipboard\-1\.5\.16|' \
+ r'jquery\-3\.2\.1|' \
+ r'relative\-date|' \
+ r'ui\-bootstrap\-tpls\-0\.9\.0|' \
+ r'angular\-route\-1\.2\.32|' \
+ r'underscore\-1\.4\.3' \
+ ')'
+
+ source_files = r'\.(js)$'
+
+ comment_prefix = '//'
+
+ def run_lint(self, source_paths):
+ """
+ Runs eslint over given files.
+
+ https://eslint.org/docs/user-guide/configuring
+ """
+
+ num_errors = 0
+
+ source_files = ' '.join(source_paths)
+ config_path = os.path.join('support', '.eslintrc.js')
+
+ process = self.run_command_in_virtualenv(
+ 'eslint {files} -c {config} -f compact'.format(
+ files=source_files,
+ config=config_path
+ )
+ )
+
+ for line in process.stdout:
+ line = line.decode(sys.stdout.encoding)
+ if "Error -" in line or "Warning -" in line:
+ sys.stderr.write(line)
+ if "Error -" in line:
+ num_errors += 1
+
+ return num_errors
+
+ def main(self, modified_files):
+ return super(JsLinter, self).main(modified_files)
+
+
+class PyLinter(LinterBase):
+ """The linter for Python files, uses pylint."""
+ linter_type = 'Python'
+
+ cli_dir = os.path.join('src', 'python', 'cli_new')
+ lib_dir = os.path.join('src', 'python', 'lib')
+ support_dir = os.path.join('support', 'python3')
+ source_dirs_to_lint_with_venv = [support_dir]
+ source_dirs_to_lint_with_tox = [cli_dir, lib_dir]
+ source_dirs = source_dirs_to_lint_with_tox + source_dirs_to_lint_with_venv
+
+ exclude_files = '(' \
+ r'protobuf\-2\.4\.1|' \
+ r'googletest\-release\-1\.8\.0|' \
+ r'glog\-0\.3\.3|' \
+ r'boost\-1\.53\.0|' \
+ r'libev\-4\.15|' \
+ r'java/jni|' \
+ r'\.virtualenv|' \
+ r'\.tox' \
+ ')'
+
+ source_files = r'\.(py)$'
+
+ comment_prefix = '#'
+
+ pylint_config = os.path.abspath(os.path.join('support', 'pylint.config'))
+
+ def run_tox(self, configfile, args, tox_env=None, recreate=False):
+ """
+ Runs tox with given configfile and args. Optionally set tox env
+ and/or recreate the tox-managed virtualenv.
+ """
+ cmd = ['tox']
+ cmd += ['-qq']
+ cmd += ['-c', configfile]
+ if tox_env is not None:
+ cmd += ['-e', tox_env]
+ if recreate:
+ cmd += ['--recreate']
+ cmd += ['--']
+ cmd += args
+
+ return subprocess.Popen(cmd, stdout=subprocess.PIPE)
+
+ def filter_source_files(self, source_dir, source_files):
+ """
+ Filters out files starting with source_dir.
+ """
+ return [f for f in source_files if f.startswith(source_dir)]
+
+ def lint_source_files_under_source_dir(self, source_dir, source_files):
+ """
+ Runs pylint directly or indirectly throgh tox on source_files which
+ are under source_dir. If tox is to be used, it must be configured
+ in source_dir, i.e. a tox.ini must be present.
+ """
+ filtered_source_files = self.filter_source_files(
+ source_dir, source_files)
+
+ if not filtered_source_files:
+ return 0
+
+ if source_dir in self.source_dirs_to_lint_with_tox:
+ process = self.run_tox(
+ configfile=os.path.join(source_dir, 'tox.ini'),
+ args=['--rcfile='+self.pylint_config] + filtered_source_files,
+ tox_env='py27-lint')
+ else:
+ process = self.run_command_in_virtualenv(
+ 'pylint --rcfile={rcfile} {files}'.format(
+ rcfile=self.pylint_config,
+ files=' '.join(filtered_source_files)))
+
+ num_errors = 0
+ for line in process.stdout:
+ line = line.decode(sys.stdout.encoding)
+ if re.match(r'^[RCWEF]: *[\d]+', line):
+ num_errors += 1
+ sys.stderr.write(line)
+
+ return num_errors
+
+ def run_lint(self, source_paths):
+ """
+ Runs pylint over given files.
+
+ https://google.github.io/styleguide/pyguide.html
+ """
+ num_errors = 0
+
+ for source_dir in self.source_dirs:
+ num_errors += self.lint_source_files_under_source_dir(
+ source_dir, source_paths)
+
+ return num_errors
+
+ def main(self, modified_files):
+ return super(PyLinter, self).main(modified_files)
+
+
+def should_build_virtualenv(modified_files):
+ """
+ Check if we should build the virtual environment required.
+ This is the case if the requirements of the environment
+ have changed or if the support script is run with no
+ arguments (meaning that the entire codebase should be linted).
+ """
+ # NOTE: If the file list is empty, we are linting the entire
+ # codebase. We should always rebuild the virtualenv in this case.
+ if not modified_files:
+ return True
+
+ basenames = [os.path.basename(path) for path in modified_files]
+
+ if 'pip-requirements.txt' in basenames:
+ print('The "pip-requirements.txt" file has changed.')
+ return True
+
+ if 'build-virtualenv' in basenames:
+ print('The "build-virtualenv" file has changed.')
+ return True
+
+ # The JS and Python linters require a virtual environment.
+ # If all the files modified are not JS or Python files,
+ # we do not need to build the virtual environment.
+ # TODO(ArmandGrillet): There should be no duplicated logic to know
+ # which linters to instantiate depending on the files to analyze.
+ if not os.path.isdir(os.path.join('support', '.virtualenv')):
+ js_and_python_files = [JsLinter().source_files, PyLinter().source_files]
+ js_and_python_files_regex = re.compile('|'.join(js_and_python_files))
+
+ for basename in basenames:
+ if js_and_python_files_regex.search(basename) is not None:
+ print('Virtualenv not detected and required... building')
+ return True
+
+ return False
+
+
+def build_virtualenv():
+ """
+ Rebuild the virtualenv by running a bootstrap script.
+ This will exit the program if there is a failure.
+ """
+ print('Rebuilding virtualenv...')
+
+ process = subprocess.Popen(
+ [os.path.join('support', 'build-virtualenv')],
+ stdout=subprocess.PIPE)
+
+ output = ''
+ for line in process.stdout:
+ output += line.decode(sys.stdout.encoding)
+
+ process.wait()
+
+ if process.returncode != 0:
+ sys.stderr.write(output)
+ sys.exit(1)
+
+if __name__ == '__main__':
+ # TODO(ArmandGrillet): We should only instantiate the linters
+ # required to lint the files to analyze. See MESOS-8351.
+ CPP_LINTER = CppLinter()
+ CPP_ERRORS = CPP_LINTER.main(sys.argv[1:])
+ JS_LINTER = JsLinter()
+ JS_ERRORS = JS_LINTER.main(sys.argv[1:])
+ PY_LINTER = PyLinter()
+ PY_ERRORS = PY_LINTER.main(sys.argv[1:])
+ sys.exit(CPP_ERRORS + JS_ERRORS + PY_ERRORS)
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/post-reviews.py
----------------------------------------------------------------------
diff --git a/support/python3/post-reviews.py b/support/python3/post-reviews.py
new file mode 100755
index 0000000..9621898
--- /dev/null
+++ b/support/python3/post-reviews.py
@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Wrapper around the post-review/rbt tool provided by Review Board.
+
+This script provides the ability to send one review for each
+commit on the current branch, instead of squashing all changes
+into a single large review. We encourage contributors to create
+logical commits which can be reviewed independently.
+
+Options to pass onto 'rbt' can be placed in a '.reviewboardrc'
+file at the top of the Mesos source directory. A default
+'.reviewboardrc' can be found at 'support/reviewboardrc'.
+Running './bootstrap' will populate this file for you.
+
+To use this script, first install 'RBTools' from Review Board:
+http://www.reviewboard.org/downloads/rbtools/
+
+$ cd /path/to/mesos
+$ [ do some work on your branch off of master, make commit(s) ]
+$ ./support/post-reviews.py
+"""
+# pylint: skip-file
+
+import argparse
+import atexit
+import imp
+import os
+import platform
+import re
+import sys
+import urllib.parse
+
+from distutils.version import LooseVersion
+
+from subprocess import check_output, Popen, PIPE, STDOUT
+
+
+def execute(command, ignore_errors=False):
+ """Execute a process and leave."""
+ process = None
+ try:
+ process = Popen(command,
+ stdin=PIPE,
+ stdout=PIPE,
+ stderr=STDOUT,
+ shell=False)
+ except Exception:
+ if not ignore_errors:
+ raise
+ return None
+
+ data, _ = process.communicate()
+ data = data.decode(sys.stdout.encoding)
+ status = process.wait()
+ if status != 0 and not ignore_errors:
+ cmdline = ' '.join(command) if isinstance(command, list) else command
+ need_login = 'Please log in to the Review Board' \
+ ' server at reviews.apache.org.'
+ if need_login in data:
+ print(need_login, '\n')
+ print("You can either:")
+ print(" (1) Run 'rbt login', or")
+ print(" (2) Set the default USERNAME/PASSWORD in '.reviewboardrc'")
+ else:
+ print('Failed to execute: \'' + cmdline + '\':')
+ print(data)
+ sys.exit(1)
+ elif status != 0:
+ return None
+ return data
+
+
+def main():
+ """Main function, post commits added to this branch as review requests."""
+ # TODO(benh): Make sure this is a git repository, apologize if not.
+
+ # Choose 'rbt' if available, otherwise choose 'post-review'.
+ post_review = None
+
+ rbt_command = 'rbt'
+ # Windows command name must have `cmd` extension.
+ if platform.system() == 'Windows':
+ rbt_command = 'rbt.cmd'
+
+ rbt_version = execute([rbt_command, '--version'], ignore_errors=True)
+ if rbt_version:
+ rbt_version = LooseVersion(rbt_version)
+ post_review = [rbt_command, 'post']
+ elif execute(['post-review', '--version'], ignore_errors=True):
+ post_review = ['post-review']
+ else:
+ print('Please install RBTools before proceeding')
+ sys.exit(1)
+
+ # Warn if people have unstaged changes.
+ diff_stat = execute(['git', 'diff', '--shortstat']).strip()
+
+ if diff_stat:
+ print('WARNING: Worktree contains unstaged changes, continuing anyway.', file=sys.stderr)
+
+ # Warn if people have uncommitted changes.
+ diff_stat = execute(['git', 'diff', '--shortstat', '--staged']).strip()
+
+ if diff_stat:
+ print('WARNING: Worktree contains staged but uncommitted changes, ' \
+ 'continuing anyway.', file=sys.stderr)
+
+ # Grab a reference to the repo's git directory. Usually this is simply
+ # .git in the repo's top level directory. However, when submodules are
+ # used, it may appear elsewhere. The most up-to-date way of finding this
+ # directory is to use `git rev-parse --git-common-dir`. This is necessary
+ # to support things like git worktree in addition to git submodules.
+ # However, as of January 2016, support for the '--git-common-dir' flag is
+ # fairly new, forcing us to fall back to the '--git-dir' flag if
+ # '--git-common-dir' is not supported. We do this by checking the output of
+ # git rev-parse --git-common-dir` and check if it gives a valid directory.
+ # If not, we set the git directory using the '--git-dir' flag instead.
+ git_dir = execute(['git', 'rev-parse', '--git-common-dir']).strip()
+ if not os.path.isdir(git_dir):
+ git_dir = execute(['git', 'rev-parse', '--git-dir']).strip()
+
+ # Grab a reference to the top level directory of this repo.
+ top_level_dir = execute(['git', 'rev-parse', '--show-toplevel']).strip()
+
+ # Use the tracking_branch specified by the user if exists.
+ parser = argparse.ArgumentParser(add_help=True)
+ parser.add_argument(
+ '--server',
+ help='Specifies the Review Board server to use.')
+ parser.add_argument(
+ '--no-markdown',
+ action='store_true',
+ help='Specifies if the commit text should not be treated as Markdown.')
+ parser.add_argument(
+ '--bugs-closed',
+ help='The comma-separated list of bug IDs closed.')
+ parser.add_argument(
+ '--target-people',
+ help='The usernames of the people who should perform the review.')
+ parser.add_argument(
+ '--tracking-branch',
+ help='The remote tracking branch from which your local branch is derived.')
+ args, _ = parser.parse_known_args()
+
+ # Try to read the .reviewboardrc in the top-level directory.
+ reviewboardrc_filepath = os.path.join(top_level_dir, '.reviewboardrc')
+ if os.path.exists(reviewboardrc_filepath):
+ # Prevent generation of '.reviewboardrcc'.
+ sys.dont_write_bytecode = True
+ reviewboardrc = imp.load_source('reviewboardrc', reviewboardrc_filepath)
+
+ if args.server:
+ reviewboard_url = args.server
+ elif 'REVIEWBOARD_URL' in dir(reviewboardrc):
+ reviewboard_url = reviewboardrc.REVIEWBOARD_URL
+ else:
+ reviewboard_url = 'https://reviews.apache.org'
+
+ if args.tracking_branch:
+ tracking_branch = args.tracking_branch
+ elif 'TRACKING_BRANCH' in dir(reviewboardrc):
+ tracking_branch = reviewboardrc.TRACKING_BRANCH
+ else:
+ tracking_branch = 'master'
+
+ branch_ref = execute(['git', 'symbolic-ref', 'HEAD']).strip()
+ branch = branch_ref.replace('refs/heads/', '', 1)
+
+ # Do not work on the tracking branch.
+ if branch == tracking_branch:
+ print("We're expecting you to be working on another branch" \
+ " from {}!".format(tracking_branch))
+ sys.exit(1)
+
+ temporary_branch = '_post-reviews_' + branch
+
+ # Always delete the temporary branch.
+ atexit.register(
+ lambda: execute(['git', 'branch', '-D', temporary_branch], True))
+
+ # Always put us back on the original branch.
+ atexit.register(lambda: execute(['git', 'checkout', branch]))
+
+ # Warn if the tracking branch is no direct ancestor of this review chain.
+ if execute([
+ 'git', 'merge-base', '--is-ancestor', tracking_branch, branch_ref],
+ ignore_errors=True) is None:
+ print("WARNING: Tracking branch '%s' is no direct ancestor of HEAD." \
+ " Did you forget to rebase?" % tracking_branch, file=sys.stderr)
+
+ try:
+ input("Press enter to continue or 'Ctrl-C' to abort.\n")
+ except KeyboardInterrupt:
+ sys.exit(0)
+
+ merge_base = execute(
+ ['git', 'merge-base', tracking_branch, branch_ref]).strip()
+
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C'
+ '(yellow)%d%Creset %s %Cgreen(%cr)%Creset',
+ merge_base + '..HEAD'])
+
+ print('Running \'%s\' across all of ...' % " ".join(post_review))
+ sys.stdout.buffer.write(output)
+
+ log = execute(['git',
+ '--no-pager',
+ 'log',
+ '--no-color',
+ '--pretty=oneline',
+ '--reverse',
+ merge_base + '..HEAD']).strip()
+
+ if len(log) <= 0:
+ print("No new changes compared with master branch!")
+ sys.exit(1)
+
+ shas = []
+
+ for line in log.split('\n'):
+ sha = line.split()[0]
+ shas.append(sha)
+
+ previous = merge_base
+ parent_review_request_id = None
+ for i, sha in enumerate(shas):
+ execute(['git', 'branch', '-D', temporary_branch], True)
+
+ message = execute(['git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%s%n%n%b',
+ previous + '..' + sha])
+
+ review_request_id = None
+
+ pos = message.find('Review:')
+ if pos != -1:
+ regex = 'Review: ({url})$'.format(
+ url=urllib.parse.urljoin(reviewboard_url, 'r/[0-9]+'))
+ pattern = re.compile(regex)
+ match = pattern.search(message[pos:].strip().strip('/'))
+ if match is None:
+ print("\nInvalid ReviewBoard URL: '{}'".format(message[pos:]))
+ sys.exit(1)
+
+ url = match.group(1)
+ review_request_id = url.split('/')[-1]
+
+ # Show the commit.
+ if review_request_id is None:
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C(yellow)%d%Creset %s',
+ previous + '..' + sha])
+ print('\nCreating diff of:')
+ sys.stdout.buffer.write(output)
+ else:
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C'
+ '(yellow)%d%Creset %s %Cgreen(%cr)%Creset',
+ previous + '..' + sha])
+ print('\nUpdating diff of:')
+ sys.stdout.buffer.write(output)
+
+ # Show the "parent" commit(s).
+ output = check_output([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%Cred%H%Creset -%C'
+ '(yellow)%d%Creset %s %Cgreen(%cr)%Creset',
+ tracking_branch + '..' + previous])
+
+ if output:
+ print('\n... with parent diff created from:')
+ sys.stdout.buffer.write(output)
+
+ try:
+ input('\nPress enter to continue or \'Ctrl-C\' to skip.\n')
+ except KeyboardInterrupt:
+ i = i + 1
+ previous = sha
+ parent_review_request_id = review_request_id
+ continue
+
+ # Strip the review url from the commit message, so that
+ # it is not included in the summary message when GUESS_FIELDS
+ # is set in .reviewboardc. Update the SHA appropriately.
+ if review_request_id:
+ stripped_message = message[:pos]
+ execute(['git', 'checkout', sha])
+ execute(['git', 'commit', '--amend', '-m', stripped_message])
+ sha = execute(['git', 'rev-parse', 'HEAD']).strip()
+ execute(['git', 'checkout', branch])
+
+ revision_range = previous + ':' + sha
+
+ # Build the post-review/rbt command up
+ # to the point where they are common.
+ command = post_review
+
+ if not args.no_markdown:
+ command = command + ['--markdown']
+
+ if args.bugs_closed:
+ command = command + ['--bugs-closed=' + args.bugs_closed]
+
+ if args.target_people:
+ command = command + ['--target-people=' + args.target_people]
+
+ if args.tracking_branch is None:
+ command = command + ['--tracking-branch=' + tracking_branch]
+
+ if review_request_id:
+ command = command + ['--review-request-id=' + review_request_id]
+
+ # Determine how to specify the revision range.
+ if rbt_command in post_review and \
+ rbt_version >= LooseVersion('RBTools 0.6'):
+ # rbt >= 0.6.1 supports '--depends-on' argument.
+ # Only set the "depends on" if this
+ # is not the first review in the chain.
+ if rbt_version >= LooseVersion('RBTools 0.6.1') and \
+ parent_review_request_id:
+ command = command + ['--depends-on=' + parent_review_request_id]
+
+ # rbt >= 0.6 revisions are passed in as args.
+ command = command + sys.argv[1:] + [previous, sha]
+ else:
+ # post-review and rbt < 0.6 revisions are
+ # passed in using the revision range option.
+ command = command + \
+ ['--revision-range=' + revision_range] + \
+ sys.argv[1:]
+
+ output = execute(command).strip()
+
+ sys.stdout.buffer.write(output)
+
+ # If we already have a request_id, continue on to the next commit in the
+ # chain. We update 'previous' from the shas[] array because we have
+ # overwritten the temporary sha variable above.
+ if review_request_id is not None:
+ previous = shas[i]
+ parent_review_request_id = review_request_id
+ i = i + 1
+ continue
+
+ # Otherwise, get the request_id from the output of post-review, append
+ # it to the commit message and rebase all other commits on top of it.
+ lines = output.split('\n')
+
+ # The last line of output in post-review is the review url.
+ # The second to the last line of output in rbt is the review url.
+ url = lines[len(lines) - 2] if rbt_command in post_review \
+ else lines[len(lines) - 1]
+
+ # Using rbt >= 0.6.3 on Linux prints out two URLs where the second
+ # one has /diff/ at the end. We want to remove this so that a
+ # subsequent call to post-reviews does not fail when looking up
+ # the reviewboard entry to edit.
+ url = url.replace('diff/', '')
+ url = url.strip('/')
+ review_request_id = os.path.basename(url)
+
+ # Construct new commit message.
+ message = message + '\n' + 'Review: ' + url + '\n'
+
+ execute(['git', 'checkout', '-b', temporary_branch])
+ execute(['git', 'reset', '--hard', sha])
+ execute(['git', 'commit', '--amend', '-m', message])
+
+ # Now rebase all remaining shas on top of this amended commit.
+ j = i + 1
+ old_sha = execute(
+ ['git', 'rev-parse', '--verify', temporary_branch]).strip()
+ previous = old_sha
+ while j < len(shas):
+ execute(['git', 'checkout', shas[j]])
+ execute(['git', 'rebase', temporary_branch])
+ # Get the sha for our detached HEAD.
+ new_sha = execute([
+ 'git',
+ '--no-pager',
+ 'log',
+ '--pretty=format:%H', '-n', '1', 'HEAD']).strip()
+ execute(['git',
+ 'update-ref',
+ 'refs/heads/' + temporary_branch,
+ new_sha,
+ old_sha])
+ old_sha = new_sha
+ shas[j] = new_sha
+ j = j + 1
+
+ # Okay, now update the actual branch to our temporary branch.
+ new_sha = old_sha
+ old_sha = execute(['git', 'rev-parse', '--verify', branch]).strip()
+ execute(['git', 'update-ref', 'refs/heads/' + branch, new_sha, old_sha])
+
+ i = i + 1
+ parent_review_request_id = review_request_id
+
+if __name__ == '__main__':
+ main()
[3/6] mesos git commit: Added python3 to list of Pylint excluded
files.
Posted by an...@apache.org.
Added python3 to list of Pylint excluded files.
This change ensures that pylint will not try to lint the new Python 3
support scripts if it is not run with Python 3. Having such a situation
results in unexpected errors such as "Unnecessary parens after 'print'
keyword". This change will not be applied in the Python 3 mesos-style.
Review: https://reviews.apache.org/r/67282/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/71315eb0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/71315eb0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/71315eb0
Branch: refs/heads/master
Commit: 71315eb0c999706b142d486b11897fbfadf94cd2
Parents: 2820028
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Thu May 24 14:58:09 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Thu May 24 14:58:09 2018 -0700
----------------------------------------------------------------------
support/mesos-style.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/71315eb0/support/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/mesos-style.py b/support/mesos-style.py
index 805ecb7..27ed553 100755
--- a/support/mesos-style.py
+++ b/support/mesos-style.py
@@ -363,7 +363,8 @@ class PyLinter(LinterBase):
r'libev\-4\.15|' \
r'java/jni|' \
r'\.virtualenv|' \
- r'\.tox' \
+ r'\.tox|' \
+ r'python3' \
')'
source_files = r'\.(py)$'
[6/6] mesos git commit: Added warning to Python version checker
script.
Posted by an...@apache.org.
Added warning to Python version checker script.
After the user has installed Python 3, if they use the Python 2
scripts, we want to alert them to start using (and therefore testing)
the ported Python 3 scripts.
Review: https://reviews.apache.org/r/67292/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ae4e7956
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ae4e7956
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ae4e7956
Branch: refs/heads/master
Commit: ae4e7956a2217ceaa02575d160a9d52a21825f07
Parents: 960df5c
Author: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Authored: Thu May 24 14:58:34 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Thu May 24 15:01:58 2018 -0700
----------------------------------------------------------------------
support/check-python3.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ae4e7956/support/check-python3.py
----------------------------------------------------------------------
diff --git a/support/check-python3.py b/support/check-python3.py
index 4cb5365..08252d8 100644
--- a/support/check-python3.py
+++ b/support/check-python3.py
@@ -30,6 +30,15 @@ def print_error():
print("The support scripts will be upgraded to Python 3 by July 1st.")
print("Make sure to install Python 3.6 on your machine before.")
+def print_warning():
+ """Prints a warning requesting to use the Python 3 scripts."""
+ print("Congratulations! You have Python 3 installed correctly.")
+ print("Please start using the scripts in `support/python3`.")
+ # NOTE: This is only either unset, or set to 3.
+ if "MESOS_SUPPORT_PYTHON" not in os.environ:
+ print("Please also set the environment variable `MESOS_SUPPORT_PYTHON` to `3`")
+ print("so that the Git hooks use the Python 3 scripts.")
+
if sys.version_info[0] < 3:
# On Windows, system-wide installations of Python 3.6 gives a tools called
# py and that we can use to know if Python 3 is installed.
@@ -51,6 +60,11 @@ if sys.version_info[0] < 3:
for x in range(0, 6):
if "3.%d." % (x) in VERSION:
print_error()
+ sys.exit()
+ # This script only gets invoked by the Python 2 scripts, so we
+ # can assume we need to warn the user to start using the
+ # Python 3 scripts.
+ print_warning()
elif sys.version_info[1] < 6:
# python is by default Python 3 but it's < 3.6.
print_error()
[2/6] mesos git commit: Updated support scripts to check for Python 3.
Posted by an...@apache.org.
Updated support scripts to check for Python 3.
Review: https://reviews.apache.org/r/67099/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2820028e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2820028e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2820028e
Branch: refs/heads/master
Commit: 2820028e913b934015b77a6dd8ddc1529067a27e
Parents: 7ecb2fe
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Thu May 24 14:58:03 2018 -0700
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Thu May 24 14:58:03 2018 -0700
----------------------------------------------------------------------
support/apply-reviews.py | 5 +++++
support/generate-endpoint-help.py | 5 +++++
support/jsonurl.py | 7 +++++++
support/mesos-gtest-runner.py | 5 +++++
support/mesos-split.py | 8 ++++++++
support/mesos-style.py | 5 +++++
support/post-reviews.py | 6 ++++++
support/push-commits.py | 9 ++++++++-
support/test-upgrade.py | 5 +++++
support/verify-reviews.py | 5 +++++
10 files changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/apply-reviews.py
----------------------------------------------------------------------
diff --git a/support/apply-reviews.py b/support/apply-reviews.py
index 0c744a9..edddefb 100755
--- a/support/apply-reviews.py
+++ b/support/apply-reviews.py
@@ -443,6 +443,11 @@ def main():
"""
options = parse_options()
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
if options['review_id']:
reviewboard(options)
else:
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/generate-endpoint-help.py
----------------------------------------------------------------------
diff --git a/support/generate-endpoint-help.py b/support/generate-endpoint-help.py
index 7e59b35..54b5408 100755
--- a/support/generate-endpoint-help.py
+++ b/support/generate-endpoint-help.py
@@ -378,6 +378,11 @@ def main():
# A dictionary of the command line options passed in.
options = parse_options()
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
# A pointer to the current subprocess for the master or agent.
# This is useful for tracking the master or agent subprocesses so
# that we can kill them if the script exits prematurely.
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/jsonurl.py
----------------------------------------------------------------------
diff --git a/support/jsonurl.py b/support/jsonurl.py
index 72fe682..c9cf1d9 100755
--- a/support/jsonurl.py
+++ b/support/jsonurl.py
@@ -25,6 +25,8 @@ play off of 'curl'.
"""
import json
+import os
+import subprocess
import sys
import urllib2
@@ -35,6 +37,11 @@ def main():
print >> sys.stderr, "USAGE: {} URL [KEY...]".format(sys.argv[0])
sys.exit(1)
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
url = sys.argv[1]
data = json.loads(urllib2.urlopen(url).read())
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/mesos-gtest-runner.py
----------------------------------------------------------------------
diff --git a/support/mesos-gtest-runner.py b/support/mesos-gtest-runner.py
index 4ae5fe3..9cabbdf 100755
--- a/support/mesos-gtest-runner.py
+++ b/support/mesos-gtest-runner.py
@@ -198,6 +198,11 @@ def parse_arguments():
if __name__ == '__main__':
EXECUTABLE, OPTIONS = parse_arguments()
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
def options_gen(executable, filter_, jobs):
"""Generator for options for a certain shard.
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/mesos-split.py
----------------------------------------------------------------------
diff --git a/support/mesos-split.py b/support/mesos-split.py
index 7ef5c86..f83986d 100755
--- a/support/mesos-split.py
+++ b/support/mesos-split.py
@@ -21,6 +21,8 @@ the projects which make up mesos.
"""
from collections import defaultdict
+import os
+import subprocess
import sys
if len(sys.argv) < 2:
@@ -60,6 +62,12 @@ def main():
See `support/hooks/pre-commit` for the canonical usage of this method.
"""
+
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
touched_projects = defaultdict(list)
for filename in sys.argv[1:]:
touched_projects[find_project(filename)].append(filename)
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/mesos-style.py b/support/mesos-style.py
index b7ee699..805ecb7 100755
--- a/support/mesos-style.py
+++ b/support/mesos-style.py
@@ -509,6 +509,11 @@ if __name__ == '__main__':
if should_build_virtualenv(sys.argv[1:]):
build_virtualenv()
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
# TODO(ArmandGrillet): We should only instantiate the linters
# required to lint the files to analyze. See MESOS-8351.
CPP_LINTER = CppLinter()
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/post-reviews.py
----------------------------------------------------------------------
diff --git a/support/post-reviews.py b/support/post-reviews.py
index a6646f2..94ece85 100755
--- a/support/post-reviews.py
+++ b/support/post-reviews.py
@@ -44,6 +44,7 @@ import imp
import os
import platform
import re
+import subprocess
import sys
import urlparse
@@ -90,6 +91,11 @@ def main():
"""Main function, post commits added to this branch as review requests."""
# TODO(benh): Make sure this is a git repository, apologize if not.
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
# Choose 'rbt' if available, otherwise choose 'post-review'.
post_review = None
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/push-commits.py
----------------------------------------------------------------------
diff --git a/support/push-commits.py b/support/push-commits.py
index cd751cb..b10e802 100755
--- a/support/push-commits.py
+++ b/support/push-commits.py
@@ -35,9 +35,11 @@ Example Usage:
import argparse
import os
import re
+import subprocess
+from subprocess import check_output
+
import sys
-from subprocess import check_output
REVIEWBOARD_URL = 'https://reviews.apache.org'
@@ -106,6 +108,11 @@ def main():
"""Main function to push the commits in this branch as review requests."""
options = parse_options()
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
current_branch_ref = check_output(['git', 'symbolic-ref', 'HEAD']).strip()
current_branch = current_branch_ref.replace('refs/heads/', '', 1)
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/test-upgrade.py
----------------------------------------------------------------------
diff --git a/support/test-upgrade.py b/support/test-upgrade.py
index 53c0688..b7c6612 100755
--- a/support/test-upgrade.py
+++ b/support/test-upgrade.py
@@ -187,6 +187,11 @@ def main():
required=True)
args = parser.parse_args()
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
# Get the version strings from the built executables.
prev_version = version(args.prev)
next_version = version(args.next)
http://git-wip-us.apache.org/repos/asf/mesos/blob/2820028e/support/verify-reviews.py
----------------------------------------------------------------------
diff --git a/support/verify-reviews.py b/support/verify-reviews.py
index fbc2460..c86db35 100755
--- a/support/verify-reviews.py
+++ b/support/verify-reviews.py
@@ -298,6 +298,11 @@ def needs_verification(review_request):
def main():
"""Main function to verify the submitted reviews."""
+ # TODO(ArmandGrillet): Remove this when we'll have switched to Python 3.
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ script_path = os.path.join(dir_path, 'check-python3.py')
+ subprocess.call('python ' + script_path, shell=True, cwd=dir_path)
+
review_requests_url = \
"%s/api/review-requests/%s" % (REVIEWBOARD_URL, QUERY_PARAMS)