You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by kl...@apache.org on 2018/07/13 20:56:30 UTC
[1/5] mesos git commit: Fixed `build-virtualenv` to correctly clear
virtual environment.
Repository: mesos
Updated Branches:
refs/heads/master 13fb36ec0 -> c2bdf07eb
Fixed `build-virtualenv` to correctly clear virtual environment.
Running `build-virtualenv` with Python 3 followed by running it again
with Python 2 leaves the directory with both versions of Python in the
virtualenv. This allows Python 3 to still be used from within the
virtualenv, which is incorrect.
This is caused by a long outstanding issue in `virtualenv` that we now
handle by unconditionally removing the 'support/.virtualenv' directory
before building it again.
Review: https://reviews.apache.org/r/67911/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4a02bbca
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4a02bbca
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4a02bbca
Branch: refs/heads/master
Commit: 4a02bbca61be7451df99f89fc2b99615cc5a7064
Parents: 13fb36e
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Fri Jul 13 22:02:41 2018 +0200
Committer: Kevin Klues <kl...@gmail.com>
Committed: Fri Jul 13 22:07:40 2018 +0200
----------------------------------------------------------------------
support/build-virtualenv | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4a02bbca/support/build-virtualenv
----------------------------------------------------------------------
diff --git a/support/build-virtualenv b/support/build-virtualenv
index b8dc1d9..eb63464 100755
--- a/support/build-virtualenv
+++ b/support/build-virtualenv
@@ -32,6 +32,12 @@ if [ "${PYTHON}" = "" ]; then
exit 1
fi
+# Old versions of virtualenv do not remove the bin directory in the
+# virtual environment even when using `--clear`. We thus remove the
+# entire directory in case the virtual environment already exists.
+# See https://github.com/pypa/virtualenv/issues/2 for more info.
+rm -rf ${VIRTUALENV_DIRECTORY}
+
PYTHON_MAJOR=$(${PYTHON} -c 'import sys; print(sys.version_info[0])')
PYTHON_MINOR=$(${PYTHON} -c 'import sys; print(sys.version_info[1])')
@@ -42,7 +48,7 @@ if [ "${PYTHON_MAJOR}" = "3" ]; then
exit 1
else
# Set up a virtual environment for the linters.
- ${PYTHON} -m venv --clear --prompt="${VIRTUALENV_NAME}" ${VIRTUALENV_DIRECTORY}
+ ${PYTHON} -m venv --prompt="${VIRTUALENV_NAME}" ${VIRTUALENV_DIRECTORY}
fi
elif [ "${PYTHON_MAJOR}" = "2" ]; then
if [ "${PYTHON_MINOR}" -lt "6" ]; then
@@ -63,7 +69,6 @@ elif [ "${PYTHON_MAJOR}" = "2" ]; then
# Set up a virtual environment for the linters.
${PYTHON} ${VIRTUALENV} --python=${PYTHON} \
- --clear \
--no-site-packages \
--prompt="(${VIRTUALENV_NAME}) " \
${VIRTUALENV_DIRECTORY}
[5/5] mesos git commit: Updated pylint usage in mesos-style.py to be
less verbose.
Posted by kl...@apache.org.
Updated pylint usage in mesos-style.py to be less verbose.
We started using Pylint 1.9 a few days ago to lint `.py` files, and this
version of pylint always prints a score after linting is complete. To
make the output less verbose, we now use the option `--score=n` when
using pylint in mesos-style.py.
Review: https://reviews.apache.org/r/67907/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c2bdf07e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c2bdf07e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c2bdf07e
Branch: refs/heads/master
Commit: c2bdf07eb8771e7e00173606bedf2445a3a709b9
Parents: ec87ab5
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Fri Jul 13 22:37:50 2018 +0200
Committer: Kevin Klues <kl...@gmail.com>
Committed: Fri Jul 13 22:41:27 2018 +0200
----------------------------------------------------------------------
support/mesos-style.py | 2 +-
support/python3/mesos-style.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c2bdf07e/support/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/mesos-style.py b/support/mesos-style.py
index 0795959..32a5f17 100755
--- a/support/mesos-style.py
+++ b/support/mesos-style.py
@@ -420,7 +420,7 @@ class PyLinter(LinterBase):
tox_env='py27-lint')
else:
process = self.run_command_in_virtualenv(
- 'pylint --rcfile={rcfile} {files}'.format(
+ 'pylint --score=n --rcfile={rcfile} {files}'.format(
rcfile=self.pylint_config,
files=' '.join(filtered_source_files)))
http://git-wip-us.apache.org/repos/asf/mesos/blob/c2bdf07e/support/python3/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-style.py b/support/python3/mesos-style.py
index fbf60c6..b15da95 100755
--- a/support/python3/mesos-style.py
+++ b/support/python3/mesos-style.py
@@ -423,7 +423,7 @@ class PyLinter(LinterBase):
tox_env='py27-lint')
else:
process = self.run_command_in_virtualenv(
- 'pylint --rcfile={rcfile} {files}'.format(
+ 'pylint --score=n --rcfile={rcfile} {files}'.format(
rcfile=self.pylint_config,
files=' '.join(filtered_source_files)))
[4/5] mesos git commit: Updated tox usage in mesos-style.py to run
directly from virtualenv.
Posted by kl...@apache.org.
Updated tox usage in mesos-style.py to run directly from virtualenv.
We were already doing this in the ptyhon2 mesos-style.py script. This
commit adds the same support to the python3 version of the script.
Review: https://reviews.apache.org/r/67906/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ec87ab57
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ec87ab57
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ec87ab57
Branch: refs/heads/master
Commit: ec87ab575e698c447477fc6bb957f8270aea8c3f
Parents: dd07b14
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Fri Jul 13 15:32:17 2018 +0200
Committer: Kevin Klues <kl...@gmail.com>
Committed: Fri Jul 13 22:41:27 2018 +0200
----------------------------------------------------------------------
support/mesos-style.py | 8 ++++++--
support/python3/mesos-style.py | 10 +++++++++-
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ec87ab57/support/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/mesos-style.py b/support/mesos-style.py
index f9b4742..0795959 100755
--- a/support/mesos-style.py
+++ b/support/mesos-style.py
@@ -378,8 +378,9 @@ class PyLinter(LinterBase):
Runs tox with given configfile and args. Optionally set tox env
and/or recreate the tox-managed virtualenv.
"""
- cmd = [os.path.join(os.path.dirname(__file__),
- '.virtualenv', 'bin', 'tox')]
+ support_dir = os.path.dirname(__file__)
+
+ cmd = [os.path.join(support_dir, '.virtualenv', 'bin', 'tox')]
cmd += ['-qq']
cmd += ['-c', configfile]
if tox_env is not None:
@@ -389,6 +390,9 @@ class PyLinter(LinterBase):
cmd += ['--']
cmd += args
+ # We do not use `run_command_in_virtualenv()` here, as we
+ # directly call `tox` from inside the virtual environment bin
+ # directory without activating the virtualenv.
return subprocess.Popen(cmd, stdout=subprocess.PIPE)
def filter_source_files(self, source_dir, source_files):
http://git-wip-us.apache.org/repos/asf/mesos/blob/ec87ab57/support/python3/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-style.py b/support/python3/mesos-style.py
index 10caf8c..fbf60c6 100755
--- a/support/python3/mesos-style.py
+++ b/support/python3/mesos-style.py
@@ -378,7 +378,12 @@ class PyLinter(LinterBase):
Runs tox with given configfile and args. Optionally set tox env
and/or recreate the tox-managed virtualenv.
"""
- cmd = ['tox']
+ # TODO(ArmandGrillet): Remove one of the calls to
+ # `os.path.dirname` below once the python 3 support scripts
+ # are moved from `support/python3/` to `support/`.
+ support_dir = os.path.dirname(os.path.dirname(__file__))
+
+ cmd = [os.path.join(support_dir, '.virtualenv', 'bin', 'tox')]
cmd += ['-qq']
cmd += ['-c', configfile]
if tox_env is not None:
@@ -388,6 +393,9 @@ class PyLinter(LinterBase):
cmd += ['--']
cmd += args
+ # We do not use `run_command_in_virtualenv()` here, as we
+ # directly call `tox` from inside the virtual environment bin
+ # directory without activating the virtualenv.
return subprocess.Popen(cmd, stdout=subprocess.PIPE)
def filter_source_files(self, source_dir, source_files):
[3/5] mesos git commit: Fixed virtualenv management in support
directory.
Posted by kl...@apache.org.
Fixed virtualenv management in support directory.
The switch from Python 2 to Python 3 creates problems with managing the
virtual environment in the support directory if a developer regularly
toggles between Python 2 and 3. For example, we want the virtual
environment to always use the same version of Python the user uses to
invoke the python linter from the command line through mesos-style.py.
This commit fixes the issue by adding a new check in the function of
`mesos-style.py` called `should_build_virtualenv` to see if the Python
interpreter version currently used is the one in the virtual
environement. If not, the virtual environment will get recreated.
Review: https://reviews.apache.org/r/67910/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/dd07b14d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/dd07b14d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/dd07b14d
Branch: refs/heads/master
Commit: dd07b14df02d1a944004e63a03d97161c413e706
Parents: 6936fb5
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Fri Jul 13 22:10:46 2018 +0200
Committer: Kevin Klues <kl...@gmail.com>
Committed: Fri Jul 13 22:41:27 2018 +0200
----------------------------------------------------------------------
support/mesos-style.py | 10 ++++++++--
support/python3/mesos-style.py | 10 +++++++++-
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/dd07b14d/support/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/mesos-style.py b/support/mesos-style.py
index 5591be0..f9b4742 100755
--- a/support/mesos-style.py
+++ b/support/mesos-style.py
@@ -450,11 +450,17 @@ def should_build_virtualenv(modified_files):
have changed or if the support script is run with no
arguments (meaning that the entire codebase should be linted).
"""
- # NOTE: If the file list is empty, we are linting the entire
- # codebase. We should always rebuild the virtualenv in this case.
+ # NOTE: If the file list is empty, we are linting the entire test
+ # codebase. We should always rebuild the vI've irtualenv in this case.
if not modified_files:
return True
+ support_dir = os.path.dirname(__file__)
+ interpreter = os.path.basename(sys.executable)
+ interpreter = os.path.join(support_dir, '.virtualenv', 'bin', interpreter)
+ if not os.path.isfile(interpreter):
+ return True
+
basenames = [os.path.basename(path) for path in modified_files]
if 'pip-requirements.txt' in basenames:
http://git-wip-us.apache.org/repos/asf/mesos/blob/dd07b14d/support/python3/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-style.py b/support/python3/mesos-style.py
index 09e26a8..10caf8c 100755
--- a/support/python3/mesos-style.py
+++ b/support/python3/mesos-style.py
@@ -450,11 +450,19 @@ def should_build_virtualenv(modified_files):
have changed or if the support script is run with no
arguments (meaning that the entire codebase should be linted).
"""
- # NOTE: If the file list is empty, we are linting the entire
+ # NOTE: If the file list is empty, we are linting the entire test
# codebase. We should always rebuild the virtualenv in this case.
if not modified_files:
return True
+ # TODO(ArmandGrillet): Remove one os.path.dirname once python 3
+ # support scripts are moved from support/python3/ to support/.
+ support_dir = os.path.dirname(os.path.dirname(__file__))
+ interpreter = os.path.basename(sys.executable)
+ interpreter = os.path.join(support_dir, '.virtualenv', 'bin', interpreter)
+ if not os.path.isfile(interpreter):
+ return True
+
basenames = [os.path.basename(path) for path in modified_files]
if 'pip-requirements.txt' in basenames:
[2/5] mesos git commit: Fixed linting errors that appear with the
recently updated pylint.
Posted by kl...@apache.org.
Fixed linting errors that appear with the recently updated pylint.
As part of this, we needed to remove some unnecessary delegate to
'super' calls that are legal in python2 but not in python3. That said,
they are also unnecessary in python2 so we remove them in both cases.
Specifically, we previously used to override 'main()' in our Python and
JS linters with:
def main(self, modified_files):
super(PyLinter, self).main(modified_files)
def main(self, modified_files):
super(JsLinter, self).main(modified_files)
However, in python3 this gives us linting errors because such a
delegation is unecessary. Just using the 'main()' function from the base
class is sufficient.
Review: http://reviews.apache.org/r/67910/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6936fb55
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6936fb55
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6936fb55
Branch: refs/heads/master
Commit: 6936fb553b463b99bf4bc07de9067dd9ca16457a
Parents: 4a02bbc
Author: Armand Grillet <ag...@mesosphere.io>
Authored: Fri Jul 13 22:21:00 2018 +0200
Committer: Kevin Klues <kl...@gmail.com>
Committed: Fri Jul 13 22:41:25 2018 +0200
----------------------------------------------------------------------
support/mesos-style.py | 15 ++++++---------
support/python3/mesos-style.py | 6 ------
2 files changed, 6 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6936fb55/support/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/mesos-style.py b/support/mesos-style.py
index 27ed553..5591be0 100755
--- a/support/mesos-style.py
+++ b/support/mesos-style.py
@@ -188,7 +188,7 @@ class LinterBase(object):
# If file paths are specified, check all file paths that are
# candidates; else check all candidates.
- file_paths = modified_files if len(modified_files) > 0 else candidates
+ file_paths = modified_files if modified_files else candidates
# Compute the set intersect of the input file paths and candidates.
# This represents the reduced set of candidates to run lint on.
@@ -198,7 +198,10 @@ class LinterBase(object):
candidates_set)
if filtered_candidates_set:
- plural = '' if len(filtered_candidates_set) == 1 else 's'
+ if len(filtered_candidates_set) == 1:
+ plural = ''
+ else:
+ plural = 's'
print 'Checking {num_files} {linter} file{plural}'.format(
num_files=len(filtered_candidates_set),
linter=self.linter_type,
@@ -340,9 +343,6 @@ class JsLinter(LinterBase):
return num_errors
- def main(self, modified_files):
- return super(JsLinter, self).main(modified_files)
-
class PyLinter(LinterBase):
"""The linter for Python files, uses pylint."""
@@ -406,7 +406,7 @@ class PyLinter(LinterBase):
filtered_source_files = self.filter_source_files(
source_dir, source_files)
- if len(filtered_source_files) == 0:
+ if not filtered_source_files:
return 0
if source_dir in self.source_dirs_to_lint_with_tox:
@@ -442,9 +442,6 @@ class PyLinter(LinterBase):
return num_errors
- def main(self, modified_files):
- return super(PyLinter, self).main(modified_files)
-
def should_build_virtualenv(modified_files):
"""
http://git-wip-us.apache.org/repos/asf/mesos/blob/6936fb55/support/python3/mesos-style.py
----------------------------------------------------------------------
diff --git a/support/python3/mesos-style.py b/support/python3/mesos-style.py
index 350ef90..09e26a8 100755
--- a/support/python3/mesos-style.py
+++ b/support/python3/mesos-style.py
@@ -344,9 +344,6 @@ class JsLinter(LinterBase):
return num_errors
- def main(self, modified_files):
- return super(JsLinter, self).main(modified_files)
-
class PyLinter(LinterBase):
"""The linter for Python files, uses pylint."""
@@ -445,9 +442,6 @@ class PyLinter(LinterBase):
return num_errors
- def main(self, modified_files):
- return super(PyLinter, self).main(modified_files)
-
def should_build_virtualenv(modified_files):
"""