You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/01/24 12:13:51 UTC
spark git commit: [SPARK-23174][BUILD][PYTHON] python code style
checker update
Repository: spark
Updated Branches:
refs/heads/master 4e7b49041 -> 7af1a325d
[SPARK-23174][BUILD][PYTHON] python code style checker update
## What changes were proposed in this pull request?
Referencing latest python code style checking from PyPi/pycodestyle
Removed pending TODO
For now, in tox.ini excluded the additional style error discovered on existing python due to latest style checker (will fallback on review comment to finalize exclusion or fix py)
Any further code styling requirement needs to be part of pycodestyle, not in SPARK.
## How was this patch tested?
./dev/run-tests
Author: Rekha Joshi <re...@gmail.com>
Author: rjoshi2 <re...@gmail.com>
Closes #20338 from rekhajoshm/SPARK-11222.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7af1a325
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7af1a325
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7af1a325
Branch: refs/heads/master
Commit: 7af1a325da57daa2e25c713472a320f4ccb43d71
Parents: 4e7b490
Author: Rekha Joshi <re...@gmail.com>
Authored: Wed Jan 24 21:13:47 2018 +0900
Committer: hyukjinkwon <gu...@gmail.com>
Committed: Wed Jan 24 21:13:47 2018 +0900
----------------------------------------------------------------------
dev/lint-python | 37 ++++++++++++++++++-------------------
dev/run-tests.py | 5 ++++-
dev/tox.ini | 4 ++--
3 files changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7af1a325/dev/lint-python
----------------------------------------------------------------------
diff --git a/dev/lint-python b/dev/lint-python
index df8df03..e069caf 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -21,7 +21,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
# Exclude auto-generated configuration file.
PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" )"
-PEP8_REPORT_PATH="$SPARK_ROOT_DIR/dev/pep8-report.txt"
+PYCODESTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-report.txt"
PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt"
PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt"
SPHINXBUILD=${SPHINXBUILD:=sphinx-build}
@@ -30,23 +30,22 @@ SPHINX_REPORT_PATH="$SPARK_ROOT_DIR/dev/sphinx-report.txt"
cd "$SPARK_ROOT_DIR"
# compileall: https://docs.python.org/2/library/compileall.html
-python -B -m compileall -q -l $PATHS_TO_CHECK > "$PEP8_REPORT_PATH"
+python -B -m compileall -q -l $PATHS_TO_CHECK > "$PYCODESTYLE_REPORT_PATH"
compile_status="${PIPESTATUS[0]}"
-# Get pep8 at runtime so that we don't rely on it being installed on the build server.
+# Get pycodestyle at runtime so that we don't rely on it being installed on the build server.
#+ See: https://github.com/apache/spark/pull/1744#issuecomment-50982162
-#+ TODOs:
-#+ - Download pep8 from PyPI. It's more "official".
-PEP8_VERSION="1.7.0"
-PEP8_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pep8-$PEP8_VERSION.py"
-PEP8_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/jcrocholl/pep8/$PEP8_VERSION/pep8.py"
+# Updated to latest official version for pep8. pep8 is formally renamed to pycodestyle.
+PYCODESTYLE_VERSION="2.3.1"
+PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$PYCODESTYLE_VERSION.py"
+PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$PYCODESTYLE_VERSION/pycodestyle.py"
-if [ ! -e "$PEP8_SCRIPT_PATH" ]; then
- curl --silent -o "$PEP8_SCRIPT_PATH" "$PEP8_SCRIPT_REMOTE_PATH"
+if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then
+ curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH"
curl_status="$?"
if [ "$curl_status" -ne 0 ]; then
- echo "Failed to download pep8.py from \"$PEP8_SCRIPT_REMOTE_PATH\"."
+ echo "Failed to download pycodestyle.py from \"$PYCODESTYLE_SCRIPT_REMOTE_PATH\"."
exit "$curl_status"
fi
fi
@@ -64,23 +63,23 @@ export "PATH=$PYTHONPATH:$PATH"
#+ first, but we do so so that the check status can
#+ be output before the report, like with the
#+ scalastyle and RAT checks.
-python "$PEP8_SCRIPT_PATH" --config=dev/tox.ini $PATHS_TO_CHECK >> "$PEP8_REPORT_PATH"
-pep8_status="${PIPESTATUS[0]}"
+python "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $PATHS_TO_CHECK >> "$PYCODESTYLE_REPORT_PATH"
+pycodestyle_status="${PIPESTATUS[0]}"
-if [ "$compile_status" -eq 0 -a "$pep8_status" -eq 0 ]; then
+if [ "$compile_status" -eq 0 -a "$pycodestyle_status" -eq 0 ]; then
lint_status=0
else
lint_status=1
fi
if [ "$lint_status" -ne 0 ]; then
- echo "PEP8 checks failed."
- cat "$PEP8_REPORT_PATH"
- rm "$PEP8_REPORT_PATH"
+ echo "PYCODESTYLE checks failed."
+ cat "$PYCODESTYLE_REPORT_PATH"
+ rm "$PYCODESTYLE_REPORT_PATH"
exit "$lint_status"
else
- echo "PEP8 checks passed."
- rm "$PEP8_REPORT_PATH"
+ echo "pycodestyle checks passed."
+ rm "$PYCODESTYLE_REPORT_PATH"
fi
# Check that the documentation builds acceptably, skip check if sphinx is not installed.
http://git-wip-us.apache.org/repos/asf/spark/blob/7af1a325/dev/run-tests.py
----------------------------------------------------------------------
diff --git a/dev/run-tests.py b/dev/run-tests.py
index fb270c4..fe75ef4 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -576,7 +576,10 @@ def main():
for f in changed_files):
# run_java_style_checks()
pass
- if not changed_files or any(f.endswith(".py") for f in changed_files):
+ if not changed_files or any(f.endswith("lint-python")
+ or f.endswith("tox.ini")
+ or f.endswith(".py")
+ for f in changed_files):
run_python_style_checks()
if not changed_files or any(f.endswith(".R")
or f.endswith("lint-r")
http://git-wip-us.apache.org/repos/asf/spark/blob/7af1a325/dev/tox.ini
----------------------------------------------------------------------
diff --git a/dev/tox.ini b/dev/tox.ini
index eb8b1eb..583c1ea 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[pep8]
-ignore=E402,E731,E241,W503,E226
+[pycodestyle]
+ignore=E402,E731,E241,W503,E226,E722,E741,E305
max-line-length=100
exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org