You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ar...@apache.org on 2022/06/29 04:28:31 UTC
[tvm] 07/37: Fix all scripts which reference python3 in the docker containers.
This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch areusch/freeze-dependencies
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit fac50636e310d5b73822dbfa5dc1b0c0f7c4b39a
Author: Andrew Reusch <ar...@gmail.com>
AuthorDate: Mon May 23 18:33:03 2022 -0700
Fix all scripts which reference python3 in the docker containers.
---
tests/lint/blocklint.sh | 6 ++++--
tests/lint/check_asf_header.sh | 4 ++--
tests/lint/cpplint.sh | 4 ++--
tests/lint/filter_untracked.py | 21 ++++++---------------
tests/lint/flake8.sh | 2 +-
tests/lint/git-black.sh | 8 ++++----
tests/lint/jnilint.sh | 2 +-
tests/lint/pylint.sh | 12 ++++++------
tests/scripts/setup-pytest-env.sh | 4 ++--
tests/scripts/task_lint.sh | 4 ++++
tests/scripts/task_mypy.sh | 20 ++++++++++----------
11 files changed, 42 insertions(+), 45 deletions(-)
diff --git a/tests/lint/blocklint.sh b/tests/lint/blocklint.sh
index 7525bfa64c..9222ef82d1 100755
--- a/tests/lint/blocklint.sh
+++ b/tests/lint/blocklint.sh
@@ -25,8 +25,10 @@ do
if ! [ "$dir" == "3rdparty" ]; then
for subdir in $(find $dir -type d -print)
do
- blocklint --blocklist blacklist,whitelist,white\ box,master\ ,\ master,master_,_master,slave $subdir \
- --skip-files tests/lint/blocklint.sh,tests/lint/pylintrc,conda/recipe/meta.yaml,rust/tvm-sys/build.rs,docs/topic/vta/dev/hardware.rst,src/target/source/codegen_vhls.cc,tests/micro/zephyr/test_utils.py
+ "${TVM_VENV}/bin/blocklint" \
+ --blocklist blacklist,whitelist,white\ box,master\ ,\ master,master_,_master,slave \
+ $subdir \
+ --skip-files tests/lint/blocklint.sh,tests/lint/pylintrc,conda/recipe/meta.yaml,rust/tvm-sys/build.rs,docs/topic/vta/dev/hardware.rst,src/target/source/codegen_vhls.cc,tests/micro/zephyr/test_utils.py
done
fi
done
diff --git a/tests/lint/check_asf_header.sh b/tests/lint/check_asf_header.sh
index 579069bb5b..bf71453899 100755
--- a/tests/lint/check_asf_header.sh
+++ b/tests/lint/check_asf_header.sh
@@ -52,7 +52,7 @@ if [ ${filter_untracked} -eq 1 ]; then
echo "NOTE: --local flag present, filtering untracked files"
processed_rat_output="${rat_output}-processed"
cat ${rat_output} | sed 's/^== File: //g' | \
- python3 $(dirname "$0")/filter_untracked.py | \
+ "${TVM_VENV}/bin/python3" $(dirname "$0")/filter_untracked.py | \
sed 's/^/== File: /g' >"${processed_rat_output}"
rat_output="${processed_rat_output}"
fi
@@ -63,7 +63,7 @@ if [ $success -eq 0 ]; then
echo "No files violate ASF header check"
else
if [ $fix -eq 1 ]; then
- python3 tests/lint/add_asf_header.py "${rat_output}"
+ "${TVM_VENV}/bin/python3" tests/lint/add_asf_header.py "${rat_output}"
echo "Ran add_asf_header.py to fix the following files:"
else
echo "Need to add ASF header to the following files."
diff --git a/tests/lint/cpplint.sh b/tests/lint/cpplint.sh
index 6c01f0eb0a..34908ff6df 100755
--- a/tests/lint/cpplint.sh
+++ b/tests/lint/cpplint.sh
@@ -19,8 +19,8 @@
set -e
echo "Running 2 cpplints (VTA and TVM)..."
-python3 3rdparty/dmlc-core/scripts/lint.py --quiet vta cpp vta/include vta/src
-python3 3rdparty/dmlc-core/scripts/lint.py --quiet tvm cpp \
+"${TVM_VENV}/bin/python3" 3rdparty/dmlc-core/scripts/lint.py --quiet vta cpp vta/include vta/src
+"${TVM_VENV}/bin/python3" 3rdparty/dmlc-core/scripts/lint.py --quiet tvm cpp \
include src \
examples/extension/src examples/graph_executor/src \
tests/cpp tests/crt \
diff --git a/tests/lint/filter_untracked.py b/tests/lint/filter_untracked.py
index 3227bc3f18..990b82b8ae 100644
--- a/tests/lint/filter_untracked.py
+++ b/tests/lint/filter_untracked.py
@@ -22,27 +22,18 @@ import subprocess
import sys
-def check_output(args, **kw):
- proc = subprocess.Popen(args, **kw, stdout=subprocess.PIPE)
- out, _ = proc.communicate()
- if proc.returncode:
- sys.stderr.write("exited with code %d: %s\n" % (proc.returncode, " ".join(args)))
- sys.exit(2)
-
- if sys.version_info[0] == 2:
- return unicode(out, "utf-8")
- else:
- return str(out, "utf-8")
-
-
def main():
script_dir = os.path.dirname(__file__) or os.getcwd()
- toplevel_dir = check_output(["git", "rev-parse", "--show-toplevel"], cwd=script_dir).strip("\n")
+ toplevel_dir = subprocess.check_output(
+ ["git", "rev-parse", "--show-toplevel"], cwd=script_dir, encoding="utf-8"
+ ).strip("\n")
# NOTE: --ignore-submodules because this can drag in some problems related to mounting a git
# worktree in the docker VM in a different location than it exists on the host. The problem
# isn't quite clear, but anyhow it shouldn't be necessary to filter untracked files in
# submodules here.
- git_status_output = check_output(["git", "status", "-s", "--ignored"], cwd=toplevel_dir)
+ git_status_output = subprocess.check_output(
+ ["git", "status", "-s", "--ignored"], encoding="utf-8", cwd=toplevel_dir
+ )
untracked = [
line[3:]
for line in git_status_output.split("\n")
diff --git a/tests/lint/flake8.sh b/tests/lint/flake8.sh
index 63dad38166..36b891fc78 100755
--- a/tests/lint/flake8.sh
+++ b/tests/lint/flake8.sh
@@ -18,4 +18,4 @@
set -e
-python3 -m flake8 . --count --select=E9,F63,F7 --show-source --statistics
+"${TVM_VENV}/bin/python3" -m flake8 . --count --select=E9,F63,F7 --show-source --statistics
diff --git a/tests/lint/git-black.sh b/tests/lint/git-black.sh
index 647aba9540..af7aa099bc 100755
--- a/tests/lint/git-black.sh
+++ b/tests/lint/git-black.sh
@@ -49,13 +49,13 @@ done
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
-if [ ! -x "$(command -v black)" ]; then
+if [ ! -x "$(command -v "${TVM_VENV}/bin/black")" ]; then
echo "Cannot find black"
exit 1
fi
# Print out specific version
-VERSION=$(black --version)
+VERSION=$("${TVM_VENV}/bin/python3" -m black --version)
echo "black version: $VERSION"
# Compute Python files which changed to compare.
@@ -74,8 +74,8 @@ fi
if [[ "$INPLACE_FORMAT" == "true" ]]; then
echo "Running black on Python files against revision" $REVISION:
- python3 -m black ${FILES[@]}
+ "${TVM_VENV}/bin/python3" -m black ${FILES[@]}
else
echo "Running black in checking mode"
- python3 -m black --diff --check ${FILES[@]}
+ "${TVM_VENV}/bin/python3" -m black --diff --check ${FILES[@]}
fi
diff --git a/tests/lint/jnilint.sh b/tests/lint/jnilint.sh
index ec359a5b89..2768bed875 100755
--- a/tests/lint/jnilint.sh
+++ b/tests/lint/jnilint.sh
@@ -18,4 +18,4 @@
set -e
-python3 3rdparty/dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src
+"${TVM_VENV}/bin/python3" 3rdparty/dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src
diff --git a/tests/lint/pylint.sh b/tests/lint/pylint.sh
index 39568fd341..071d49c789 100755
--- a/tests/lint/pylint.sh
+++ b/tests/lint/pylint.sh
@@ -17,9 +17,9 @@
# under the License.
set -euxo pipefail
-python3 -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc
-python3 -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc
-python3 -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc
-python3 -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc
-python3 -m pylint tests/python/relay/aot/*.py --rcfile="$(dirname "$0")"/pylintrc
-python3 -m pylint tests/python/ci --rcfile="$(dirname "$0")"/pylintrc
+"${TVM_VENV}/bin/python3" -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc
+"${TVM_VENV}/bin/python3" -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc
+"${TVM_VENV}/bin/python3" -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc
+"${TVM_VENV}/bin/python3" -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc
+"${TVM_VENV}/bin/python3" -m pylint tests/python/relay/aot/*.py --rcfile="$(dirname "$0")"/pylintrc
+"${TVM_VENV}/bin/python3" -m pylint tests/python/ci --rcfile="$(dirname "$0")"/pylintrc
diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh
index 63145c9909..d0a2ab17bb 100755
--- a/tests/scripts/setup-pytest-env.sh
+++ b/tests/scripts/setup-pytest-env.sh
@@ -39,7 +39,7 @@ function cleanup() {
set +x
if [ "${#pytest_errors[@]}" -gt 0 ]; then
echo "These pytest invocations failed, the results can be found in the Jenkins 'Tests' tab or by scrolling up through the raw logs here."
- python3 tests/scripts/pytest_wrapper.py "${pytest_errors[@]}"
+ "${TVM_VENV}/bin/python3" tests/scripts/pytest_wrapper.py "${pytest_errors[@]}"
exit 1
fi
set -x
@@ -59,7 +59,7 @@ function run_pytest() {
suite_name="${test_suite_name}-${ffi_type}"
exit_code=0
- TVM_FFI=${ffi_type} python3 -m pytest \
+ TVM_FFI=${ffi_type} "${TVM_VENV}/bin/python3" -m pytest \
-o "junit_suite_name=${suite_name}" \
"--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \
"--junit-prefix=${ffi_type}" \
diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh
index a05f7ca36b..bb5b7e5071 100755
--- a/tests/scripts/task_lint.sh
+++ b/tests/scripts/task_lint.sh
@@ -28,6 +28,8 @@ trap cleanup 0
# These shards are solely for CI to enable the lint job to have some parallelism.
function shard1 {
+ . "${TVM_VENV}/bin/activate"
+
echo "Convert scripts to Python..."
tests/scripts/task_convert_scripts_to_python.sh
@@ -57,6 +59,8 @@ function shard1 {
}
function shard2 {
+ . "${TVM_VENV}/bin/activate"
+
echo "Linting the Python code with pylint..."
tests/lint/pylint.sh
diff --git a/tests/scripts/task_mypy.sh b/tests/scripts/task_mypy.sh
index 1ef7db5894..2cbe0ed723 100755
--- a/tests/scripts/task_mypy.sh
+++ b/tests/scripts/task_mypy.sh
@@ -21,26 +21,26 @@ set -euxo pipefail
source tests/scripts/setup-pytest-env.sh
echo "Checking MyPy Type defs in the TensorIR schedule package."
-mypy --check-untyped-defs python/tvm/tir/schedule
+"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/tir/schedule
echo "Checking MyPy Type defs in the meta schedule package."
-mypy --check-untyped-defs python/tvm/meta_schedule
+"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/meta_schedule
echo "Checking MyPy Type defs in the analysis package."
-mypy --check-untyped-defs python/tvm/tir/analysis/
+"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/tir/analysis/
echo "Checking MyPy Type defs in the transform package."
-mypy --check-untyped-defs python/tvm/tir/transform/
+"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/tir/transform/
echo "Checking MyPy Type defs in the TIR package with unittest"
-MYPYPATH=$TVM_PATH/python mypy --check-untyped-defs tests/python/unittest/test_tvmscript_type.py
+MYPYPATH=$TVM_PATH/python "${TVM_VENV}/bin/mypy" --check-untyped-defs tests/python/unittest/test_tvmscript_type.py
echo "Checking MyPy Type defs in tvm.relay.op.contrib"
-mypy --disallow-untyped-defs python/tvm/relay/op/contrib/cublas.py
-mypy --disallow-untyped-defs python/tvm/relay/op/contrib/cudnn.py
-mypy --disallow-untyped-defs python/tvm/relay/op/contrib/te_target.py
-mypy --disallow-untyped-defs python/tvm/relay/op/contrib/tensorrt.py
+"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/cublas.py
+"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/cudnn.py
+"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/te_target.py
+"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/tensorrt.py
#TODO(@mikepapadim): This is failing atm
# echo "Checking MyPy Type defs in the tvm.relay.backend.contrib.ethosu package."
-# mypy --check-untyped-defs python/tvm/relay/backend/contrib/ethosu/
+# "${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/relay/backend/contrib/ethosu/