You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2023/12/07 09:33:57 UTC
(flink-connector-shared-utils) branch ci_utils updated: [FLINK-33556][CI] Test infrastructure for externalized python code
This is an automated email from the ASF dual-hosted git repository.
gaborgsomogyi pushed a commit to branch ci_utils
in repository https://gitbox.apache.org/repos/asf/flink-connector-shared-utils.git
The following commit(s) were added to refs/heads/ci_utils by this push:
new 7691962 [FLINK-33556][CI] Test infrastructure for externalized python code
7691962 is described below
commit 7691962b8031536a2baa4d39252a38198ba91dc5
Author: pvary <pe...@gmail.com>
AuthorDate: Thu Dec 7 10:33:52 2023 +0100
[FLINK-33556][CI] Test infrastructure for externalized python code
---
.github/workflows/python_ci.yml | 86 ++++
python/README.md | 6 +
python/build-wheels.sh | 52 +++
python/glibc_version_fix.h | 17 +
python/install_command.sh | 31 ++
python/lint-python.sh | 876 ++++++++++++++++++++++++++++++++++++++++
6 files changed, 1068 insertions(+)
diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml
new file mode 100644
index 0000000..89e10d9
--- /dev/null
+++ b/.github/workflows/python_ci.yml
@@ -0,0 +1,86 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+on:
+ workflow_call:
+ inputs:
+ flink_version:
+ description: "Flink version to test against."
+ required: true
+ type: string
+ timeout_global:
+ description: "The timeout in minutes for the entire workflow."
+ required: false
+ type: number
+ default: 80
+ timeout_test:
+ description: "The timeout in minutes for the test compile&step."
+ required: false
+ type: number
+ default: 50
+ connector_branch:
+ description: "Branch that need to be checked out"
+ required: false
+ type: string
+
+jobs:
+ python_test:
+ runs-on: ubuntu-latest
+ timeout-minutes: ${{ inputs.timeout_global }}
+ env:
+ MVN_COMMON_OPTIONS: -U -B --no-transfer-progress -Dflink.version=${{ inputs.flink_version }} -DskipTests
+ MVN_CONNECTION_OPTIONS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
+ MVN_BUILD_OUTPUT_FILE: "/tmp/mvn_build_output.out"
+ steps:
+ - run: echo "Running CI pipeline for JDK version 8"
+
+ - name: Check out repository code
+ uses: actions/checkout@v3
+ with:
+ ref: "${{ inputs.connector_branch }}"
+
+ - name: Set JDK
+ uses: actions/setup-java@v3
+ with:
+ java-version: 8
+ distribution: 'temurin'
+ cache: 'maven'
+
+ - name: Set Maven 3.8.6
+ uses: stCarolas/setup-maven@v4.5
+ with:
+ maven-version: 3.8.6
+
+ - name: Compile
+ timeout-minutes: ${{ inputs.timeout_test }}
+ run: |
+ set -o pipefail
+
+ mvn clean install ${MVN_COMMON_OPTIONS} \
+ ${{ env.MVN_CONNECTION_OPTIONS }} \
+ -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties \
+ | tee ${{ env.MVN_BUILD_OUTPUT_FILE }}
+
+ - name: Run Python test
+ timeout-minutes: ${{ inputs.timeout_test }}
+ run: |
+ set -o pipefail
+
+ cd flink-python
+ chmod a+x dev/*
+ ./dev/lint-python.sh -e mypy,sphinx | tee ${{ env.MVN_BUILD_OUTPUT_FILE }}
diff --git a/python/README.md b/python/README.md
new file mode 100644
index 0000000..0dc8217
--- /dev/null
+++ b/python/README.md
@@ -0,0 +1,6 @@
+This directory contains commonly used scripts for testing and creating python packages for connectors.
+
+The original version of the files are based on
+https://github.com/apache/flink/tree/release-1.17.2/flink-python/dev.
+
+Created FLINK-33762 to make these scripts versioned to allow backward incompatible changes in the future.
diff --git a/python/build-wheels.sh b/python/build-wheels.sh
new file mode 100755
index 0000000..7f18a91
--- /dev/null
+++ b/python/build-wheels.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e -x
+
+## 1. install python env
+dev/lint-python.sh -s py_env
+
+PY_ENV_DIR=`pwd`/dev/.conda/envs
+py_env=("3.7" "3.8" "3.9" "3.10")
+## 2. install dependency
+for ((i=0;i<${#py_env[@]};i++)) do
+ ${PY_ENV_DIR}/${py_env[i]}/bin/pip install -r dev/dev-requirements.txt
+done
+
+## 3. build wheels
+for ((i=0;i<${#py_env[@]};i++)) do
+ if [[ "$(uname)" != "Darwin" ]]; then
+ # force the linker to use the older glibc version in Linux
+ export CFLAGS="-I. -include dev/glibc_version_fix.h"
+ fi
+ ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel
+done
+
+## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux
+if [[ "$(uname)" != "Darwin" ]]; then
+ source `pwd`/dev/.conda/bin/activate
+ # 4.1 install patchelf
+ conda install -c conda-forge patchelf=0.11 -y
+ # 4.2 install auditwheel
+ pip install auditwheel==3.2.0
+ # 4.3 convert Linux wheel
+ for wheel_file in dist/*.whl; do
+ auditwheel repair ${wheel_file} -w dist
+ rm -f ${wheel_file}
+ done
+ source deactivate
+fi
+## see the result
+ls -al dist/
diff --git a/python/glibc_version_fix.h b/python/glibc_version_fix.h
new file mode 100755
index 0000000..f260492
--- /dev/null
+++ b/python/glibc_version_fix.h
@@ -0,0 +1,17 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+__asm__(".symver memcpy,memcpy@GLIBC_2.2.5");
diff --git a/python/install_command.sh b/python/install_command.sh
new file mode 100755
index 0000000..f914200
--- /dev/null
+++ b/python/install_command.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+retry_times=3
+install_command="python -m pip install $@"
+${install_command}
+status=$?
+while [[ ${status} -ne 0 ]] && [[ ${retry_times} -gt 0 ]]; do
+ retry_times=$((retry_times-1))
+ # sleep 3 seconds and then reinstall.
+ sleep 3
+ ${install_command}
+ status=$?
+done
+
+exit ${status}
diff --git a/python/lint-python.sh b/python/lint-python.sh
new file mode 100755
index 0000000..1c0452b
--- /dev/null
+++ b/python/lint-python.sh
@@ -0,0 +1,876 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# lint-python.sh
+# This script will prepare a virtual environment for many kinds of checks, such as tox check, flake8 check.
+#
+# You can refer to the README.MD in ${flink-python} to learn how easy to run the script.
+#
+
+# Download some software, such as miniconda.sh
+function download() {
+ local DOWNLOAD_STATUS=
+ if hash "wget" 2>/dev/null; then
+ # because of the difference of all versions of wget, so we turn of the option --show-progress
+ wget "$1" -O "$2" -q -T20 -t3
+ DOWNLOAD_STATUS="$?"
+ else
+ curl "$1" -o "$2" --progress-bar --connect-timeout 20 --retry 3
+ DOWNLOAD_STATUS="$?"
+ fi
+ if [ $DOWNLOAD_STATUS -ne 0 ]; then
+ echo "Download failed.You can try again"
+ exit $DOWNLOAD_STATUS
+ fi
+}
+
+# Printing infos both in log and console
+function print_function() {
+ local STAGE_LENGTH=48
+ local left_edge_len=
+ local right_edge_len=
+ local str
+ case "$1" in
+ "STAGE")
+ left_edge_len=$(((STAGE_LENGTH-${#2})/2))
+ right_edge_len=$((STAGE_LENGTH-${#2}-left_edge_len))
+ str="$(seq -s "=" $left_edge_len | tr -d "[:digit:]")""$2""$(seq -s "=" $right_edge_len | tr -d "[:digit:]")"
+ ;;
+ "STEP")
+ str="$2"
+ ;;
+ *)
+ str="seq -s "=" $STAGE_LENGTH | tr -d "[:digit:]""
+ ;;
+ esac
+ echo $str | tee -a $LOG_FILE
+}
+
+function regexp_match() {
+ if echo $1 | grep -e $2 &>/dev/null; then
+ echo true
+ else
+ echo false
+ fi
+}
+
+# decide whether a array contains a specified element.
+function contains_element() {
+ arr=($1)
+ if echo "${arr[@]}" | grep -w "$2" &>/dev/null; then
+ echo true
+ else
+ echo false
+ fi
+}
+
+# Checkpoint the stage:step for convenient to re-exec the script with
+# skipping those success steps.
+# The format is "${Stage}:${Step}". e.g. Install:4
+function checkpoint_stage() {
+ if [ ! -d `dirname $STAGE_FILE` ]; then
+ mkdir -p `dirname $STAGE_FILE`
+ fi
+ echo "$1:$2">"$STAGE_FILE"
+}
+
+# Restore the stage:step
+function restore_stage() {
+ if [ -f "$STAGE_FILE" ]; then
+ local lines=$(awk '{print NR}' $STAGE_FILE)
+ if [ $lines -eq 1 ]; then
+ local first_field=$(cat $STAGE_FILE | cut -d ":" -f 1)
+ local second_field=$(cat $STAGE_FILE | cut -d ":" -f 2)
+ check_valid_stage $first_field $second_field
+ if [ $? -eq 0 ]; then
+ STAGE=$first_field
+ STEP=$second_field
+ return
+ fi
+ fi
+ fi
+ STAGE="install"
+ STEP=0
+}
+
+# Decide whether the stage:step is valid.
+function check_valid_stage() {
+ case $1 in
+ "install")
+ if [ $2 -le $STAGE_INSTALL_STEPS ] && [ $2 -ge 0 ]; then
+ return 0
+ fi
+ ;;
+ *)
+ ;;
+ esac
+ return 1
+}
+
+function parse_component_args() {
+ local REAL_COMPONENTS=()
+ for component in ${INSTALLATION_COMPONENTS[@]}; do
+ # because all other components depends on conda, the install of conda is
+ # required component.
+ if [[ "$component" == "basic" ]] || [[ "$component" == "miniconda" ]]; then
+ continue
+ fi
+ if [[ "$component" == "all" ]]; then
+ component="environment"
+ fi
+ if [[ `contains_element "${SUPPORTED_INSTALLATION_COMPONENTS[*]}" "${component}"` = true ]]; then
+ REAL_COMPONENTS+=(${component})
+ else
+ echo "unknown install component ${component}, currently we only support installing basic,py_env,tox,flake8,sphinx,mypy,all."
+ exit 1
+ fi
+ done
+ if [[ `contains_element "${REAL_COMPONENTS[*]}" "environment"` = false ]]; then
+ SUPPORTED_INSTALLATION_COMPONENTS=(${REAL_COMPONENTS[@]})
+ fi
+}
+
+# For convenient to index something binded to OS.
+# Now, the script only make a distinction between 'Mac' and 'Non-Mac'.
+function get_os_index() {
+ if [ $1 == "Darwin" ]; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+# Considering the file size of miniconda.sh,
+# "wget" is better than "curl" in the weak network environment.
+function install_wget() {
+ if [ $1 == "Darwin" ]; then
+ hash "brew" 2>/dev/null
+ if [ $? -ne 0 ]; then
+ $((/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)") 2>&1 >/dev/null)
+ if [ $? -ne 0 ]; then
+ echo "Failed to install brew"
+ exit 1
+ fi
+ fi
+
+ hash "wget" 2>/dev/null
+ if [ $? -ne 0 ]; then
+ brew install wget 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "Failed to install wget"
+ exit 1
+ fi
+ fi
+ fi
+}
+
+# The script choose miniconda as our package management tool.
+# The script use miniconda to create all kinds of python versions and
+# some packages including checks such as tox and flake8.
+
+function install_miniconda() {
+ OS_TO_CONDA_URL=("https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh" \
+ "https://repo.continuum.io/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh")
+ if [ ! -f "$CONDA_INSTALL" ]; then
+ print_function "STEP" "download miniconda..."
+ download ${OS_TO_CONDA_URL[$1]} $CONDA_INSTALL_SH
+ chmod +x $CONDA_INSTALL_SH
+ if [ $? -ne 0 ]; then
+ echo "Please manually chmod +x $CONDA_INSTALL_SH"
+ exit 1
+ fi
+ if [ -d "$CURRENT_DIR/.conda" ]; then
+ rm -rf "$CURRENT_DIR/.conda"
+ if [ $? -ne 0 ]; then
+ echo "Please manually rm -rf $CURRENT_DIR/.conda directory.\
+ Then retry to exec the script."
+ exit 1
+ fi
+ fi
+ print_function "STEP" "download miniconda... [SUCCESS]"
+ fi
+
+ if [ ! -d "$CURRENT_DIR/.conda" ]; then
+ print_function "STEP" "installing conda..."
+ $CONDA_INSTALL_SH -b -p $CURRENT_DIR/.conda 2>&1 >/dev/null
+
+ # orjson depend on pip >= 20.3
+ print_function "STEP" "upgrade pip..."
+ $CURRENT_DIR/.conda/bin/python -m pip install --upgrade pip 2>&1 >/dev/null
+ print_function "STEP" "upgrade pip... [SUCCESS]"
+
+ if [ $? -ne 0 ]; then
+ echo "install miniconda failed"
+ exit $CONDA_INSTALL_STATUS
+ fi
+ print_function "STEP" "install conda ... [SUCCESS]"
+ fi
+}
+
+# Install some kinds of py env.
+function install_py_env() {
+ if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then
+ py_env=("3.10")
+ else
+ py_env=("3.7" "3.8" "3.9" "3.10")
+ fi
+ for ((i=0;i<${#py_env[@]};i++)) do
+ if [ -d "$CURRENT_DIR/.conda/envs/${py_env[i]}" ]; then
+ rm -rf "$CURRENT_DIR/.conda/envs/${py_env[i]}"
+ if [ $? -ne 0 ]; then
+ echo "rm -rf $CURRENT_DIR/.conda/envs/${py_env[i]} failed, please \
+ rm -rf $CURRENT_DIR/.conda/envs/${py_env[i]} manually.\
+ Then retry to exec the script."
+ exit 1
+ fi
+ fi
+ print_function "STEP" "installing python${py_env[i]}..."
+ max_retry_times=3
+ retry_times=0
+ install_command="$CONDA_PATH create --name ${py_env[i]} -y -q python=${py_env[i]}"
+ ${install_command} 2>&1 >/dev/null
+ status=$?
+ while [[ ${status} -ne 0 ]] && [[ ${retry_times} -lt ${max_retry_times} ]]; do
+ retry_times=$((retry_times+1))
+ # sleep 3 seconds and then reinstall.
+ sleep 3
+ echo "conda install ${py_env[i]} retrying ${retry_times}/${max_retry_times}"
+ ${install_command} 2>&1 >/dev/null
+ status=$?
+ done
+ if [[ ${status} -ne 0 ]]; then
+ echo "conda install ${py_env[i]} failed after retrying ${max_retry_times} times.\
+ You can retry to execute the script again."
+ exit 1
+ fi
+ print_function "STEP" "install python${py_env[i]}... [SUCCESS]"
+ done
+}
+
+# Install tox.
+# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh
+function install_tox() {
+ source $CONDA_HOME/bin/activate
+ if [ -f "$TOX_PATH" ]; then
+ $PIP_PATH uninstall tox -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "pip uninstall tox failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ fi
+
+ # tox 3.14.0 depends on both 0.19 and 0.23 of importlib_metadata at the same time and
+ # conda will try to install both these two versions and it will cause problems occasionally.
+ # Using pip as the package manager could avoid this problem.
+ $CURRENT_DIR/install_command.sh -q virtualenv==20.10.0 tox==3.14.0 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "pip install tox failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ conda deactivate
+}
+
+# Install flake8.
+# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh
+function install_flake8() {
+ source $CONDA_HOME/bin/activate
+ if [ -f "$FLAKE8_PATH" ]; then
+ $PIP_PATH uninstall flake8 -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "pip uninstall flake8 failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ fi
+
+ $CURRENT_DIR/install_command.sh -q flake8==4.0.1 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "pip install flake8 failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ conda deactivate
+}
+
+# Install sphinx.
+# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh
+function install_sphinx() {
+ source $CONDA_HOME/bin/activate
+ if [ -f "$SPHINX_PATH" ]; then
+ $PIP_PATH uninstall Sphinx -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "pip uninstall sphinx failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ fi
+
+ $CURRENT_DIR/install_command.sh -q Sphinx==4.5.0 importlib-metadata==4.4.0 Docutils==0.17.1 pydata_sphinx_theme==0.11.0 sphinx_mdinclude==0.5.3 "Jinja2<3.1.0" 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "pip install sphinx failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ conda deactivate
+}
+
+
+# Install mypy.
+# In some situations, you need to run the script with "sudo". e.g. sudo ./lint-python.sh
+function install_mypy() {
+ source ${CONDA_HOME}/bin/activate
+ if [[ -f "$MYPY_PATH" ]]; then
+ ${PIP_PATH} uninstall mypy -y -q 2>&1 >/dev/null
+ if [[ $? -ne 0 ]]; then
+ echo "pip uninstall mypy failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ fi
+ ${CURRENT_DIR}/install_command.sh -q mypy==0.790 2>&1 >/dev/null
+ if [[ $? -ne 0 ]]; then
+ echo "pip install mypy failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ conda deactivate
+}
+
+function need_install_component() {
+ if [[ `contains_element "${SUPPORTED_INSTALLATION_COMPONENTS[*]}" "$1"` = true ]]; then
+ echo true
+ else
+ echo false
+ fi
+}
+
+
+# In this function, the script will prepare all kinds of python environments and checks.
+function install_environment() {
+
+ print_function "STAGE" "installing environment"
+
+ local sys_os=`uname -s`
+ #get the index of the SUPPORT_OS array for convinient to intall tool.
+ get_os_index $sys_os
+ local os_index=$?
+
+ # step-1 install wget
+ # the file size of the miniconda.sh is too big to use "wget" tool to download instead
+ # of the "curl" in the weak network environment.
+ if [ $STEP -lt 1 ]; then
+ print_function "STEP" "installing wget..."
+ install_wget ${SUPPORT_OS[$os_index]}
+ STEP=1
+ checkpoint_stage $STAGE $STEP
+ print_function "STEP" "install wget... [SUCCESS]"
+ fi
+
+ # step-2 install miniconda
+ if [ $STEP -lt 2 ]; then
+ print_function "STEP" "installing miniconda..."
+ create_dir $CURRENT_DIR/download
+ install_miniconda $os_index
+ STEP=2
+ checkpoint_stage $STAGE $STEP
+ print_function "STEP" "install miniconda... [SUCCESS]"
+ fi
+
+ # step-3 install python environment which includes
+ # 3.7 3.8 3.9 3.10
+ if [ $STEP -lt 3 ] && [ `need_install_component "py_env"` = true ]; then
+ print_function "STEP" "installing python environment..."
+ install_py_env
+ STEP=3
+ checkpoint_stage $STAGE $STEP
+ print_function "STEP" "install python environment... [SUCCESS]"
+ fi
+
+ # step-4 install tox
+ if [ $STEP -lt 4 ] && [ `need_install_component "tox"` = true ]; then
+ print_function "STEP" "installing tox..."
+ install_tox
+ STEP=4
+ checkpoint_stage $STAGE $STEP
+ print_function "STEP" "install tox... [SUCCESS]"
+ fi
+
+ # step-5 install flake8
+ if [ $STEP -lt 5 ] && [ `need_install_component "flake8"` = true ]; then
+ print_function "STEP" "installing flake8..."
+ install_flake8
+ STEP=5
+ checkpoint_stage $STAGE $STEP
+ print_function "STEP" "install flake8... [SUCCESS]"
+ fi
+
+ # step-6 install sphinx
+ if [ $STEP -lt 6 ] && [ `need_install_component "sphinx"` = true ]; then
+ print_function "STEP" "installing sphinx..."
+ install_sphinx
+ STEP=6
+ checkpoint_stage $STAGE $STEP
+ print_function "STEP" "install sphinx... [SUCCESS]"
+ fi
+
+ # step-7 install mypy
+ if [[ ${STEP} -lt 7 ]] && [[ `need_install_component "mypy"` = true ]]; then
+ print_function "STEP" "installing mypy..."
+ install_mypy
+ STEP=7
+ checkpoint_stage ${STAGE} ${STEP}
+ print_function "STEP" "install mypy... [SUCCESS]"
+ fi
+
+ print_function "STAGE" "install environment... [SUCCESS]"
+}
+
+# create dir if needed
+function create_dir() {
+ if [ ! -d $1 ]; then
+ mkdir -p $1
+ if [ $? -ne 0 ]; then
+ echo "mkdir -p $1 failed. you can mkdir manually or exec the script with \
+ the command: sudo ./lint-python.sh"
+ exit 1
+ fi
+ fi
+}
+
+# Set created py-env in $PATH for tox's creating virtual env
+function activate () {
+ if [ ! -d $CURRENT_DIR/.conda/envs ]; then
+ echo "For some unknown reasons, missing the directory $CURRENT_DIR/.conda/envs,\
+ you should exec the script with the option: -f"
+ exit 1
+ fi
+
+ for py_dir in $CURRENT_DIR/.conda/envs/*
+ do
+ PATH=$py_dir/bin:$PATH
+ done
+ export PATH 2>/dev/null
+ if [ $? -ne 0 ]; then
+ echo "For some unknown reasons, the py package is not complete,\
+ you should exec the script with the option: -f"
+ exit 1
+ fi
+}
+
+# Reset the $PATH
+function deactivate() {
+ # reset old environment variables
+ # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
+ if ! [ -z "${_OLD_PATH+_}" ] ; then
+ PATH="$_OLD_PATH"
+ export PATH
+ unset _OLD_PATH
+ fi
+}
+
+# Collect checks
+function collect_checks() {
+ if [ ! -z "$EXCLUDE_CHECKS" ] && [ ! -z "$INCLUDE_CHECKS" ]; then
+ echo "You can't use option -s and -e simultaneously."
+ exit 1
+ fi
+ if [ ! -z "$EXCLUDE_CHECKS" ]; then
+ for (( i = 0; i < ${#EXCLUDE_CHECKS[@]}; i++)); do
+ if [[ `contains_element "${SUPPORT_CHECKS[*]}" "${EXCLUDE_CHECKS[i]}_check"` = true ]]; then
+ SUPPORT_CHECKS=("${SUPPORT_CHECKS[@]/${EXCLUDE_CHECKS[i]}_check}")
+ else
+ echo "the check ${EXCLUDE_CHECKS[i]} is invalid."
+ exit 1
+ fi
+ done
+ fi
+ if [ ! -z "$INCLUDE_CHECKS" ]; then
+ REAL_SUPPORT_CHECKS=()
+ for (( i = 0; i < ${#INCLUDE_CHECKS[@]}; i++)); do
+ if [[ `contains_element "${SUPPORT_CHECKS[*]}" "${INCLUDE_CHECKS[i]}_check"` = true ]]; then
+ REAL_SUPPORT_CHECKS+=("${INCLUDE_CHECKS[i]}_check")
+ else
+ echo "the check ${INCLUDE_CHECKS[i]} is invalid."
+ exit 1
+ fi
+ done
+ SUPPORT_CHECKS=(${REAL_SUPPORT_CHECKS[@]})
+ fi
+}
+
+# If the check stage is needed
+function include_stage() {
+ if [[ `contains_element "${SUPPORT_CHECKS[*]}" "$1"` = true ]]; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+# get all supported checks functions
+function get_all_supported_checks() {
+ _OLD_IFS=$IFS
+ IFS=$'\n'
+ SUPPORT_CHECKS=()
+ for fun in $(declare -F); do
+ if [[ `regexp_match "$fun" "_check$"` = true ]]; then
+ SUPPORT_CHECKS+=("${fun:11}")
+ fi
+ done
+ IFS=$_OLD_IFS
+}
+
+# get all supported install components functions
+function get_all_supported_install_components() {
+ _OLD_IFS=$IFS
+ IFS=$'\n'
+ for fun in $(declare -F); do
+ if [[ `regexp_match "${fun:11}" "^install_"` = true ]]; then
+ SUPPORTED_INSTALLATION_COMPONENTS+=("${fun:19}")
+ fi
+ done
+ IFS=$_OLD_IFS
+ # we don't need to expose "install_wget" to user.
+ local DELETE_COMPONENTS=("wget")
+ local REAL_COMPONENTS=()
+ for component in ${SUPPORTED_INSTALLATION_COMPONENTS[@]}; do
+ if [[ `contains_element "${DELETE_COMPONENTS[*]}" "${component}"` = false ]]; then
+ REAL_COMPONENTS+=("${component}")
+ fi
+ done
+ SUPPORTED_INSTALLATION_COMPONENTS=(${REAL_COMPONENTS[@]})
+}
+
+# exec all selected check stages
+function check_stage() {
+ print_function "STAGE" "checks starting"
+ for fun in ${SUPPORT_CHECKS[@]}; do
+ $fun
+ done
+ echo "All the checks are finished, the detailed information can be found in: $LOG_FILE"
+}
+
+
+###############################################################All Checks Definitions###############################################################
+#########################
+# This part defines all check functions such as tox_check and flake8_check
+# We make a rule that all check functions are suffixed with _ check. e.g. tox_check, flake8_chek
+#########################
+# Tox check
+function tox_check() {
+ LATEST_PYTHON="py310"
+ print_function "STAGE" "tox checks"
+ # Set created py-env in $PATH for tox's creating virtual env
+ activate
+ # Ensure the permission of the scripts set correctly
+ chmod +x $FLINK_PYTHON_DIR/../build-target/bin/*
+ chmod +x $FLINK_PYTHON_DIR/dev/*
+
+ if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then
+ # Only run test in latest python version triggered by a Git push
+ $TOX_PATH -vv -c $FLINK_PYTHON_DIR/tox.ini -e ${LATEST_PYTHON} --recreate 2>&1 | tee -a $LOG_FILE
+ else
+ $TOX_PATH -vv -c $FLINK_PYTHON_DIR/tox.ini --recreate 2>&1 | tee -a $LOG_FILE
+ fi
+
+ TOX_RESULT=$((grep -c "congratulations :)" "$LOG_FILE") 2>&1)
+ if [ $TOX_RESULT -eq '0' ]; then
+ print_function "STAGE" "tox checks... [FAILED]"
+ else
+ print_function "STAGE" "tox checks... [SUCCESS]"
+ fi
+ # Reset the $PATH
+ deactivate
+ # If check failed, stop the running script.
+ if [ $TOX_RESULT -eq '0' ]; then
+ exit 1
+ fi
+}
+
+# Flake8 check
+function flake8_check() {
+ local PYTHON_SOURCE="$(find . \( -path ./dev -o -path ./.tox \) -prune -o -type f -name "*.py" -print )"
+
+ print_function "STAGE" "flake8 checks"
+ if [ ! -f "$FLAKE8_PATH" ]; then
+ echo "For some unknown reasons, the flake8 package is not complete,\
+ you should exec the script with the parameter: -f"
+ fi
+
+ if [[ ! "$PYTHON_SOURCE" ]]; then
+ echo "No python files found! Something is wrong exiting."
+ exit 1;
+ fi
+
+ # the return value of a pipeline is the status of the last command to exit
+ # with a non-zero status or zero if no command exited with a non-zero status
+ set -o pipefail
+ ($FLAKE8_PATH --config=tox.ini $PYTHON_SOURCE) 2>&1 | tee -a $LOG_FILE
+
+ PYCODESTYLE_STATUS=$?
+ if [ $PYCODESTYLE_STATUS -ne 0 ]; then
+ print_function "STAGE" "flake8 checks... [FAILED]"
+ # Stop the running script.
+ exit 1;
+ else
+ print_function "STAGE" "flake8 checks... [SUCCESS]"
+ fi
+}
+
+# Sphinx check
+function sphinx_check() {
+ export SPHINXBUILD=$SPHINX_PATH
+ # cd to $FLINK_PYTHON_DIR
+ pushd "$FLINK_PYTHON_DIR"/docs &> /dev/null
+ make clean
+
+ # the return value of a pipeline is the status of the last command to exit
+ # with a non-zero status or zero if no command exited with a non-zero status
+ set -o pipefail
+ (SPHINXOPTS="-a -W" make html) 2>&1 | tee -a $LOG_FILE
+
+ SPHINXBUILD_STATUS=$?
+ if [ $SPHINXBUILD_STATUS -ne 0 ]; then
+ print_function "STAGE" "sphinx checks... [FAILED]"
+ # Stop the running script.
+ exit 1;
+ else
+ print_function "STAGE" "sphinx checks... [SUCCESS]"
+ fi
+ popd
+}
+
+# mypy check
+function mypy_check() {
+ print_function "STAGE" "mypy checks"
+
+ # the return value of a pipeline is the status of the last command to exit
+ # with a non-zero status or zero if no command exited with a non-zero status
+ set -o pipefail
+ (${MYPY_PATH} --config-file tox.ini) 2>&1 | tee -a ${LOG_FILE}
+ TYPE_HINT_CHECK_STATUS=$?
+ if [ ${TYPE_HINT_CHECK_STATUS} -ne 0 ]; then
+ print_function "STAGE" "mypy checks... [FAILED]"
+ # Stop the running script.
+ exit 1;
+ else
+ print_function "STAGE" "mypy checks... [SUCCESS]"
+ fi
+}
+###############################################################All Checks Definitions###############################################################
+
+# CURRENT_DIR is "flink/flink-python/dev/"
+CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)"
+
+# FLINK_PYTHON_DIR is "flink/flink-python"
+FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR")
+
+# conda home path
+if [ -z "${FLINK_CONDA_HOME+x}" ]; then
+ CONDA_HOME="$CURRENT_DIR/.conda"
+ ENV_HOME="$CONDA_HOME"
+else
+ CONDA_HOME=$FLINK_CONDA_HOME
+ ENV_HOME="${CONDA_PREFIX-$CONDA_HOME}"
+fi
+
+# conda path
+CONDA_PATH=$CONDA_HOME/bin/conda
+
+# pip path
+PIP_PATH=$ENV_HOME/bin/pip
+
+# tox path
+TOX_PATH=$ENV_HOME/bin/tox
+
+# flake8 path
+FLAKE8_PATH=$ENV_HOME/bin/flake8
+
+# sphinx path
+SPHINX_PATH=$ENV_HOME/bin/sphinx-build
+
+# mypy path
+MYPY_PATH=$ENV_HOME/bin/mypy
+
+_OLD_PATH="$PATH"
+
+SUPPORT_OS=("Darwin" "Linux")
+
+# the file stores the success step in installing progress.
+STAGE_FILE=$CURRENT_DIR/.stage.txt
+
+# the dir includes all kinds of py env installed.
+VIRTUAL_ENV=$CONDA_HOME/envs
+
+LOG_DIR=$CURRENT_DIR/log
+
+if [ "$FLINK_IDENT_STRING" == "" ]; then
+ FLINK_IDENT_STRING="$USER"
+fi
+if [ "$HOSTNAME" == "" ]; then
+ HOSTNAME="$HOST"
+fi
+
+# the log file stores the checking result.
+LOG_FILE=$LOG_DIR/flink-$FLINK_IDENT_STRING-python-$HOSTNAME.log
+create_dir $LOG_DIR
+
+# clean LOG_FILE content
+echo >$LOG_FILE
+
+# miniconda script
+CONDA_INSTALL_SH=$CURRENT_DIR/download/miniconda.sh
+
+# stage "install" includes the num of steps.
+STAGE_INSTALL_STEPS=7
+
+# whether force to restart the script.
+FORCE_START=0
+
+SUPPORT_CHECKS=()
+
+# search all supported check functions and put them into SUPPORT_CHECKS array
+get_all_supported_checks
+
+EXCLUDE_CHECKS=""
+
+INCLUDE_CHECKS=""
+
+SUPPORTED_INSTALLATION_COMPONENTS=()
+
+# search all supported install functions and put them into SUPPORTED_INSTALLATION_COMPONENTS array
+get_all_supported_install_components
+
+INSTALLATION_COMPONENTS=()
+
+# whether remove the installed python environment.
+CLEAN_UP_FLAG=0
+
+# parse_opts
+USAGE="
+usage: $0 [options]
+-h print this help message and exit
+-f force to exec from the progress of installing environment
+-s [basic,py_env,tox,flake8,sphinx,mypy,all]
+ install environment with specified components which split by comma(,)
+ note:
+ This option is used to install environment components and will skip all subsequent checks,
+ so do not use this option with -e,-i simultaneously.
+-e [tox,flake8,sphinx,mypy]
+ exclude checks which split by comma(,)
+-i [tox,flake8,sphinx,mypy]
+ include checks which split by comma(,)
+-l list all checks supported.
+Examples:
+ ./lint-python -s basic => install environment with basic components.
+ ./lint-python -s py_env => install environment with python env(3.7,3.8,3.9,3.10).
+ ./lint-python -s all => install environment with all components such as python env,tox,flake8,sphinx,mypy etc.
+ ./lint-python -s tox,flake8 => install environment with tox,flake8.
+ ./lint-python -s tox -f => reinstall environment with tox.
+ ./lint-python -e tox,flake8 => exclude checks tox,flake8.
+ ./lint-python -i flake8 => include checks flake8.
+ ./lint-python => exec all checks.
+ ./lint-python -f => reinstall environment with all components and exec all checks.
+ ./lint-python -l => list all checks supported.
+ ./lint-python -r => clean up python environment.
+"
+while getopts "hfs:i:e:lr" arg; do
+ case "$arg" in
+ h)
+ printf "%s\\n" "$USAGE"
+ exit 2
+ ;;
+ f)
+ FORCE_START=1
+ ;;
+ s)
+ INSTALLATION_COMPONENTS=($(echo $OPTARG | tr ',' ' ' ))
+ ;;
+ e)
+ EXCLUDE_CHECKS=($(echo $OPTARG | tr ',' ' ' ))
+ ;;
+ i)
+ INCLUDE_CHECKS=($(echo $OPTARG | tr ',' ' ' ))
+ ;;
+ l)
+ printf "current supported checks includes:\n"
+ for fun in ${SUPPORT_CHECKS[@]}; do
+ echo ${fun%%_check*}
+ done
+ exit 2
+ ;;
+ r)
+ printf "clean up python environment:\n"
+ CLEAN_UP_FLAG=1
+ ;;
+ ?)
+ printf "ERROR: did not recognize option '%s', please try -h\\n" "$1"
+ exit 1
+ ;;
+ esac
+done
+
+# decides whether to skip check stage
+skip_checks=0
+
+if [[ ${CLEAN_UP_FLAG} -eq 1 ]]; then
+ printf "clean up python environment"
+ rm -rf ${CONDA_HOME}
+ rm -rf ${STAGE_FILE}
+ rm -rf ${FLINK_PYTHON_DIR}/.tox
+ skip_checks=1
+fi
+
+if [ ! -z "$INSTALLATION_COMPONENTS" ]; then
+ parse_component_args
+ skip_checks=1
+fi
+
+# collect checks according to the options
+collect_checks
+
+# If exec the script with the param: -f, all progress will be re-run
+if [ $FORCE_START -eq 1 ]; then
+ STAGE="install"
+ STEP=0
+ checkpoint_stage $STAGE $STEP
+else
+ restore_stage
+fi
+
+# install environment
+if [[ ${CLEAN_UP_FLAG} -eq 0 ]]; then
+ install_environment
+fi
+
+pushd "$FLINK_PYTHON_DIR" &> /dev/null
+# exec all selected checks
+if [ $skip_checks -eq 0 ]; then
+ check_stage
+fi