You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ji...@apache.org on 2019/05/07 13:43:33 UTC
[flink] branch master updated: [FLINK-12330][python]Add integrated
Tox for ensuring compatibility of multi-version of python.
This is an automated email from the ASF dual-hosted git repository.
jincheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 5991d15 [FLINK-12330][python]Add integrated Tox for ensuring compatibility of multi-version of python.
5991d15 is described below
commit 5991d15149cd949a3545fb87881fecb43daba5d5
Author: sunjincheng121 <su...@gmail.com>
AuthorDate: Mon May 6 16:22:49 2019 +0800
[FLINK-12330][python]Add integrated Tox for ensuring compatibility of multi-version of python.
This closes #8355.
---
.gitignore | 5 +
.../main/flink-bin/bin/pyflink-gateway-server.sh | 2 +-
flink-python/README.md | 17 +-
flink-python/dev/lint-python.sh | 510 +++++++++++++++++++++
flink-python/pyflink/find_flink_home.py | 3 +-
flink-python/pyflink/java_gateway.py | 6 +-
flink-python/pyflink/table/__init__.py | 3 +-
flink-python/pyflink/table/table.py | 3 +-
flink-python/pyflink/table/table_environment.py | 5 +-
flink-python/pyflink/table/table_source.py | 6 +-
flink-python/setup.py | 5 +
.../{pyflink/find_flink_home.py => tox.ini} | 44 +-
pom.xml | 5 +-
13 files changed, 571 insertions(+), 43 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5eb17c9..d664bee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,11 @@ flink-runtime-web/web-dashboard/web/
flink-python/dist/
flink-python/build/
flink-python/pyflink.egg-info/
+flink-python/.tox/
+flink-python/dev/download
+flink-python/dev/.conda/
+flink-python/dev/log/
+flink-python/dev/.stage.txt
atlassian-ide-plugin.xml
out/
/docs/api
diff --git a/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh b/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh
index fe49e8f..b63a2ac 100644
--- a/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh
+++ b/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh
@@ -47,7 +47,7 @@ do
esac
done
-log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-client-$HOSTNAME.log
+log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-python-$HOSTNAME.log
log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j-cli.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback.xml)
TABLE_JAR_PATH=`echo "$FLINK_ROOT_DIR"/opt/flink-table*.jar`
diff --git a/flink-python/README.md b/flink-python/README.md
index 5605b52..a0117a0 100644
--- a/flink-python/README.md
+++ b/flink-python/README.md
@@ -10,15 +10,20 @@ In this initial version only Table API is supported, you can find the documentat
## Installation
-Currently, you can install PyFlink from Flink source code.
-First, you need build the whole Flink project using `mvn clean install -DskipTests` and set the value of the environment variable FLINK_HOME to the `build-target` directory under the root directory of Flink.
-Then enter the directory where this README.md file is located and execute `python setup.py install` to install PyFlink on your device.
+Currently, we can install PyFlink from Flink source code. Enter the directory where this README.md file is located and install PyFlink on your device by executing
-## Running Tests
+```
+python setup.py install
+```
-Currently you can perform an end-to-end test of PyFlink in the directory where this file is located with the following command:
+## Running test cases
- PYTHONPATH=$PYTHONPATH:./ python ./pyflink/table/tests/test_end_to_end.py
+Currently, we use conda and tox to verify the compatibility of the Flink Python API for multiple versions of Python and will integrate some useful plugins with tox, such as flake8.
+We can enter the directory where this README.md file is located and run test cases by executing
+
+```
+./dev/lint-python.sh
+```
## Python Requirements
diff --git a/flink-python/dev/lint-python.sh b/flink-python/dev/lint-python.sh
new file mode 100755
index 0000000..4ef02bb
--- /dev/null
+++ b/flink-python/dev/lint-python.sh
@@ -0,0 +1,510 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# lint-python.sh
+# This script will prepare a virtual environment for many kinds of checks, such as pytest, flake8 codestyle.
+#
+# You can refer to the README.MD in ${flink-python} to learn how easy to run the script.
+#
+
+# Download some software, such as miniconda.sh
+function download() {
+ local DOWNLOAD_STATUS=
+ if hash "wget" 2>/dev/null; then
+ wget "$1" -O "$2" -q --show-progress
+ DOWNLOAD_STATUS="$?"
+ else
+ curl "$1" -o "$2" --progress-bar
+ DOWNLOAD_STATUS="$?"
+ fi
+ if [ $DOWNLOAD_STATUS -ne 0 ]; then
+ echo "Dowload failed.You can try again"
+ exit $DOWNLOAD_STATUS
+ fi
+}
+
+# for print infos both in log and console
+function print_function() {
+ local STAGE_LENGTH=48
+ local left_edge_len=
+ local right_edge_len=
+ local str
+ case "$1" in
+ "STAGE")
+ left_edge_len=$(((STAGE_LENGTH-${#2})/2))
+ right_edge_len=$((STAGE_LENGTH-${#2}-left_edge_len))
+ str="$(seq -s "=" $left_edge_len | tr -d "[:digit:]")""$2""$(seq -s "=" $right_edge_len | tr -d "[:digit:]")"
+ ;;
+ "STEP")
+ str="$2"
+ ;;
+ *)
+ str="seq -s "=" $STAGE_LENGTH | tr -d "[:digit:]""
+ ;;
+ esac
+ echo $str | tee -a $LOG_FILE
+}
+
+# Checkpoint the stage:step for convenient to re-exec the script with
+# skipping those success steps.
+# The format is "${Stage}:${Step}". e.g. Install:4
+function checkpoint_stage() {
+ if [ ! -d `dirname $STAGE_FILE` ]; then
+ mkdir -p `dirname $STAGE_FILE`
+ fi
+ echo "$1:$2">"$STAGE_FILE"
+}
+
+# Restore the stage:step
+function restore_stage() {
+ if [ -f "$STAGE_FILE" ]; then
+ local lines=$(awk '{print NR}' $STAGE_FILE)
+ if [ $lines -eq 1 ]; then
+ local first_field=$(cat $STAGE_FILE | cut -d ":" -f 1)
+ local second_field=$(cat $STAGE_FILE | cut -d ":" -f 2)
+ check_valid_stage $first_field $second_field
+ if [ $? -eq 0 ]; then
+ STAGE=$first_field
+ STEP=$second_field
+ return
+ fi
+ fi
+ fi
+ STAGE="install"
+ STEP=0
+}
+
+# Decide whether the stage:step is valid.
+function check_valid_stage() {
+ case $1 in
+ "install")
+ if [ $2 -le $STAGE_INSTALL_STEPS ] && [ $2 -ge 0 ]; then
+ return 0
+ fi
+ ;;
+ "tox")
+ if [ $2 -eq 0 ]; then
+ return 0
+ fi
+ ;;
+ "flake8")
+ if [ $2 -eq 0 ]; then
+ return 0
+ fi
+ ;;
+ *)
+ ;;
+ esac
+ return 1
+}
+
+# For convenient to index something binded to OS.
+# Now, the script only make a distinction between 'Mac' and 'Non-Mac'.
+function get_os_index() {
+ if [ $1 == "Darwin" ]; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+# Considering the file size of miniconda.sh,
+# "wget" is better than curl in the weak network environment.
+function install_wget() {
+ if [ $1 == "Darwin" ]; then
+ hash "brew" 2>/dev/null
+ if [ $? -ne 0 ]; then
+ $((/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)") 2>&1 >/dev/null)
+ if [ $? -ne 0 ]; then
+ echo "Failed to install brew"
+ exit 1
+ fi
+ fi
+
+ hash "wget" 2>/dev/null
+ if [ $? -ne 0 ]; then
+ brew install wget 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "Failed to install wget"
+ exit 1
+ fi
+ fi
+ fi
+}
+
+# The script choose miniconda as our package management tool.
+# The script use miniconda to create all kinds of python versions and
+# some pakcages such as tox and flake8.
+
+function install_miniconda() {
+ OS_TO_CONDA_URL=("https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" \
+ "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh")
+ print_function "STEP" "download miniconda..."
+ if [ ! -f "$CONDA_INSTALL" ]; then
+ download ${OS_TO_CONDA_URL[$1]} $CONDA_INSTALL_SH
+ chmod +x $CONDA_INSTALL_SH
+ if [ $? -ne 0 ]; then
+ echo "Please manually chmod +x $CONDA_INSTALL_SH"
+ exit 1
+ fi
+ if [ -d "$CURRENT_DIR/.conda" ]; then
+ rm -rf "$CURRENT_DIR/.conda"
+ if [ $? -ne 0 ]; then
+ echo "Please manually rm -rf $CURRENT_DIR/.conda directory.\
+ Then retry to exec the script."
+ exit 1
+ fi
+ fi
+ fi
+ print_function "STEP" "download miniconda... [SUCCESS]"
+
+ print_function "STEP" "installing conda..."
+ if [ ! -d "$CURRENT_DIR/.conda" ]; then
+ $CONDA_INSTALL_SH -b -p $CURRENT_DIR/.conda 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "install miniconda failed"
+ exit $CONDA_INSTALL_STATUS
+ fi
+ fi
+ print_function "STEP" "install conda ... [SUCCESS]"
+}
+
+# Install some kinds of py env.
+function install_py_env() {
+ py_env=("2.7" "3.3" "3.4" "3.5" "3.6" "3.7")
+ for ((i=0;i<${#py_env[@]};i++)) do
+ if [ -d "$CURRENT_DIR/.conda/envs/${py_env[i]}" ]; then
+ rm -rf "$CURRENT_DIR/.conda/envs/${py_env[i]}"
+ if [ $? -ne 0 ]; then
+ echo "rm -rf $CURRENT_DIR/.conda/envs/${py_env[i]} failed, please \
+ rm -rf $CURRENT_DIR/.conda/envs/${py_env[i]} manually.\
+ Then retry to exec the script."
+ exit 1
+ fi
+ fi
+ print_function "STEP" "installing python${py_env[i]}..."
+ ${CONDA_PATH} create --name ${py_env[i]} -y -q python=${py_env[i]} 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "conda install ${py_env[i]} failed.\
+ You can retry to exec the script."
+ exit 1
+ fi
+ print_function "STEP" "install python${py_env[i]}... [SUCCESS]"
+ done
+}
+
+# Install tox.
+# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh
+function install_tox() {
+ if [ -f "$TOX_PATH" ]; then
+ ${CONDA_PATH} remove tox -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "conda remove tox failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ fi
+
+ ${CONDA_PATH} install -c conda-forge tox -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "conda install tox failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+}
+
+# Install flake8.
+# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh
+function install_flake8() {
+ if [ -f "$FLAKE8_PATH" ]; then
+ ${CONDA_PATH} remove flake8 -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "conda remove flake8 failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+ fi
+
+ ${CONDA_PATH} install -c anaconda flake8 -y -q 2>&1 >/dev/null
+ if [ $? -ne 0 ]; then
+ echo "conda install flake8 failed \
+ please try to exec the script again.\
+ if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f"
+ exit 1
+ fi
+}
+
+
+# In this function, the script will prepare all kinds of python environments and checks.
+function install_environment() {
+
+ print_function "STAGE" "Stage 1:installing environment"
+
+ local sys_os=`uname -s`
+ #get the index of the SUPPORT_OS array for convinient to intall tool.
+ get_os_index $sys_os
+ local os_index=$?
+
+ # step-1 install wget
+ # the file size of the miniconda.sh is too big to use "wget" tool to download instead
+ # of the "curl" in the weak network environment.
+ print_function "STEP" "installing wget..."
+ if [ $STEP -lt 1 ]; then
+ install_wget ${SUPPORT_OS[$os_index]}
+ STEP=1
+ checkpoint_stage $STAGE $STEP
+ fi
+ print_function "STEP" "install wget... [SUCCESS]"
+
+ # step-2 install miniconda
+ print_function "STEP" "installing miniconda..."
+ if [ $STEP -lt 2 ]; then
+ create_dir $CURRENT_DIR/download
+ install_miniconda $os_index
+ STEP=2
+ checkpoint_stage $STAGE $STEP
+ fi
+ print_function "STEP" "install miniconda... [SUCCESS]"
+
+ # step-3 install python environment whcih includes
+ # 2.7 3.3 3.4 3.5 3.6 3.7
+ print_function "STEP" "installing python environment..."
+ if [ $STEP -lt 3 ]; then
+ install_py_env
+ STEP=3
+ checkpoint_stage $STAGE $STEP
+ fi
+ print_function "STEP" "install python environment... [SUCCESS]"
+
+ # step-4 install tox
+ print_function "STEP" "installing tox..."
+ if [ $STEP -lt 4 ]; then
+ install_tox
+ STEP=4
+ checkpoint_stage $STAGE $STEP
+ fi
+ print_function "STEP" "install tox... [SUCCESS]"
+
+ # step-5 install flake8
+ print_function "STEP" "installing flake8..."
+ if [ $STEP -lt 5 ]; then
+ install_flake8
+ STEP=5
+ checkpoint_stage $STAGE $STEP
+ fi
+ print_function "STEP" "install flake8... [SUCCESS]"
+
+ print_function "STAGE" "Stage 1:finishing install environment"
+
+ STAGE="tox"
+ STEP=0
+}
+
+# create dir if needed
+function create_dir() {
+ if [ ! -d $1 ]; then
+ mkdir -p $1
+ if [ $? -ne 0 ]; then
+ echo "mkdir -p $1 failed. you can mkdir manually or exec the script with \
+ the command: sudo ./lint-python.sh"
+ exit 1
+ fi
+ fi
+}
+
+# Set created py-env in $PATH for tox's creating virtual env
+function activate () {
+ if [ ! -d ${CURRENT_DIR}/.conda/envs ]; then
+ echo "For some unkown reasons,missing the directory ${CURRENT_DIR}/.conda/envs,\
+ you should exec the script with the parameter: -f"
+ exit 1
+ fi
+
+ for py_dir in ${CURRENT_DIR}/.conda/envs/*
+ do
+ PATH=$py_dir/bin:$PATH
+ done
+ export PATH 2>/dev/null
+ if [ $? -ne 0 ]; then
+ echo "For some unkown reasons, the py package is not complete,\
+ you should exec the script with the parameter: -f"
+ exit 1
+ fi
+}
+
+# Reset the $PATH
+function deactivate() {
+ # reset old environment variables
+ # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
+ if ! [ -z "${_OLD_PATH+_}" ] ; then
+ PATH="$_OLD_PATH"
+ export PATH
+ unset _OLD_PATH
+ fi
+}
+
+# Tox pytest Check
+function tox_check() {
+ print_function "STAGE" "Stage 2:tox check"
+ # Set created py-env in $PATH for tox's creating virtual env
+ activate
+ $TOX_PATH -c $FLINK_PYTHON_DIR/tox.ini --recreate 2>&1 | tee -a $LOG_FILE
+
+ TOX_RESULT=$((grep -c "congratulations :)" "$LOG_FILE") 2>&1)
+ if [ $TOX_RESULT -eq '0' ]; then
+ print_function "STAGE" "tox checked ... [FAILED]"
+ else
+ print_function "STAGE" "tox checked ... [SUCCESS]"
+ fi
+ # Reset the $PATH
+ deactivate
+ # If check failed, stop the running script.
+ if [ $TOX_RESULT -eq '0' ]; then
+ exit 1
+ fi
+ STAGE="flake8"
+ STEP="0"
+}
+
+# Flake8 codestyle check
+function flake8_check {
+ local PYTHON_SOURCE="$(find . \( -path ./dev -o -path ./.tox \) -prune -o -type f -name "*.py" -print )"
+
+ print_function "STAGE" "Stage 3:flake8 code style check"
+ if [ ! -f "$FLAKE8_PATH" ]; then
+ echo "For some unkown reasons, the flake8 package is not complete,\
+ you should exec the script with the parameter: -f"
+ fi
+
+ if [[ ! "$PYTHON_SOURCE" ]]; then
+ echo "No python files found! Something is wrong exiting."
+ exit 1;
+ fi
+
+ # the return value of a pipeline is the status of the last command to exit
+ # with a non-zero status or zero if no command exited with a non-zero status
+ set -o pipefail
+ ($FLAKE8_PATH --config=tox.ini $PYTHON_SOURCE) 2>&1 | tee -a $LOG_FILE
+
+ PYCODESTYLE_STATUS=$?
+ if [ $PYCODESTYLE_STATUS -ne 0 ]; then
+ print_function "STAGE" "python code style checks ... [FAILED]"
+ # Stop the running script.
+ exit 1;
+ else
+ print_function "STAGE" "python code style checks ... [SUCCESS]"
+ fi
+}
+
+# CURRENT_DIR is "flink/flink-python/dev/"
+CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)"
+
+# FLINK_PYTHON_DIR is "flink/flink-python"
+FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR")
+pushd "$FLINK_PYTHON_DIR" &> /dev/null
+
+# conda path
+CONDA_PATH=$CURRENT_DIR/.conda/bin/conda
+
+# tox path
+TOX_PATH=$CURRENT_DIR/.conda/bin/tox
+
+# flake8 path
+FLAKE8_PATH=$CURRENT_DIR/.conda/bin/flake8
+
+_OLD_PATH="$PATH"
+
+SUPPORT_OS=("Darwin" "Linux")
+
+# the file stores the success progress.
+STAGE_FILE=$CURRENT_DIR/.stage.txt
+
+# the dir includes all kinds of py env installed.
+VIRTUAL_ENV=$CURRENT_DIR/.conda/envs
+
+LOG_DIR=$CURRENT_DIR/log
+
+if [ "$FLINK_IDENT_STRING" == "" ]; then
+ FLINK_IDENT_STRING="$USER"
+fi
+if [ "$HOSTNAME" == "" ]; then
+ HOSTNAME="$HOST"
+fi
+
+# the log file stores the checking result.
+LOG_FILE=$LOG_DIR/flink-$FLINK_IDENT_STRING-python-$HOSTNAME.log
+create_dir $LOG_DIR
+
+# clean LOG_FILE content
+echo >$LOG_FILE
+
+# miniconda script
+CONDA_INSTALL_SH=$CURRENT_DIR/download/miniconda.sh
+
+# stage "install" includes the num of steps.
+STAGE_INSTALL_STEPS=5
+
+# whether force to restart the script.
+FORCE_START=0
+# parse_opts
+USAGE="
+usage: $0 [options]$
+-h print this help message and exit
+-f force to exec from the start
+"
+while getopts "hf" arg; do
+ case "$arg" in
+ h)
+ printf "%s\\n" "$USAGE"
+ exit 2
+ ;;
+ f)
+ FORCE_START=1
+ ;;
+ ?)
+ printf "ERROR: did not recognize option '%s', please try -h\\n" "$1"
+ exit 1
+ ;;
+ esac
+done
+# If exec the script with the param: -f, all progress will be re-run
+if [ $FORCE_START -eq 1 ]; then
+ STAGE="install"
+ STEP=0
+ checkpoint_stage $STAGE $STEP
+else
+ restore_stage
+fi
+# Stage 1:install environment
+if [ $STAGE == "install" ]; then
+ install_environment
+fi
+# Stage 2:tox check python compatibility
+if [ $STAGE == "tox" ]; then
+ tox_check
+fi
+# Stage 3:code style test
+if [ $STAGE == "flake8" ]; then
+ flake8_check
+fi
+
+echo "All the checks are finished, the detailed information can be found in: $LOG_FILE"
diff --git a/flink-python/pyflink/find_flink_home.py b/flink-python/pyflink/find_flink_home.py
index e216ba0..b84a7c3 100644
--- a/flink-python/pyflink/find_flink_home.py
+++ b/flink-python/pyflink/find_flink_home.py
@@ -37,7 +37,8 @@ def _find_flink_home():
return build_target
except Exception:
pass
- print("Could not find valid FLINK_HOME in current environment.", file=sys.stderr)
+ print("Could not find valid FLINK_HOME(Flink distribution directory) "
+ "in current environment.", file=sys.stderr)
sys.exit(-1)
diff --git a/flink-python/pyflink/java_gateway.py b/flink-python/pyflink/java_gateway.py
index 2a8cf89..5c931de 100644
--- a/flink-python/pyflink/java_gateway.py
+++ b/flink-python/pyflink/java_gateway.py
@@ -94,7 +94,8 @@ def launch_gateway():
shutil.rmtree(conn_info_dir)
# Connect to the gateway
- gateway = JavaGateway(gateway_parameters=GatewayParameters(port=gateway_port, auto_convert=True))
+ gateway = JavaGateway(
+ gateway_parameters=GatewayParameters(port=gateway_port, auto_convert=True))
# Import the classes used by PyFlink
java_import(gateway.jvm, "org.apache.flink.table.api.*")
@@ -105,6 +106,7 @@ def launch_gateway():
java_import(gateway.jvm, "org.apache.flink.api.common.typeinfo.TypeInformation")
java_import(gateway.jvm, "org.apache.flink.api.common.typeinfo.Types")
java_import(gateway.jvm, "org.apache.flink.api.java.ExecutionEnvironment")
- java_import(gateway.jvm, "org.apache.flink.streaming.api.environment.StreamExecutionEnvironment")
+ java_import(gateway.jvm,
+ "org.apache.flink.streaming.api.environment.StreamExecutionEnvironment")
return gateway
diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py
index 501d5fe..05575fc 100644
--- a/flink-python/pyflink/table/__init__.py
+++ b/flink-python/pyflink/table/__init__.py
@@ -34,7 +34,8 @@ Important classes of Flink Table API:
"""
from pyflink.table.table import Table
from pyflink.table.table_config import TableConfig
-from pyflink.table.table_environment import TableEnvironment, StreamTableEnvironment, BatchTableEnvironment
+from pyflink.table.table_environment import (TableEnvironment, StreamTableEnvironment,
+ BatchTableEnvironment)
from pyflink.table.table_sink import TableSink, CsvTableSink
from pyflink.table.table_source import TableSource, CsvTableSource
from pyflink.table.types import DataTypes
diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py
index 86118a4..2321c59 100644
--- a/flink-python/pyflink/table/table.py
+++ b/flink-python/pyflink/table/table.py
@@ -108,7 +108,8 @@ class Table(object):
def insert_into(self, table_name):
"""
- Writes the :class:`Table` to a :class:`TableSink` that was registered under the specified name.
+ Writes the :class:`Table` to a :class:`TableSink` that was registered under
+ the specified name.
Example:
::
diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py
index 73322fa..d3494ee 100644
--- a/flink-python/pyflink/table/table_environment.py
+++ b/flink-python/pyflink/table/table_environment.py
@@ -81,8 +81,9 @@ class TableEnvironment(object):
"""
gateway = get_gateway()
j_field_names = utils.to_jarray(gateway.jvm.String, field_names)
- j_field_types = utils.to_jarray(gateway.jvm.TypeInformation,
- [type_utils.to_java_type(field_type) for field_type in field_types])
+ j_field_types = utils.to_jarray(
+ gateway.jvm.TypeInformation,
+ [type_utils.to_java_type(field_type) for field_type in field_types])
self._j_tenv.registerTableSink(name, j_field_names, j_field_types, table_sink._j_table_sink)
def scan(self, *table_path):
diff --git a/flink-python/pyflink/table/table_source.py b/flink-python/pyflink/table/table_source.py
index 4395984..225abf2 100644
--- a/flink-python/pyflink/table/table_source.py
+++ b/flink-python/pyflink/table/table_source.py
@@ -48,5 +48,7 @@ class CsvTableSource(TableSource):
gateway = get_gateway()
j_field_names = utils.to_jarray(gateway.jvm.String, field_names)
j_field_types = utils.to_jarray(gateway.jvm.TypeInformation,
- [type_utils.to_java_type(field_type) for field_type in field_types])
- super(CsvTableSource, self).__init__(gateway.jvm.CsvTableSource(source_path, j_field_names, j_field_types))
+ [type_utils.to_java_type(field_type)
+ for field_type in field_types])
+ super(CsvTableSource, self).__init__(
+ gateway.jvm.CsvTableSource(source_path, j_field_names, j_field_types))
diff --git a/flink-python/setup.py b/flink-python/setup.py
index acf7592..b3713e5 100644
--- a/flink-python/setup.py
+++ b/flink-python/setup.py
@@ -53,6 +53,7 @@ setup(
author='Flink Developers',
author_email='dev@flink.apache.org',
install_requires=['py4j==0.10.8.1'],
+ tests_require=['pytest==4.4.1'],
description='Apache Flink Python API',
long_description=long_description,
long_description_content_type='text/markdown',
@@ -60,5 +61,9 @@ setup(
'Development Status :: 1 - Planning',
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7']
)
diff --git a/flink-python/pyflink/find_flink_home.py b/flink-python/tox.ini
similarity index 52%
copy from flink-python/pyflink/find_flink_home.py
copy to flink-python/tox.ini
index e216ba0..23d250a 100644
--- a/flink-python/pyflink/find_flink_home.py
+++ b/flink-python/tox.ini
@@ -15,31 +15,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
-from __future__ import print_function
-import os
-import sys
+[tox]
+# tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions.
+# new environments will be excluded by default unless explicitly added to envlist.
+envlist = py27, py33, py34, py35, py36, py37
-def _find_flink_home():
- """
- Find the FLINK_HOME.
- """
- # If the environment has set FLINK_HOME, trust it.
- if 'FLINK_HOME' in os.environ:
- return os.environ['FLINK_HOME']
- else:
- try:
- flink_root_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../../")
- build_target = flink_root_dir + "/build-target"
- pyflink_file = build_target + "/bin/pyflink-gateway-server.sh"
- if os.path.isfile(pyflink_file):
- os.environ['FLINK_HOME'] = build_target
- return build_target
- except Exception:
- pass
- print("Could not find valid FLINK_HOME in current environment.", file=sys.stderr)
- sys.exit(-1)
+[testenv]
+deps =
+ pytest
+commands =
+ python --version
+ python setup.py install
+ pytest
-
-if __name__ == "__main__":
- print(_find_flink_home())
+[flake8]
+# We follow PEP 8 (https://www.python.org/dev/peps/pep-0008/) with one exception: lines can be
+# up to 100 characters in length, not 79.
+ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
+max-line-length=100
+exclude=.tox/*,dev/*,lib/*,target/*,build/*
diff --git a/pom.xml b/pom.xml
index 235e154..539e9d0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1415,8 +1415,9 @@ under the License.
<exclude>flink-jepsen/docker/id_rsa*</exclude>
<exclude>flink-jepsen/docker/nodes</exclude>
- <!-- Py4j -->
- <exclude>flink-python/lib/py4j-LICENSE.txt</exclude>
+ <!-- flink-python -->
+ <exclude>flink-python/lib/**</exclude>
+ <exclude>flink-python/dev/download/**</exclude>
</excludes>
</configuration>
</plugin>