You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by hx...@apache.org on 2022/06/27 02:36:30 UTC
[flink] branch release-1.14 updated: [FLINK-28226][python] Limit grpcio<1.47
This is an automated email from the ASF dual-hosted git repository.
hxb pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.14 by this push:
new a473be67375 [FLINK-28226][python] Limit grpcio<1.47
a473be67375 is described below
commit a473be67375578ebbd330c98dbdd330dbd34dd66
Author: huangxingbo <hx...@apache.org>
AuthorDate: Mon Jun 27 10:36:10 2022 +0800
[FLINK-28226][python] Limit grpcio<1.47
---
.../test-scripts/docker-hadoop-secure-cluster/Dockerfile | 2 +-
.../test-scripts/test_kubernetes_pyflink_application.sh | 4 ++++
flink-end-to-end-tests/test-scripts/test_pyflink.sh | 11 +++++++----
flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh | 2 ++
flink-python/dev/dev-requirements.txt | 3 ++-
5 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
index cdb6c480994..21119a045f3 100644
--- a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
@@ -36,7 +36,7 @@ RUN set -x \
# install dev tools
RUN set -x \
&& apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
- curl tar sudo openssh-server openssh-client rsync unzip krb5-user net-tools
+ curl tar sudo openssh-server openssh-client rsync unzip krb5-user net-tools build-essential
# Kerberos client
RUN set -x \
diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh
index 6659362d722..a7b6c93a548 100755
--- a/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh
+++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh
@@ -78,13 +78,17 @@ mkdir -p "$PYFLINK_DOCKER_DIR"
cp "${FLINK_PYTHON_DIR}/dist/${PYFLINK_PACKAGE_FILE}" $PYFLINK_DOCKER_DIR/
cp "${FLINK_PYTHON_DIR}/apache-flink-libraries/dist/${PYFLINK_LIBRARIES_PACKAGE_FILE}" $PYFLINK_DOCKER_DIR/
cp "${FLINK_PYTHON_DIR}/dev/lint-python.sh" $PYFLINK_DOCKER_DIR/
+cp "${FLINK_PYTHON_DIR}/dev/dev-requirements.txt" $PYFLINK_DOCKER_DIR/
if [[ -d "dist" ]]; then rm -Rf dist; fi
cd ${PYFLINK_DOCKER_DIR}
echo "FROM ${PURE_FLINK_IMAGE_NAME}" >> Dockerfile
+echo "RUN apt-get update && apt-get install build-essential -y" >> Dockerfile
echo "COPY lint-python.sh /tmp/lint-python.sh" >> Dockerfile
+echo "COPY dev-requirements.txt /tmp/dev-requirements.txt" >> Dockerfile
echo "RUN bash /tmp/lint-python.sh -s basic" >> Dockerfile
echo "COPY ${PYFLINK_PACKAGE_FILE} ${PYFLINK_PACKAGE_FILE}" >> Dockerfile
echo "COPY ${PYFLINK_LIBRARIES_PACKAGE_FILE} ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile
+echo "RUN /tmp/.conda/bin/python -m pip install -r /tmp/dev-requirements.txt" >> Dockerfile
echo "RUN /tmp/.conda/bin/python -m pip install ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile
echo "RUN /tmp/.conda/bin/python -m pip install ${PYFLINK_PACKAGE_FILE}" >> Dockerfile
echo "RUN rm ${PYFLINK_LIBRARIES_PACKAGE_FILE}" >> Dockerfile
diff --git a/flink-end-to-end-tests/test-scripts/test_pyflink.sh b/flink-end-to-end-tests/test-scripts/test_pyflink.sh
index 630e5f97d97..54dd42a58fa 100755
--- a/flink-end-to-end-tests/test-scripts/test_pyflink.sh
+++ b/flink-end-to-end-tests/test-scripts/test_pyflink.sh
@@ -19,12 +19,13 @@
set -Eeuo pipefail
-KAFKA_VERSION="2.2.0"
-CONFLUENT_VERSION="5.0.0"
-CONFLUENT_MAJOR_VERSION="5.0"
+KAFKA_VERSION="2.8.1"
+CONFLUENT_VERSION="6.2.2"
+CONFLUENT_MAJOR_VERSION="6.2"
+# Check the Confluent Platform <> Apache Kafka compatibility matrix when updating KAFKA_VERSION
KAFKA_SQL_VERSION="universal"
SQL_JARS_DIR=${END_TO_END_DIR}/flink-sql-client-test/target/sql-jars
-KAFKA_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "kafka_" )
+KAFKA_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "kafka" )
function create_data_stream_kafka_source {
topicName="test-python-data-stream-source"
@@ -85,6 +86,8 @@ cd "${FLINK_PYTHON_DIR}"
rm -rf dist
+pip install -r dev/dev-requirements.txt
+
python setup.py sdist
pushd apache-flink-libraries
diff --git a/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh b/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh
index 1d7ec822cf9..bd8d3300942 100755
--- a/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh
+++ b/flink-end-to-end-tests/test-scripts/test_pyflink_yarn.sh
@@ -34,6 +34,7 @@ start_hadoop_cluster_and_prepare_flink
# copy test files
docker cp "${FLINK_PYTHON_DIR}/dev/lint-python.sh" master:/tmp/
+docker cp "${FLINK_PYTHON_DIR}/dev/dev-requirements.txt" master:/tmp/
docker cp "${FLINK_PYTHON_TEST_DIR}/target/PythonUdfSqlJobExample.jar" master:/tmp/
docker cp "${FLINK_PYTHON_TEST_DIR}/python/add_one.py" master:/tmp/
docker cp "${REQUIREMENTS_PATH}" master:/tmp/
@@ -47,6 +48,7 @@ docker cp "${FLINK_PYTHON_DIR}/apache-flink-libraries/dist/${PYFLINK_LIBRARIES_P
docker exec master bash -c "
/tmp/lint-python.sh -s miniconda
source /tmp/.conda/bin/activate
+pip install -r /tmp/dev-requirements.txt
pip install /tmp/${PYFLINK_LIBRARIES_PACKAGE_FILE}
pip install /tmp/${PYFLINK_PACKAGE_FILE}
conda install -y -q zip=3.0
diff --git a/flink-python/dev/dev-requirements.txt b/flink-python/dev/dev-requirements.txt
index 25bfb6cc622..f937e360ea5 100755
--- a/flink-python/dev/dev-requirements.txt
+++ b/flink-python/dev/dev-requirements.txt
@@ -25,5 +25,6 @@ pyarrow>=0.15.1,<3.0.0
pytz>=2018.3
numpy>=1.14.3,<1.20
fastavro>=0.21.4,<0.24
-grpcio>=1.29.0,<2
+grpcio>=1.29.0,<1.47
grpcio-tools>=1.3.5,<=1.14.2
+protobuf<3.18