You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by se...@apache.org on 2020/09/04 02:53:48 UTC
[bigtop] branch master updated: BIGTOP-3382. Upgrade Flink to
1.11.1 (#665)
This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 6f920a1 BIGTOP-3382. Upgrade Flink to 1.11.1 (#665)
6f920a1 is described below
commit 6f920a14401a65b072a31d5f935b3156840c6c28
Author: okumurays <58...@users.noreply.github.com>
AuthorDate: Fri Sep 4 11:53:40 2020 +0900
BIGTOP-3382. Upgrade Flink to 1.11.1 (#665)
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 9 ++++++---
.../puppet/modules/flink/manifests/init.pp | 2 +-
.../puppet/modules/flink/templates/flink-conf.yaml | 23 ++++++----------------
.../src/common/flink/flink-jobmanager.svc | 12 ++++++-----
.../src/common/flink/flink-taskmanager.svc | 12 ++++++-----
bigtop-packages/src/common/flink/install_flink.sh | 2 +-
.../flink/patch0-fix-ApplicationReport-api.diff | 12 -----------
bigtop-tests/smoke-tests/flink/TestFlink.groovy | 2 +-
bigtop.bom | 2 +-
9 files changed, 30 insertions(+), 46 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index d73b929..1f85244 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -203,9 +203,12 @@ zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')
# Flink
flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}"
flink::common::jobmanager_port: "6123"
-
-flink::common::ui_port: "8081"
-flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}"
+flink::common::jobmanager_memory: "1600m"
+flink::common::taskmanager_memory: "1728m"
+flink::common::taskmanager_number_of_taskslots: "1"
+flink::common::parallelism_default: "1"
+flink::common::jobmanager_failover_strategy: "region"
+flink::common::rest_port: "8081"
gpdb::common::nodes: ["master_node_fqdn", "segment_node_fqdn"]
gpdb::common::gp_home: "/usr/lib/gpdb"
diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
index e31d182..c1e6205 100644
--- a/bigtop-deploy/puppet/modules/flink/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
@@ -21,7 +21,7 @@ class flink {
}
}
- class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) {
+ class common($jobmanager_host, $jobmanager_port, $jobmanager_memory, $taskmanager_memory, $taskmanager_number_of_taskslots, $parallelism_default, $jobmanager_failover_strategy, $rest_port) {
# make sure flink is installed
package { "flink":
ensure => latest
diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
index cb9c1f4..ead8d08 100644
--- a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
+++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
@@ -20,20 +20,9 @@
# Configuration values managed by puppet:
jobmanager.rpc.address: <%= @jobmanager_host %>
jobmanager.rpc.port: <%= @jobmanager_port %>
-jobmanager.web.port: <%= @ui_port %>
-
-<% if defined?(storage_dirs) %>
-taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %>
-<% end %>
-
-# Starting from Flink v1.5, there is a rewrite of Flink’s deployment and process
-# model (internally known as FLIP-6). A new option 'mode' was introduced to
-# indicate which executions to use. However, the way Bigtop puppet deploys
-# 'flink-jobmanager' and 'flink-taskmanager', ie. calling 'flink-daemon.sh
-# jobmanager' and 'flink-daemon.sh taskmanager', is legacy. Without setting
-# mode to 'legacy', flink's built-in exmaples fail with error
-# "JobSubmissionException: Failed to submit JobGraph"
-mode: legacy
-
-# For performance reasons its highly recommended to allocate as much memory to the
-# Flink TaskManager as possible by setting 'taskmanager.heap.mb'.
+jobmanager.memory.process.size: <%= @jobmanager_memory %>
+taskmanager.memory.process.size: <%= @taskmanager_memory %>
+taskmanager.numberOfTaskSlots: <%= @taskmanager_number_of_taskslots %>
+parallelism.default: <%= @parallelism_default %>
+jobmanager.execution.failover-strategy: <%= @jobmanager_failover_strategy %>
+rest.port: <%=@rest_port %>
diff --git a/bigtop-packages/src/common/flink/flink-jobmanager.svc b/bigtop-packages/src/common/flink/flink-jobmanager.svc
index ac7106e..7e74e80 100644
--- a/bigtop-packages/src/common/flink/flink-jobmanager.svc
+++ b/bigtop-packages/src/common/flink/flink-jobmanager.svc
@@ -16,12 +16,12 @@
TYPE="jobmanager"
DAEMON="flink-${TYPE}"
DESC="Flink ${TYPE}"
-EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
+EXEC_PATH="/usr/lib/flink/bin/jobmanager.sh"
WORKING_DIR="/var/lib/flink"
DAEMON_FLAGS=jobmanager
CONF_DIR="/etc/flink/conf"
SVC_USER="flink"
-PIDFILE="/var/run/flink/flink-$SVC_USER-jobmanager.pid"
+PIDFILE="/var/run/flink/flink-$SVC_USER-standalonesession.pid"
EXEC_MODE="cluster"
generate_start() {
@@ -31,6 +31,10 @@ start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "
+ if command -v hadoop > /dev/null; then
+ export HADOOP_CLASSPATH=$(hadoop classpath)
+ fi
+ export FLINK_CONF_DIR="${CONF_DIR}"
export FLINK_LOG_DIR="/var/log/flink"
export FLINK_PID_DIR="/var/run/flink"
mkdir -p $FLINK_PID_DIR
@@ -44,9 +48,7 @@ start() {
LOG_FILE=/var/log/flink/${DAEMON}.out
# Flink will set the pid file
- runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir $CONF_DIR --executionMode cluster \
- > $LOG_FILE 2>&1 & "
+ runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 ${EXEC_PATH} start > $LOG_FILE 2>&1 & "
sleep 3
diff --git a/bigtop-packages/src/common/flink/flink-taskmanager.svc b/bigtop-packages/src/common/flink/flink-taskmanager.svc
index 5bbd616..612148c 100644
--- a/bigtop-packages/src/common/flink/flink-taskmanager.svc
+++ b/bigtop-packages/src/common/flink/flink-taskmanager.svc
@@ -16,12 +16,12 @@
TYPE="taskmanager"
DAEMON="flink-${TYPE}"
DESC="Flink ${TYPE}"
-EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
+EXEC_PATH="/usr/lib/flink/bin/taskmanager.sh"
WORKING_DIR="/var/lib/flink"
DAEMON_FLAGS="taskmanager"
SVC_USER="flink"
CONF_DIR="/etc/flink/conf"
-PIDFILE="/var/run/flink/flink-$SVC_USER-taskmanager.pid"
+PIDFILE="/var/run/flink/flink-$SVC_USER-taskexecutor.pid"
generate_start() {
@@ -31,6 +31,10 @@ start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "
+ if command -v hadoop > /dev/null; then
+ export HADOOP_CLASSPATH=$(hadoop classpath)
+ fi
+ export FLINK_CONF_DIR="${CONF_DIR}"
export FLINK_LOG_DIR="/var/log/flink"
export FLINK_PID_DIR="/var/run/flink"
mkdir -p $FLINK_PID_DIR
@@ -44,9 +48,7 @@ start() {
LOG_FILE=/var/log/flink/${DAEMON}.out
- runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir ${CONF_DIR} \
- > $LOG_FILE 2>&1 & "
+ runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 ${EXEC_PATH} start > $LOG_FILE 2>&1 & "
sleep 3
diff --git a/bigtop-packages/src/common/flink/install_flink.sh b/bigtop-packages/src/common/flink/install_flink.sh
index 6baa669..ac8e5b6 100644
--- a/bigtop-packages/src/common/flink/install_flink.sh
+++ b/bigtop-packages/src/common/flink/install_flink.sh
@@ -120,7 +120,7 @@ ln -s /etc/flink/conf $PREFIX/$LIB_DIR/conf
cp -ra ${BUILD_DIR}/examples/* $PREFIX/${LIB_DIR}/examples/
-cp ${BUILD_DIR}/{LICENSE,NOTICE,README.txt} ${PREFIX}/${LIB_DIR}/
+cp ${BUILD_DIR}/{LICENSE,README.txt} ${PREFIX}/${LIB_DIR}/
# Copy in the /usr/bin/flink wrapper
install -d -m 0755 $PREFIX/$BIN_DIR
diff --git a/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff b/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
deleted file mode 100644
index c76d3c3..0000000
--- a/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
-index 8ee08db..fcd56d0 100644
---- a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
-+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
-@@ -98,6 +98,7 @@ public class AbstractYarnClusterTest extends TestLogger {
- yarnApplicationState,
- null,
- null,
-+ 0,
- 1L,
- 2L,
- finalApplicationStatus,
diff --git a/bigtop-tests/smoke-tests/flink/TestFlink.groovy b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
index b35d3cb..dba11aa 100644
--- a/bigtop-tests/smoke-tests/flink/TestFlink.groovy
+++ b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
@@ -59,7 +59,7 @@ class TestFlink {
// read JM address and port from conf
execCommand("awk '{if(/jobmanager.rpc.address:/) print \$2}' < "+ config_file);
final String jmHost = sh.out.join('\n');
- execCommand("awk '{if(/jobmanager.web.port:/) print \$2}' < "+config_file);
+ execCommand("awk '{if(/rest.port:/) print \$2}' < "+config_file);
final String webPort = sh.out.join('\n');
// check web API
execCommand("curl http://"+jmHost+":"+webPort+"/config");
diff --git a/bigtop.bom b/bigtop.bom
index ea22671..74defb8 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -287,7 +287,7 @@ bigtop {
'flink' {
name = 'flink'
relNotes = 'Apache Flink'
- version { base = '1.6.4'; pkg = base; release = 1 }
+ version { base = '1.11.1'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}-src.tgz" }
url { download_path = "/$name/$name-${version.base}"