You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by wu...@apache.org on 2022/11/08 18:12:31 UTC
[ambari] branch trunk updated: AMBARI-25767: Upgrade Flink for BIGTOP to be compatible with bigtop-select (#3460)
This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2c48c3b604 AMBARI-25767: Upgrade Flink for BIGTOP to be compatible with bigtop-select (#3460)
2c48c3b604 is described below
commit 2c48c3b60485dfae03a4d43426e99cf811aa9a9f
Author: yaolei <le...@163.com>
AuthorDate: Wed Nov 9 02:12:22 2022 +0800
AMBARI-25767: Upgrade Flink for BIGTOP to be compatible with bigtop-select (#3460)
---
.../stacks/BIGTOP/3.2.0/properties/stack_packages.json | 5 +++--
.../3.2.0/services/FLINK/configuration/flink-conf.xml | 7 +++++--
.../3.2.0/services/FLINK/package/scripts/flink_client.py | 4 ----
.../BIGTOP/3.2.0/services/FLINK/package/scripts/params.py | 13 ++++++++++---
.../3.2.0/services/FLINK/package/scripts/service_check.py | 2 +-
.../3.2.0/services/FLINK/package/scripts/setup_flink.py | 1 -
6 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
index 73137b7c75..5339b079b9 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
@@ -282,10 +282,11 @@
"flink-client"
]
},
- "FLINK_JOBHISTORYSERVER": {
+ "FLINK_HISTORYSERVER": {
"STACK-SELECT-PACKAGE": "flink-historyserver",
"INSTALL": [
- "flink-historyserver"
+ "flink-historyserver",
+ "flink-client"
],
"PATCH": [
"flink-historyserver"
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/configuration/flink-conf.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/configuration/flink-conf.xml
index 5a85b41064..b405890c78 100755
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/configuration/flink-conf.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/configuration/flink-conf.xml
@@ -116,7 +116,7 @@ jobmanager.bind-host: localhost
#
# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.
-jobmanager.memory.process.size: 1600m
+jobmanager.memory.process.size: 1024m
# The host interface the TaskManager will bind to. By default, this is localhost, and will prevent
# the TaskManager from communicating outside the machine/container it is running on.
@@ -144,7 +144,7 @@ taskmanager.host: localhost
#
# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.
-taskmanager.memory.process.size: 1728m
+taskmanager.memory.process.size: 1024m
# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.
# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.
@@ -181,6 +181,9 @@ env.hadoop.conf.dir: {{hadoop_conf_dir}}
#Defines the directory where the flink-<host>-<process>.pid files are saved.
env.pid.dir: {{flink_pid_dir}}
+#Defines the directory where the Flink logs are saved.
+env.log.dir: {{flink_log_dir}}
+
#==============================================================================
# High Availability
#==============================================================================
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/flink_client.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/flink_client.py
index 073b96b732..0603450107 100755
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/flink_client.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/flink_client.py
@@ -26,10 +26,6 @@ from setup_flink import *
class FlinkClient(Script):
- def pre_install(self, env):
- import params
- env.set_params(params)
-
def configure(self, env, config_dir=None, upgrade_type=None):
import params
env.set_params(params)
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/params.py
index a19ca33def..8473f10e9f 100755
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/params.py
@@ -32,6 +32,14 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.script.script import Script
+# a map of the Ambari role to the component name
+# for use with <stack-root>/current/<component>
+SERVER_ROLE_DIRECTORY_MAP = {
+ 'FLINK_HISTORYSERVER' : 'flink-historyserver',
+ 'FLINK_CLIENT' : 'flink-client'
+}
+component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "FLINK_CLIENT")
+
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
@@ -60,9 +68,8 @@ hdfs_site = config['configurations']['hdfs-site']
hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore"
flink_etc_dir = "/etc/flink"
-flink_config_dir = "/etc/flink/conf"
-flink_dir = "/usr/lib/flink"
-flink_bin_dir = "/usr/lib/flink/bin"
+flink_config_dir = format("{flink_etc_dir}/conf")
+flink_dir = format("{stack_root}/current/{component_directory}")
flink_log_dir = config['configurations']['flink-env']['flink_log_dir']
flink_pid_dir = config['configurations']['flink-env']['flink_pid_dir']
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/service_check.py
index 717ca36a9b..f1bd550272 100755
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/service_check.py
@@ -35,7 +35,7 @@ class FlinkServiceCheck(Script):
Execute(flink_kinit_cmd, user=params.smokeuser)
job_cmd_opts= "-m yarn-cluster -yD classloader.check-leaked-classloader=false "
- run_flink_wordcount_job = format("export HADOOP_CLASSPATH=`hadoop classpath`;{flink_bin_dir}/flink run {job_cmd_opts} {flink_bin_dir}/../examples/batch/WordCount.jar")
+ run_flink_wordcount_job = format("export HADOOP_CLASSPATH=`hadoop classpath`;{flink_dir}/bin/flink run {job_cmd_opts} {flink_dir}/examples/batch/WordCount.jar")
Execute(run_flink_wordcount_job,
logoutput=True,
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/setup_flink.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/setup_flink.py
index 75a8b6d16f..86c67d1dcf 100755
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/setup_flink.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/FLINK/package/scripts/setup_flink.py
@@ -42,7 +42,6 @@ def setup_flink(env, type, upgrade_type = None, action = None):
create_parents = True)
Directory(params.flink_log_dir,mode=0767)
- Link(params.flink_dir + '/log',to=params.flink_log_dir)
if type == 'historyserver' and action == 'config':
params.HdfsResource(params.flink_hdfs_user_dir,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ambari.apache.org
For additional commands, e-mail: commits-help@ambari.apache.org