You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by iw...@apache.org on 2021/03/02 06:01:08 UTC
[bigtop] branch master updated: BIGTOP-3456: Add hadoop-yarn-router
and hadoop-hdfs-dfsrouter to hadoop services (#703)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 8c51d52 BIGTOP-3456: Add hadoop-yarn-router and hadoop-hdfs-dfsrouter to hadoop services (#703)
8c51d52 is described below
commit 8c51d52b461e8d2737c25c18bda30da15e93c6a3
Author: zhuqi <82...@qq.com>
AuthorDate: Tue Mar 2 14:01:01 2021 +0800
BIGTOP-3456: Add hadoop-yarn-router and hadoop-hdfs-dfsrouter to hadoop services (#703)
---
.../src/common/hadoop/do-component-build | 2 +-
.../src/common/hadoop/hadoop-hdfs-datanode.svc | 2 +-
...hdfs-datanode.svc => hadoop-hdfs-dfsrouter.svc} | 49 +---------------------
.../src/common/hadoop/hadoop-yarn-router.svc | 25 +++++++++++
.../src/common/hadoop/install_hadoop.sh | 1 +
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 27 +++++++++++-
6 files changed, 54 insertions(+), 52 deletions(-)
diff --git a/bigtop-packages/src/common/hadoop/do-component-build b/bigtop-packages/src/common/hadoop/do-component-build
index 10eec01..221ea7f 100644
--- a/bigtop-packages/src/common/hadoop/do-component-build
+++ b/bigtop-packages/src/common/hadoop/do-component-build
@@ -125,7 +125,7 @@ MAVEN_OPTS+="-DskipTests -DskipTest -DskipITs "
. $(dirname ${0})/maven_deploy.sh
# Build artifacts
-mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar ${MAVEN_OPTS} install package ${EXTRA_GOALS} "$@"
+mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install package ${EXTRA_GOALS} "$@"
mvn site site:stage ${MAVEN_OPTS} $@
(cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
diff --git a/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc b/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc
index 7dbc618..caa8bd4 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc
@@ -29,7 +29,7 @@ start() {
[ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
[ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
- if [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ if [ -n "$HDFS_DATANODE_SECURE_USER" ]; then
TARGET_USER=root
else
TARGET_USER=${HADOOP_DATANODE_USER:-hdfs}
diff --git a/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc b/bigtop-packages/src/common/hadoop/hadoop-hdfs-dfsrouter.svc
similarity index 51%
copy from bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc
copy to bigtop-packages/src/common/hadoop/hadoop-hdfs-dfsrouter.svc
index 7dbc618..c4eb588 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-hdfs-datanode.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-hdfs-dfsrouter.svc
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-HADOOP_NAME="datanode"
+HADOOP_NAME="dfsrouter"
DAEMON="hadoop-hdfs-$HADOOP_NAME"
DESC="Hadoop $HADOOP_NAME"
EXEC_PATH="/usr/lib/hadoop/sbin/hadoop-daemon.sh"
@@ -22,50 +22,3 @@ DAEMON_FLAGS="$HADOOP_NAME"
CONF_DIR="/etc/hadoop/conf"
PIDFILE="/var/run/hadoop-hdfs/hadoop-$SVC_USER-$HADOOP_NAME.pid"
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
-
- if [ -n "$HADOOP_SECURE_DN_USER" ]; then
- TARGET_USER=root
- else
- TARGET_USER=${HADOOP_DATANODE_USER:-hdfs}
- fi
-
- export HADOOP_IDENT_STRING=$TARGET_USER
- runuser -s /bin/bash $TARGET_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS"
-
- # Some processes are slow to start
- sleep $SLEEP_TIME
- checkstatusofproc
- RETVAL=$?
-
- if [ $RETVAL -eq $STATUS_RUNNING ]; then
- touch $LOCKFILE
- log_success_msg "Started ${DESC} (${DAEMON}): "
- else
- log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL"
- fi
-
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_extra_commands() {
-
-cat <<'__EOT__'
- rollback)
- DAEMON_FLAGS="$DAEMON_FLAGS -${1}"
- start
- ;;
- *)
- echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart|rollback}"
- exit 1
-__EOT__
-
-}
diff --git a/bigtop-packages/src/common/hadoop/hadoop-yarn-router.svc b/bigtop-packages/src/common/hadoop/hadoop-yarn-router.svc
new file mode 100644
index 0000000..50a9abf
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/hadoop-yarn-router.svc
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+HADOOP_NAME="router"
+DAEMON="hadoop-yarn-$HADOOP_NAME"
+DESC="Hadoop $HADOOP_NAME"
+EXEC_PATH="/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh"
+SVC_USER="yarn"
+WORKING_DIR="/var/lib/hadoop-yarn"
+DAEMON_FLAGS="$HADOOP_NAME"
+CONF_DIR="/etc/hadoop/conf"
+PIDFILE="/var/run/hadoop-yarn/hadoop-$SVC_USER-$HADOOP_NAME.pid"
+CHKCONFIG="2345 95 15"
+
diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh
index 7ab5505..a85f38d 100755
--- a/bigtop-packages/src/common/hadoop/install_hadoop.sh
+++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh
@@ -211,6 +211,7 @@ chmod 644 ${HADOOP_DIR}/lib/*.jar ${MAPREDUCE_DIR}/lib/*.jar ${HDFS_DIR}/lib/*.j
# Install webapps
cp -ra ${BUILD_DIR}/share/hadoop/hdfs/webapps ${HDFS_DIR}/
+cp -ra ${BUILD_DIR}/share/hadoop/yarn/webapps ${YARN_DIR}/
# bin
install -d -m 0755 ${HADOOP_DIR}/bin
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index 107cf11..6f59500 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -58,8 +58,8 @@
%define httpfs_services httpfs
%define kms_services kms
%define mapreduce_services mapreduce-historyserver
-%define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc hdfs-journalnode
-%define yarn_services yarn-resourcemanager yarn-nodemanager yarn-proxyserver yarn-timelineserver
+%define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc hdfs-journalnode hdfs-dfsrouter
+%define yarn_services yarn-resourcemanager yarn-nodemanager yarn-proxyserver yarn-timelineserver yarn-router
%define hadoop_services %{hdfs_services} %{mapreduce_services} %{yarn_services} %{httpfs_services} %{kms_services}
# Hadoop outputs built binaries into %{hadoop_build}
%define hadoop_build_path build
@@ -333,6 +333,16 @@ The Data Nodes in the Hadoop Cluster are responsible for serving up
blocks of data over the network to Hadoop Distributed Filesystem
(HDFS) clients.
+%package hdfs-dfsrouter
+Summary: HDFS Router Server
+Group: System/Daemons
+Requires: %{name}-hdfs = %{version}-%{release}
+Requires(pre): %{name} = %{version}-%{release}
+Requires(pre): %{name}-hdfs = %{version}-%{release}
+
+%description hdfs-dfsrouter
+HDFS Router Server which supports Router Based Federation.
+
%package httpfs
Summary: HTTPFS for Hadoop
Group: System/Daemons
@@ -403,6 +413,17 @@ Requires: %{name}-hdfs = %{version}-%{release}
Requires(pre): %{name} = %{version}-%{release}
Requires(pre): %{name}-mapreduce = %{version}-%{release}
+%package yarn-router
+Summary: YARN Router Server
+Group: System/Daemons
+Requires: %{name}-yarn = %{version}-%{release}
+Requires(pre): %{name} = %{version}-%{release}
+Requires(pre): %{name}-yarn = %{version}-%{release}
+
+%description yarn-router
+YARN Router Server which supports YARN Federation.
+
+
%description mapreduce-historyserver
The History server keeps records of the different activities being performed on a Apache Hadoop cluster
@@ -764,10 +785,12 @@ fi
%service_macro hdfs-zkfc
%service_macro hdfs-journalnode
%service_macro hdfs-datanode
+%service_macro hdfs-dfsrouter
%service_macro yarn-resourcemanager
%service_macro yarn-nodemanager
%service_macro yarn-proxyserver
%service_macro yarn-timelineserver
+%service_macro yarn-router
%service_macro mapreduce-historyserver
# Pseudo-distributed Hadoop installation