You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by iw...@apache.org on 2021/01/21 06:34:21 UTC
[bigtop] branch master updated: BIGTOP-3477: Fix puppet manifests
for hadoop-3 (#714)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 36e035d BIGTOP-3477: Fix puppet manifests for hadoop-3 (#714)
36e035d is described below
commit 36e035d2cf85ec239e01464511c8918598b2a1a6
Author: Yuqi Gu <yu...@arm.com>
AuthorDate: Thu Jan 21 14:34:14 2021 +0800
BIGTOP-3477: Fix puppet manifests for hadoop-3 (#714)
Change-Id: I35158c48b452c7a22a918bf2904065ffea6b1c76
Signed-off-by: Yuqi Gu <yu...@arm.com>
---
.../puppet/modules/hadoop/templates/hadoop-env.sh | 8 +++---
.../src/common/hadoop/conf.secure/hadoop-env.sh | 4 +--
.../hadoop/hadoop-mapreduce-historyserver.svc | 2 +-
.../src/common/hadoop/hadoop-yarn-nodemanager.svc | 2 +-
.../src/common/hadoop/hadoop-yarn-proxyserver.svc | 2 +-
.../common/hadoop/hadoop-yarn-resourcemanager.svc | 2 +-
.../common/hadoop/hadoop-yarn-timelineserver.svc | 2 +-
bigtop-packages/src/common/hadoop/hadoop.default | 1 -
.../src/common/hadoop/mapreduce.default | 6 ++---
.../hadoop/patch3-fix-broken-dir-detection.diff | 31 ++++++++++++++++++++++
bigtop-packages/src/common/hadoop/yarn.default | 8 +++---
.../src/deb/hadoop/hadoop-hdfs.postinst | 3 ++-
.../src/deb/hadoop/hadoop-mapreduce.postinst | 2 +-
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 4 +++
14 files changed, 56 insertions(+), 21 deletions(-)
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
index a1397c2..27667c5 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
@@ -44,9 +44,9 @@
<%= shell_config("HADOOP_OPTS") %>
# Command specific options appended to HADOOP_OPTS when specified
-<%= shell_config("HADOOP_NAMENODE_OPTS") %>
+<%= shell_config("HDFS_NAMENODE_OPTS") %>
<%= shell_config("HADOOP_SECONDARYNAMENODE_OPTS") %>
-<%= shell_config("HADOOP_DATANODE_OPTS") %>
+<%= shell_config("HDFS_DATANODE_OPTS") %>
<%= shell_config("HADOOP_BALANCER_OPTS") %>
<%= shell_config("HADOOP_JOBTRACKER_OPTS") %>
<%= shell_config("HADOOP_TASKTRACKER_OPTS") %>
@@ -91,8 +91,8 @@ export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JA
### WARNING: the following is NOT really optional. It is a shame that stock Hadoop
### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off
### and have a working Hadoop cluster.
-export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
+export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HDFS_NAMENODE_OPTS"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
-export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
+export HDFS_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HDFS_DATANODE_OPTS"
export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"
diff --git a/bigtop-packages/src/common/hadoop/conf.secure/hadoop-env.sh b/bigtop-packages/src/common/hadoop/conf.secure/hadoop-env.sh
index a9ebb5b..392c92c 100644
--- a/bigtop-packages/src/common/hadoop/conf.secure/hadoop-env.sh
+++ b/bigtop-packages/src/common/hadoop/conf.secure/hadoop-env.sh
@@ -28,10 +28,10 @@
export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
# Command specific options appended to HADOOP_OPTS when specified
-export HADOOP_NAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
+export HDFS_NAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HDFS_NAMENODE_OPTS}"
HADOOP_JOBTRACKER_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dmapred.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
HADOOP_TASKTRACKER_OPTS="-Dsecurity.audit.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
-HADOOP_DATANODE_OPTS="-Dsecurity.audit.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
+HDFS_DATANODE_OPTS="-Dsecurity.audit.logger=ERROR,DRFAS ${HDFS_DATANODE_OPT}"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
diff --git a/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc b/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc
index b5b6935..79670d7 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-mapreduce-historyserver.svc
@@ -20,5 +20,5 @@ SVC_USER="mapred"
WORKING_DIR="/var/lib/hadoop-mapreduce"
DAEMON_FLAGS="$HADOOP_NAME"
CONF_DIR="/etc/hadoop/conf"
-PIDFILE="/var/run/hadoop-mapreduce/mapred-$SVC_USER-$HADOOP_NAME.pid"
+PIDFILE="/var/run/hadoop-mapreduce/hadoop-$SVC_USER-$HADOOP_NAME.pid"
CHKCONFIG="2345 95 15"
diff --git a/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc b/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc
index 925371a..e74aa28 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-yarn-nodemanager.svc
@@ -20,5 +20,5 @@ SVC_USER="yarn"
WORKING_DIR="/var/lib/hadoop-yarn"
DAEMON_FLAGS="$HADOOP_NAME"
CONF_DIR="/etc/hadoop/conf"
-PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+PIDFILE="/var/run/hadoop-yarn/hadoop-$SVC_USER-$HADOOP_NAME.pid"
CHKCONFIG="2345 95 15"
diff --git a/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc b/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc
index c97a21b..4839751 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-yarn-proxyserver.svc
@@ -20,5 +20,5 @@ SVC_USER="yarn"
WORKING_DIR="/var/lib/hadoop-yarn"
DAEMON_FLAGS="$HADOOP_NAME"
CONF_DIR="/etc/hadoop/conf"
-PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+PIDFILE="/var/run/hadoop-yarn/hadoop-$SVC_USER-$HADOOP_NAME.pid"
CHKCONFIG="2345 95 15"
diff --git a/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc b/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc
index 4c402d9..f56709c 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-yarn-resourcemanager.svc
@@ -20,5 +20,5 @@ SVC_USER="yarn"
WORKING_DIR="/var/lib/hadoop-yarn"
DAEMON_FLAGS="$HADOOP_NAME"
CONF_DIR="/etc/hadoop/conf"
-PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+PIDFILE="/var/run/hadoop-yarn/hadoop-$SVC_USER-$HADOOP_NAME.pid"
CHKCONFIG="2345 95 15"
diff --git a/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc b/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc
index 11481cc..511ef35 100644
--- a/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc
+++ b/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc
@@ -20,5 +20,5 @@ SVC_USER="yarn"
WORKING_DIR="/var/lib/hadoop-yarn"
DAEMON_FLAGS="$HADOOP_NAME"
CONF_DIR="/etc/hadoop/conf"
-PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid"
+PIDFILE="/var/run/hadoop-yarn/hadoop-$SVC_USER-$HADOOP_NAME.pid"
CHKCONFIG="2345 95 15"
diff --git a/bigtop-packages/src/common/hadoop/hadoop.default b/bigtop-packages/src/common/hadoop/hadoop.default
index 3aeadc2..3a7ce85 100644
--- a/bigtop-packages/src/common/hadoop/hadoop.default
+++ b/bigtop-packages/src/common/hadoop/hadoop.default
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PREFIX=/usr/lib/hadoop
export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
export HADOOP_CONF_DIR=/etc/hadoop/conf
diff --git a/bigtop-packages/src/common/hadoop/mapreduce.default b/bigtop-packages/src/common/hadoop/mapreduce.default
index cfd35cc..9ab2b64 100644
--- a/bigtop-packages/src/common/hadoop/mapreduce.default
+++ b/bigtop-packages/src/common/hadoop/mapreduce.default
@@ -13,6 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export HADOOP_MAPRED_IDENT_STRING=mapred
-export HADOOP_MAPRED_PID_DIR=/var/run/hadoop-mapreduce
-export HADOOP_MAPRED_LOG_DIR=/var/log/hadoop-mapreduce
+export HADOOP_IDENT_STRING=mapred
+export HADOOP_PID_DIR=/var/run/hadoop-mapreduce
+export HADOOP_LOG_DIR=/var/log/hadoop-mapreduce
diff --git a/bigtop-packages/src/common/hadoop/patch3-fix-broken-dir-detection.diff b/bigtop-packages/src/common/hadoop/patch3-fix-broken-dir-detection.diff
new file mode 100644
index 0000000..d0b8ebd
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch3-fix-broken-dir-detection.diff
@@ -0,0 +1,31 @@
+diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+index b3b8afcc79d..d24887512d2 100755
+--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
++++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+@@ -870,26 +870,6 @@ function hadoop_basic_init
+ export HADOOP_MAPRED_HOME="${HADOOP_HOME}"
+ fi
+
+- if [[ ! -d "${HADOOP_COMMON_HOME}" ]]; then
+- hadoop_error "ERROR: Invalid HADOOP_COMMON_HOME"
+- exit 1
+- fi
+-
+- if [[ ! -d "${HADOOP_HDFS_HOME}" ]]; then
+- hadoop_error "ERROR: Invalid HADOOP_HDFS_HOME"
+- exit 1
+- fi
+-
+- if [[ ! -d "${HADOOP_YARN_HOME}" ]]; then
+- hadoop_error "ERROR: Invalid HADOOP_YARN_HOME"
+- exit 1
+- fi
+-
+- if [[ ! -d "${HADOOP_MAPRED_HOME}" ]]; then
+- hadoop_error "ERROR: Invalid HADOOP_MAPRED_HOME"
+- exit 1
+- fi
+-
+ # if for some reason the shell doesn't have $USER defined
+ # (e.g., ssh'd in to execute a command)
+ # let's get the effective username and use that
diff --git a/bigtop-packages/src/common/hadoop/yarn.default b/bigtop-packages/src/common/hadoop/yarn.default
index 5748f16..1fc0fda 100644
--- a/bigtop-packages/src/common/hadoop/yarn.default
+++ b/bigtop-packages/src/common/hadoop/yarn.default
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-export YARN_IDENT_STRING=yarn
-export YARN_PID_DIR=/var/run/hadoop-yarn
-export YARN_LOG_DIR=/var/log/hadoop-yarn
-export YARN_CONF_DIR=/etc/hadoop/conf
+export HADOOP_IDENT_STRING=yarn
+export HADOOP_PID_DIR=/var/run/hadoop-yarn
+export HADOOP_LOG_DIR=/var/log/hadoop-yarn
+export HADOOP_CONF_DIR=/etc/hadoop/conf
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst b/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst
index 4c3d904..2d97267 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst
+++ b/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst
@@ -21,12 +21,13 @@ set -e
case "$1" in
configure)
- mkdir -p /var/log/hadoop-hdfs /var/run/hadoop-hdfs || :
+ mkdir -p /var/log/hadoop-hdfs /var/run/hadoop-hdfs /var/lib/hadoop-hdfs/cache /usr/lib/hadoop/logs || :
chgrp -R hadoop /var/log/hadoop-hdfs /var/run/hadoop-hdfs
chmod g+w /var/run/hadoop-hdfs /var/log/hadoop-hdfs
chown hdfs:hadoop /var/lib/hadoop-hdfs/ /var/lib/hadoop-hdfs/cache
chmod 0755 /var/lib/hadoop-hdfs
chmod 1777 /var/lib/hadoop-hdfs/cache
+ chmod 1777 /usr/lib/hadoop/logs
;;
abort-upgrade|abort-remove|abort-deconfigure)
diff --git a/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst b/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst
index edd984b..8013cbb 100644
--- a/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst
+++ b/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst
@@ -21,7 +21,7 @@ set -e
case "$1" in
configure)
- mkdir -p /var/log/hadoop-mapreduce /var/run/hadoop-mapreduce || :
+ mkdir -p /var/log/hadoop-mapreduce /var/run/hadoop-mapreduce /var/lib/hadoop-mapreduce/cache || :
chgrp -R hadoop /var/log/hadoop-mapreduce /var/run/hadoop-mapreduce
chmod g+w /var/run/hadoop-mapreduce /var/log/hadoop-mapreduce
chown mapred:hadoop /var/lib/hadoop-mapreduce /var/lib/hadoop-mapreduce/cache
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index 10dff7c..9153ae9 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -567,6 +567,8 @@ done
%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_httpfs}
%__install -d -m 0755 $RPM_BUILD_ROOT/%{run_kms}
+%__install -d -m 1777 $RPM_BUILD_ROOT/%{lib_hadoop}/logs
+
%pre
getent group hadoop >/dev/null || groupadd -r hadoop
@@ -690,6 +692,7 @@ fi
%{lib_hadoop}/sbin
%{lib_hadoop}/bin
%{lib_hadoop}/etc
+%{lib_hadoop}/logs
%{lib_hadoop}/libexec/hadoop-config.sh
%{lib_hadoop}/libexec/hadoop-layout.sh
%{lib_hadoop}/libexec/hadoop-functions.sh
@@ -699,6 +702,7 @@ fi
%{man_hadoop}/man1/yarn.1.*
%{man_hadoop}/man1/hdfs.1.*
%{man_hadoop}/man1/mapred.1.*
+%attr(1777,hdfs,hadoop) %{lib_hadoop}/logs
# Shouldn't the following be moved to hadoop-hdfs?
%exclude %{lib_hadoop}/bin/fuse_dfs