You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2025/12/01 02:03:39 UTC

(ranger) branch RANGER-5413-Patch created (now 635f76eb5)

This is an automated email from the ASF dual-hosted git repository.

rmani pushed a change to branch RANGER-5413-Patch
in repository https://gitbox.apache.org/repos/asf/ranger.git


      at 635f76eb5 RANGER-5413:Beeline failing in Hive Docker setup due to hive failure to connect to YARN resource manager

This branch includes the following new commits:

     new 635f76eb5 RANGER-5413:Beeline failing in Hive Docker setup due to hive failure to connect to YARN resource manager

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(ranger) 01/01: RANGER-5413:Beeline failing in Hive Docker setup due to hive failure to connect to YARN resource manager

Posted by rm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rmani pushed a commit to branch RANGER-5413-Patch
in repository https://gitbox.apache.org/repos/asf/ranger.git

commit 635f76eb5706b3de9ff18b804d5569a5a3eedb90
Author: Ramesh Mani <rm...@apache.org>
AuthorDate: Sun Nov 30 18:01:52 2025 -0800

    RANGER-5413:Beeline failing in Hive Docker setup due to hive failure to connect to YARN resource manager
---
 dev-support/ranger-docker/Dockerfile.ranger-hive   |  2 +
 .../scripts/hive/hive-log4j2.properties            | 97 ++++++++++++++++++++++
 .../scripts/hive/hive-metastore-log4j2.properties  | 97 ++++++++++++++++++++++
 .../scripts/hive/ranger-hive-setup.sh              |  6 ++
 .../ranger-docker/scripts/hive/ranger-hive.sh      |  4 +-
 5 files changed, 204 insertions(+), 2 deletions(-)

diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive
index e0bcdea3a..b4798cec3 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hive
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hive
@@ -40,6 +40,8 @@ COPY ./scripts/hive/ranger-hive-setup.sh                  ${RANGER_SCRIPTS}/
 COPY ./scripts/hive/ranger-hive.sh                        ${RANGER_SCRIPTS}/
 COPY ./scripts/hive/ranger-hive-plugin-install.properties ${RANGER_SCRIPTS}/
 COPY ./scripts/hive/hive-site-${RANGER_DB_TYPE}.xml       ${RANGER_SCRIPTS}/hive-site.xml
+COPY ./scripts/hive/hive-log4j2.properties                ${RANGER_SCRIPTS}/
+COPY ./scripts/hive/hive-metastore-log4j2.properties      ${RANGER_SCRIPTS}/
 COPY ./scripts/hadoop/core-site.xml                       ${RANGER_SCRIPTS}/
 COPY ./scripts/wait_for_keytab.sh                         ${RANGER_SCRIPTS}/
 COPY ./scripts/kdc/krb5.conf                              /etc/krb5.conf
diff --git a/dev-support/ranger-docker/scripts/hive/hive-log4j2.properties b/dev-support/ranger-docker/scripts/hive/hive-log4j2.properties
new file mode 100644
index 000000000..09b0d1b9c
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/hive/hive-log4j2.properties
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = INFO
+name = HiveLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+# Change to DEBUG for detailed troubleshooting
+property.hive.log.level = DEBUG
+property.hive.root.logger = DRFA
+property.hive.log.dir = /opt/hive/logs
+property.hive.log.file = hiveserver2.log
+property.hive.perflogger.log.level = INFO
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingRandomAccessFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${hive.log.dir}/${hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp, RangerAuth
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.AmazonAws.name=com.amazonaws
+logger.AmazonAws.level = INFO
+
+logger.ApacheHttp.name=org.apache.http
+logger.ApacheHttp.level = INFO
+
+logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
+logger.PerfLogger.level = ${hive.perflogger.log.level}
+
+# Ranger authorization logger
+logger.RangerAuth.name = org.apache.ranger 
+logger.RangerAuth.level = DEBUG
+
+#
+# Hive authentication
+# logger.HiveAuth.name = org.apache.hadoop.hive.ql.security
+# logger.HiveAuth.level = DEBUG
+#
+# Security and Kerberos
+# logger.SecurityAuth.name = org.apache.hadoop.security
+# logger.SecurityAuth.level = DEBUG
+
+# root logger (controls all packages unless overridden above)
+rootLogger.level = ${hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${hive.root.logger}
diff --git a/dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties b/dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties
new file mode 100644
index 000000000..88a8a4925
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = INFO
+name = HiveMetastoreLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+# Change to DEBUG for detailed troubleshooting
+property.hive.log.level = INFO
+property.hive.root.logger = DRFA
+property.hive.log.dir = /opt/hive/logs
+property.hive.log.file = metastore.log
+property.hive.perflogger.log.level = INFO
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingRandomAccessFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${hive.log.dir}/${hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp, RangerAuth
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.AmazonAws.name=com.amazonaws
+logger.AmazonAws.level = INFO
+
+logger.ApacheHttp.name=org.apache.http
+logger.ApacheHttp.level = INFO
+
+logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
+logger.PerfLogger.level = ${hive.perflogger.log.level}
+
+# Ranger authorization logger
+logger.RangerAuth.name = org.apache.ranger
+logger.RangerAuth.level = INFO
+#
+# Hive authentication
+# logger.HiveAuth.name = org.apache.hadoop.hive.ql.security
+# logger.HiveAuth.level = INFO
+#
+# Security and Kerberos
+# logger.SecurityAuth.name = org.apache.hadoop.security
+# logger.SecurityAuth.level = INFO
+
+# root logger (controls all packages unless overridden above)
+rootLogger.level = ${hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${hive.root.logger}
+
diff --git a/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh b/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh
index 591e2182b..bacf00400 100755
--- a/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh
+++ b/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh
@@ -31,6 +31,8 @@ fi
 
 cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hive-site.xml
 cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hiveserver2-site.xml
+cp ${RANGER_SCRIPTS}/hive-log4j2.properties ${HIVE_HOME}/conf/hive-log4j2.properties
+cp ${RANGER_SCRIPTS}/hive-metastore-log4j2.properties ${HIVE_HOME}/conf/hive-metastore-log4j2.properties
 
 # Configure Tez
 mkdir -p ${TEZ_HOME}/conf
@@ -77,6 +79,10 @@ cat <<EOF > ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
     <name>yarn.resourcemanager.address</name>
     <value>ranger-hadoop:8032</value>
   </property>
+  <property>
+    <name>yarn.resourcemanager.principal</name>
+    <value>rm/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
 </configuration>
 EOF
 
diff --git a/dev-support/ranger-docker/scripts/hive/ranger-hive.sh b/dev-support/ranger-docker/scripts/hive/ranger-hive.sh
index ee380b13f..3606f1bb2 100755
--- a/dev-support/ranger-docker/scripts/hive/ranger-hive.sh
+++ b/dev-support/ranger-docker/scripts/hive/ranger-hive.sh
@@ -91,11 +91,11 @@ cd "${HIVE_HOME}" || exit
 
 # Start Hive MetaStore
 echo "Starting Hive MetaStore..."
-su -c "nohup ${HIVE_HOME}/bin/hive --service metastore > metastore.log 2>&1 &" hive
+su -c "export HADOOP_CLIENT_OPTS='${HADOOP_CLIENT_OPTS} -Dlog4j2.configurationFile=file:${HIVE_HOME}/conf/hive-metastore-log4j2.properties' && nohup ${HIVE_HOME}/bin/hive --service metastore &" hive
 
 # Start HiveServer2
 echo "Starting HiveServer2..."
-su -c "nohup ${HIVE_HOME}/bin/hiveserver2 > hive-server2.log 2>&1 &" hive
+su -c "export HADOOP_CLIENT_OPTS='${HADOOP_CLIENT_OPTS} -Dlog4j2.configurationFile=file:${HIVE_HOME}/conf/hive-log4j2.properties' && nohup ${HIVE_HOME}/bin/hiveserver2 &" hive
 
 # Wait for services to initialize
 echo "Waiting for Hive services to initialize..."