You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/10/21 04:36:04 UTC
[ranger] branch master updated: RANGER-3053: docker scripts updated
for recent component version upgrades
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 1b60559 RANGER-3053: docker scripts updated for recent component version upgrades
1b60559 is described below
commit 1b6055993f2f59cf557964a99da9d3e0e98c10e1
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Mon Oct 19 17:40:42 2020 -0700
RANGER-3053: docker scripts updated for recent component version upgrades
---
dev-support/ranger-docker/.env | 4 +-
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 1 +
dev-support/ranger-docker/Dockerfile.ranger-hbase | 1 +
dev-support/ranger-docker/scripts/hbase-site.xml | 52 ++++++++++++++++++++++
.../scripts/ranger-admin-install.properties | 5 ++-
.../{ranger-hadoop.sh => ranger-hadoop-mkdir.sh} | 39 ++++++----------
dev-support/ranger-docker/scripts/ranger-hadoop.sh | 8 ++++
.../scripts/ranger-hbase-plugin-install.properties | 6 +--
.../ranger-docker/scripts/ranger-hbase-setup.sh | 1 +
.../scripts/ranger-hdfs-plugin-install.properties | 6 +--
.../scripts/ranger-kafka-plugin-install.properties | 6 +--
.../scripts/ranger-yarn-plugin-install.properties | 6 +--
12 files changed, 94 insertions(+), 41 deletions(-)
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
index a3e40d9..5968960 100644
--- a/dev-support/ranger-docker/.env
+++ b/dev-support/ranger-docker/.env
@@ -6,6 +6,6 @@ BRANCH=master
POSTGRES_PASSWORD=rangerR0cks!
RANGER_VERSION=3.0.0-SNAPSHOT
-HADOOP_VERSION=3.1.1
-HBASE_VERSION=2.0.3
+HADOOP_VERSION=3.3.0
+HBASE_VERSION=2.2.6
KAFKA_VERSION=2.4.0
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
index c9513b6..71ff1c2 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -25,6 +25,7 @@ COPY ./dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/
COPY ./scripts/ranger-hadoop-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-hadoop.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hadoop-mkdir.sh /home/ranger/scripts/
COPY ./scripts/ranger-hdfs-plugin-install.properties /home/ranger/scripts/
COPY ./scripts/ranger-yarn-plugin-install.properties /home/ranger/scripts/
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase
index 5dea37a..a0a6827 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hbase
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase
@@ -25,6 +25,7 @@ COPY ./dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz /home/ranger/dist/
COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/
COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/
+COPY ./scripts/hbase-site.xml /home/ranger/scripts/
RUN curl https://archive.apache.org/dist/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz --output /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
tar xvfz /tmp/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
diff --git a/dev-support/ranger-docker/scripts/hbase-site.xml b/dev-support/ranger-docker/scripts/hbase-site.xml
new file mode 100644
index 0000000..ef4bc26
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/hbase-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--><configuration>
+ <!--
+ The following properties are set for running HBase as a single process on a
+ developer workstation. With this configuration, HBase is running in
+ "stand-alone" mode and without a distributed file system. In this mode, and
+ without further configuration, HBase and ZooKeeper data are stored on the
+ local filesystem, in a path under the value configured for `hbase.tmp.dir`.
+ This value is overridden from its default value of `/tmp` because many
+ systems clean `/tmp` on a regular basis. Instead, it points to a path within
+ this HBase installation directory.
+
+ Running against the `LocalFileSystem`, as opposed to a distributed
+ filesystem, runs the risk of data integrity issues and data loss. Normally
+ HBase will refuse to run in such an environment. Setting
+ `hbase.unsafe.stream.capability.enforce` to `false` overrides this behavior,
+ permitting operation. This configuration is for the developer workstation
+ only and __should not be used in production!__
+
+ See also https://hbase.apache.org/book.html#standalone_dist
+ -->
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.root.dir</name>
+ <value>hdfs://ranger-hadoop:9000/hbase</value>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>hdfs://ranger-hadoop:9000/hbase</value>
+ </property>
+</configuration>
diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install.properties b/dev-support/ranger-docker/scripts/ranger-admin-install.properties
index 9bb8c75..86019c2 100644
--- a/dev-support/ranger-docker/scripts/ranger-admin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-admin-install.properties
@@ -47,7 +47,7 @@ audit_solr_collection_name=ranger_audits
policymgr_external_url=http://ranger-admin:6080
policymgr_http_enabled=true
-unix_user=rangerusersync
+unix_user=ranger
unix_user_pwd=ranger
unix_group=ranger
@@ -76,3 +76,6 @@ ranger_admin_max_heap_size=1g
#retry DB and Java patches after the given time in seconds.
PATCH_RETRY_INTERVAL=120
STALE_PATCH_ENTRY_HOLD_TIME=10
+
+hadoop_conf=
+authentication_method=UNIX
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
similarity index 51%
copy from dev-support/ranger-docker/scripts/ranger-hadoop.sh
copy to dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
index 5c33f75..ab9911a 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
@@ -16,29 +16,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-service ssh start
-
-if [ ! -e ${HADOOP_HOME}/.setupDone ]
-then
- su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs
- su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hdfs
- su -c "chmod 0600 ~/.ssh/authorized_keys" hdfs
-
- su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" yarn
- su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" yarn
- su -c "chmod 0600 ~/.ssh/authorized_keys" yarn
-
- echo "ssh" > /etc/pdsh/rcmd_default
-
- ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh
-
- su -c "${HADOOP_HOME}/bin/hdfs namenode -format" hdfs
-
- touch ${HADOOP_HOME}/.setupDone
-fi
-
-su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
-su -c "${HADOOP_HOME}/sbin/start-yarn.sh" yarn
-
-# prevent the container from exiting
-/bin/bash
+${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseRegional
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/kafka
+
+${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
+${HADOOP_HOME}/bin/hdfs dfs -chown hdfs:hadoop /ranger/audit/hdfs
+${HADOOP_HOME}/bin/hdfs dfs -chown yarn:hadoop /ranger/audit/yarn
+${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseMaster
+${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseRegional
+${HADOOP_HOME}/bin/hdfs dfs -chown kafka:hadoop /ranger/audit/kafka
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
index 5c33f75..8e3ee28 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
@@ -18,6 +18,8 @@
service ssh start
+CREATE_HDFS_DIR=false
+
if [ ! -e ${HADOOP_HOME}/.setupDone ]
then
su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs
@@ -34,11 +36,17 @@ then
su -c "${HADOOP_HOME}/bin/hdfs namenode -format" hdfs
+ CREATE_HDFS_DIR=true
touch ${HADOOP_HOME}/.setupDone
fi
su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
su -c "${HADOOP_HOME}/sbin/start-yarn.sh" yarn
+if [ "${CREATE_HDFS_DIR}" == "true" ]
+then
+ su -c "${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh" hdfs
+fi
+
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
index bc80a6d..eda6c8f 100644
--- a/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
@@ -57,8 +57,8 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
-XAAUDIT.HDFS.ENABLE=false
-XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/hdfs/spool
XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
@@ -70,7 +70,7 @@ XAAUDIT.LOG4J.ENABLE=false
XAAUDIT.LOG4J.IS_ASYNC=false
XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
-XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
index bc6d226..95a1bdf 100755
--- a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
@@ -24,6 +24,7 @@ Host *
UserKnownHostsFile=/dev/null
EOF
+cp ${RANGER_SCRIPTS}/hbase-site.xml /opt/hbase/conf/hbase-site.xml
chown -R hbase:hadoop /opt/hbase/
cd ${RANGER_HOME}/ranger-hbase-plugin
diff --git a/dev-support/ranger-docker/scripts/ranger-hdfs-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hdfs-plugin-install.properties
index 686cda0..806a83c 100644
--- a/dev-support/ranger-docker/scripts/ranger-hdfs-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-hdfs-plugin-install.properties
@@ -54,8 +54,8 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
-XAAUDIT.HDFS.ENABLE=false
-XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/hdfs/spool
XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
@@ -67,7 +67,7 @@ XAAUDIT.LOG4J.ENABLE=false
XAAUDIT.LOG4J.IS_ASYNC=false
XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
-XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties
index ccff25b..028c8b5 100644
--- a/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties
@@ -57,8 +57,8 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
-XAAUDIT.HDFS.ENABLE=false
-XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/kafka/audit/hdfs/spool
XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
@@ -70,7 +70,7 @@ XAAUDIT.LOG4J.ENABLE=false
XAAUDIT.LOG4J.IS_ASYNC=false
XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
-XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
diff --git a/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties
index f7cc53b..1ff6127 100644
--- a/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties
@@ -54,8 +54,8 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
-XAAUDIT.HDFS.ENABLE=false
-XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/yarn/audit/hdfs/spool
XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
@@ -67,7 +67,7 @@ XAAUDIT.LOG4J.ENABLE=false
XAAUDIT.LOG4J.IS_ASYNC=false
XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
-XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks