You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/08/27 23:53:45 UTC
[ranger] branch master updated: RANGER-2975: Docker setup to enable
Ranger authorization in YARN
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new ac4e663 RANGER-2975: Docker setup to enable Ranger authorization in YARN
ac4e663 is described below
commit ac4e6639af5a549d6a9253272143a1dc295ff58c
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Thu Aug 27 09:11:29 2020 -0700
RANGER-2975: Docker setup to enable Ranger authorization in YARN
---
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/Dockerfile.ranger | 1 +
dev-support/ranger-docker/Dockerfile.ranger-base | 1 +
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 8 ++-
dev-support/ranger-docker/README.md | 2 +-
.../ranger-docker/docker-compose.ranger-hadoop.yml | 1 +
.../ranger-docker/scripts/ranger-hadoop-setup.sh | 5 ++
dev-support/ranger-docker/scripts/ranger-hadoop.sh | 6 +-
.../scripts/ranger-yarn-plugin-install.properties | 76 ++++++++++++++++++++++
.../scripts/ranger-yarn-service-dev_yarn.py | 8 +++
dev-support/ranger-docker/scripts/ranger.sh | 1 +
11 files changed, 107 insertions(+), 3 deletions(-)
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index 3ffb780..e7be836 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -3,6 +3,7 @@
!dist/version
!dist/ranger-*-admin.tar.gz
!dist/ranger-*-hdfs-plugin.tar.gz
+!dist/ranger-*-yarn-plugin.tar.gz
!dist/ranger-*-hive-plugin.tar.gz
!dist/ranger-*-hbase-plugin.tar.gz
!scripts/*
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
index fca32ae..90d56f1 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -21,6 +21,7 @@ COPY ./dist/version ${RANGER_DIST}/
COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-admin-install.properties ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hdfs-service-dev_hdfs.py ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-yarn-service-dev_yarn.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hive-service-dev_hive.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hbase-service-dev_hbase.py ${RANGER_SCRIPTS}/
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base
index fff84eb..00caf16 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-base
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -49,6 +49,7 @@ RUN groupadd ranger && \
useradd -g ranger -ms /bin/bash rangerkms && \
groupadd hadoop && \
useradd -g hadoop -ms /bin/bash hdfs && \
+ useradd -g hadoop -ms /bin/bash yarn && \
useradd -g hadoop -ms /bin/bash hive && \
useradd -g hadoop -ms /bin/bash hbase && \
mkdir -p /home/ranger/dist && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
index d6046af..6f1e89f 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -19,9 +19,11 @@ FROM ranger-base:latest
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
+COPY ./dist/ranger-${RANGER_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/
COPY ./scripts/ranger-hadoop-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-hadoop.sh /home/ranger/scripts/
COPY ./scripts/ranger-hdfs-plugin-install.properties /home/ranger/scripts/
+COPY ./scripts/ranger-yarn-plugin-install.properties /home/ranger/scripts/
RUN curl https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz --output /tmp/hadoop-${HADOOP_VERSION}.tar.gz && \
tar xvfz /tmp/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
@@ -30,7 +32,11 @@ RUN curl https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/
tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz --directory=/opt/ranger && \
ln -s /opt/ranger/ranger-${RANGER_VERSION}-hdfs-plugin /opt/ranger/ranger-hdfs-plugin && \
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz && \
- cp -f /home/ranger/scripts/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties
+ cp -f /home/ranger/scripts/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties && \
+ tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-yarn-plugin.tar.gz --directory=/opt/ranger && \
+ ln -s /opt/ranger/ranger-${RANGER_VERSION}-yarn-plugin /opt/ranger/ranger-yarn-plugin && \
+ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-yarn-plugin.tar.gz && \
+ cp -f /home/ranger/scripts/ranger-yarn-plugin-install.properties /opt/ranger/ranger-yarn-plugin/install.properties
ENV HADOOP_HOME /opt/hadoop
ENV HADOOP_CONF_DIR /opt/hadoop/etc/hadoop
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 0fad420..6de0d06 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -81,7 +81,7 @@ deploy Apache Ranger and its dependent services in containers.
This steps includes downloading of Hadoop tar balls, and can take a while to complete.
4.10. Execute following command to install and run Ranger enabled HDFS in a container:
- docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
+ docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 -p 8088:8088 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
This might take few minutes to complete.
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
index a92f3d2..e521345 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
@@ -13,6 +13,7 @@ services:
- ranger
ports:
- "9000:9000"
+ - "8088:8088"
depends_on:
- ranger
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh
index ebf25ce..fa22613 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh
@@ -55,7 +55,12 @@ cat <<EOF > ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
</configuration>
EOF
+mkdir -p /opt/hadoop/logs
chown -R hdfs:hadoop /opt/hadoop/
+chmod g+w /opt/hadoop/logs
cd ${RANGER_HOME}/ranger-hdfs-plugin
./enable-hdfs-plugin.sh
+
+cd ${RANGER_HOME}/ranger-yarn-plugin
+./enable-yarn-plugin.sh
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
index 8dc5cd4..5c33f75 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
@@ -24,6 +24,10 @@ then
su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hdfs
su -c "chmod 0600 ~/.ssh/authorized_keys" hdfs
+ su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" yarn
+ su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" yarn
+ su -c "chmod 0600 ~/.ssh/authorized_keys" yarn
+
echo "ssh" > /etc/pdsh/rcmd_default
${RANGER_SCRIPTS}/ranger-hadoop-setup.sh
@@ -34,7 +38,7 @@ then
fi
su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
-su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs
+su -c "${HADOOP_HOME}/sbin/start-yarn.sh" yarn
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties
new file mode 100644
index 0000000..f7cc53b
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-yarn-plugin-install.properties
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+POLICY_MGR_URL=http://ranger:6080
+REPOSITORY_NAME=dev_yarn
+COMPONENT_INSTALL_DIR_NAME=/opt/hadoop
+
+CUSTOM_USER=yarn
+CUSTOM_GROUP=hadoop
+
+XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
+
+# Following properties are needed to get past installation script! Please don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hadoop
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hadoop/yarn/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hadoop/yarn/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/yarn/audit/solr/spool
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=false
+XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/yarn/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
diff --git a/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py b/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py
new file mode 100644
index 0000000..b3e8a28
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py
@@ -0,0 +1,8 @@
+from apache_ranger.model.ranger_service import RangerService
+from apache_ranger.client.ranger_client import RangerClient
+
+ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+
+service = RangerService(name='dev_yarn', type='yarn', configs={'username':'yarn', 'password':'yarn', 'yarn.url': 'http://ranger-hadoop:8088'})
+
+ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh
index 3076556..ef46369 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger.sh
@@ -39,6 +39,7 @@ then
sleep 30
python3 ${RANGER_SCRIPTS}/ranger-hdfs-service-dev_hdfs.py
+ python3 ${RANGER_SCRIPTS}/ranger-yarn-service-dev_yarn.py
python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py
python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py
fi