You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/08/28 20:49:51 UTC
[ranger] branch ranger-2.1 updated: RANGER-2974: Docker setup to
run Ranger enabled Kafka
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.1
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.1 by this push:
new a53a891 RANGER-2974: Docker setup to run Ranger enabled Kafka
a53a891 is described below
commit a53a891114feb310483d8797a3c49ca68d840d5c
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Thu Aug 27 12:47:00 2020 -0700
RANGER-2974: Docker setup to run Ranger enabled Kafka
(cherry picked from commit 6ad164124fc68b2541161f76cf87228e665432d1)
---
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/Dockerfile.ranger | 1 +
dev-support/ranger-docker/Dockerfile.ranger-base | 2 +
dev-support/ranger-docker/Dockerfile.ranger-kafka | 38 +++++++++++
dev-support/ranger-docker/README.md | 20 ++++--
.../ranger-docker/docker-compose.ranger-kafka.yml | 18 +++++
.../scripts/ranger-kafka-plugin-install.properties | 79 ++++++++++++++++++++++
.../scripts/ranger-kafka-service-dev_kafka.py | 8 +++
.../scripts/{ranger.sh => ranger-kafka-setup.sh} | 36 +++-------
.../scripts/{ranger.sh => ranger-kafka.sh} | 33 ++++-----
dev-support/ranger-docker/scripts/ranger.sh | 1 +
11 files changed, 184 insertions(+), 53 deletions(-)
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index e7be836..d0a6bc7 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -6,4 +6,5 @@
!dist/ranger-*-yarn-plugin.tar.gz
!dist/ranger-*-hive-plugin.tar.gz
!dist/ranger-*-hbase-plugin.tar.gz
+!dist/ranger-*-kafka-plugin.tar.gz
!scripts/*
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
index 90d56f1..d414592 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -24,6 +24,7 @@ COPY ./scripts/ranger-hdfs-service-dev_hdfs.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-yarn-service-dev_yarn.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hive-service-dev_hive.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hbase-service-dev_hbase.py ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-kafka-service-dev_kafka.py ${RANGER_SCRIPTS}/
COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /tmp/
RUN tar xvfz /tmp/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base
index 74e5a73..4d47a37 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-base
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -21,6 +21,7 @@ ENV RANGER_VERSION 2.1.0
ENV HADOOP_VERSION 3.1.1
ENV HIVE_VERSION 3.1.2
ENV HBASE_VERSION 2.0.3
+ENV KAFKA_VERSION 2.4.0
# Install curl, wget, tzdata, Python, Java, python-requests
RUN apt-get update && \
@@ -52,6 +53,7 @@ RUN groupadd ranger && \
useradd -g hadoop -ms /bin/bash yarn && \
useradd -g hadoop -ms /bin/bash hive && \
useradd -g hadoop -ms /bin/bash hbase && \
+ useradd -g hadoop -ms /bin/bash kafka && \
mkdir -p /home/ranger/dist && \
mkdir -p /home/ranger/scripts && \
chown -R ranger:ranger /home/ranger && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka
new file mode 100644
index 0000000..42fb90f
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ranger-base:latest
+
+
+COPY ./dist/version /home/ranger/dist/
+COPY ./dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz /home/ranger/dist/
+COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/
+COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/
+COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/
+
+RUN curl https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz --output /tmp/kafka_2.11-${KAFKA_VERSION}.tgz && \
+ tar xvfz /tmp/kafka_2.11-${KAFKA_VERSION}.tgz --directory=/opt/ && \
+ ln -s /opt/kafka_2.11-${KAFKA_VERSION} /opt/kafka && \
+ rm -f /tmp/kafka_2.11-${KAFKA_VERSION}.tgz && \
+ tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz --directory=/opt/ranger && \
+ ln -s /opt/ranger/ranger-${RANGER_VERSION}-kafka-plugin /opt/ranger/ranger-kafka-plugin && \
+ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz && \
+ cp -f /home/ranger/scripts/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties
+
+ENV KAFKA_HOME /opt/kafka
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-kafka.sh" ]
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 6de0d06..483da70 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -40,7 +40,7 @@ deploy Apache Ranger and its dependent services in containers.
status of ${HOME}/.m2 directory cache.
3.2. Execute following command to start Ranger, Ranger enabled HDFS, Ranger enabled HBase, and dependeny services (Solr, DB) in continers:
- docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -f docker-compose.ranger-hbase.yml -d
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml up -d
4. Alternatively docker command can be used to build and deploy Apache Ranger.
4.1. Execute following command to build Docker image **ranger-base**:
@@ -78,21 +78,31 @@ deploy Apache Ranger and its dependent services in containers.
4.9. Execute following command to build Docker image **ranger-hadoop**:
docker build -f Dockerfile.ranger-hadoop -t ranger-hadoop .
- This steps includes downloading of Hadoop tar balls, and can take a while to complete.
+ This step includes downloading of Hadoop tar balls, and can take a while to complete.
4.10. Execute following command to install and run Ranger enabled HDFS in a container:
docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 -p 8088:8088 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
- This might take few minutes to complete.
+ This might take few minutes to complete.
4.11. Execute following command to build Docker image **ranger-hbase**:
docker build -f Dockerfile.ranger-hbase -t ranger-hbase .
- This steps includes downloading of HBase tar ball, and can take a while to complete.
+ This step includes downloading of HBase tar ball, and can take a while to complete.
4.12. Execute following command to install and run Ranger enabled HBase in a container:
docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-hbase
- This might take few minutes to complete.
+ This might take few minutes to complete.
+
+ 4.13. Execute following command to build Docker image **ranger-kafka**:
+ docker build -f Dockerfile.ranger-kafka -t ranger-kafka .
+
+ This step includes downloading of Kafka tar ball, and can take a while to complete.
+
+ 4.12. Execute following command to install and run Ranger enabled Kafka in a container:
+ docker run -it -d --name ranger-kafka --hostname ranger-kafka.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-kafka
+
+ This might take few minutes to complete.
5. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!)
diff --git a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
new file mode 100644
index 0000000..1d14f1b
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
@@ -0,0 +1,18 @@
+version: '3'
+services:
+ ranger-kafka:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-kafka
+ image: ranger-kafka
+ container_name: ranger-kafka
+ hostname: ranger-kafka.example.com
+ stdin_open: true
+ tty: true
+ networks:
+ - ranger
+ depends_on:
+ - ranger
+
+networks:
+ ranger:
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties
new file mode 100644
index 0000000..ccff25b
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+POLICY_MGR_URL=http://ranger:6080
+REPOSITORY_NAME=dev_kafka
+COMPONENT_INSTALL_DIR_NAME=/opt/kafka
+
+CUSTOM_USER=kafka
+CUSTOM_GROUP=hadoop
+
+XAAUDIT.SUMMARY.ENABLE=true
+UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
+
+XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
+
+# Following properties are needed to get past installation script! Please don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hadoop
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/kafka/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/kafka/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/kafka/audit/solr/spool
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=false
+XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/kafka/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py b/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
new file mode 100644
index 0000000..2274d32
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
@@ -0,0 +1,8 @@
+from apache_ranger.model.ranger_service import RangerService
+from apache_ranger.client.ranger_client import RangerClient
+
+ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+
+service = RangerService(name='dev_kafka', type='kafka', configs={'username':'kafka', 'password':'kafka', 'zookeeper.connect': 'ranger-kafka:2181'})
+
+ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
similarity index 53%
copy from dev-support/ranger-docker/scripts/ranger.sh
copy to dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
index ef46369..77cf465 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
@@ -16,33 +16,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+cat <<EOF > /etc/ssh/ssh_config
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile=/dev/null
+EOF
-if [ ! -e ${RANGER_HOME}/.setupDone ]
-then
- SETUP_RANGER=true
-else
- SETUP_RANGER=false
-fi
+chown -R kafka:hadoop /opt/kafka/
-if [ "${SETUP_RANGER}" == "true" ]
-then
- su -c "cd ${RANGER_HOME}/admin && ./setup.sh" ranger
+cd ${RANGER_HOME}/ranger-kafka-plugin
+./enable-kafka-plugin.sh
- touch ${RANGER_HOME}/.setupDone
-fi
-
-su -c "cd ${RANGER_HOME}/admin && ./ews/ranger-admin-services.sh start" ranger
-
-if [ "${SETUP_RANGER}" == "true" ]
-then
- # Wait for Ranger Admin to become ready
- sleep 30
-
- python3 ${RANGER_SCRIPTS}/ranger-hdfs-service-dev_hdfs.py
- python3 ${RANGER_SCRIPTS}/ranger-yarn-service-dev_yarn.py
- python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py
- python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py
-fi
-
-# prevent the container from exiting
-/bin/bash
+echo "authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" >> ${KAFKA_HOME}/config/server.properties
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger-kafka.sh
similarity index 57%
copy from dev-support/ranger-docker/scripts/ranger.sh
copy to dev-support/ranger-docker/scripts/ranger-kafka.sh
index ef46369..e5145f8 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger-kafka.sh
@@ -16,33 +16,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+service ssh start
-if [ ! -e ${RANGER_HOME}/.setupDone ]
+if [ ! -e ${KAFKA_HOME}/.setupDone ]
then
- SETUP_RANGER=true
-else
- SETUP_RANGER=false
-fi
-
-if [ "${SETUP_RANGER}" == "true" ]
-then
- su -c "cd ${RANGER_HOME}/admin && ./setup.sh" ranger
+ su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" kafka
+ su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" kafka
+ su -c "chmod 0600 ~/.ssh/authorized_keys" kafka
- touch ${RANGER_HOME}/.setupDone
-fi
+ echo "ssh" > /etc/pdsh/rcmd_default
-su -c "cd ${RANGER_HOME}/admin && ./ews/ranger-admin-services.sh start" ranger
+ ${RANGER_SCRIPTS}/ranger-kafka-setup.sh
-if [ "${SETUP_RANGER}" == "true" ]
-then
- # Wait for Ranger Admin to become ready
- sleep 30
-
- python3 ${RANGER_SCRIPTS}/ranger-hdfs-service-dev_hdfs.py
- python3 ${RANGER_SCRIPTS}/ranger-yarn-service-dev_yarn.py
- python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py
- python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py
+ touch ${KAFKA_HOME}/.setupDone
fi
+su -c "cd ${KAFKA_HOME} && ./bin/zookeeper-server-start.sh config/zookeeper.properties &" kafka
+sleep 30
+su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config ./bin/kafka-server-start.sh config/server.properties &" kafka
+
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh
index ef46369..0b62344 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger.sh
@@ -42,6 +42,7 @@ then
python3 ${RANGER_SCRIPTS}/ranger-yarn-service-dev_yarn.py
python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py
python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py
+ python3 ${RANGER_SCRIPTS}/ranger-kafka-service-dev_kafka.py
fi
# prevent the container from exiting