You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2021/03/13 17:39:50 UTC
[ranger] branch ranger-2.2 updated: RANGER-3196: Docker build
updated to use cached archives to avoid repeated downloads
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.2
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.2 by this push:
new ed9a6be RANGER-3196: Docker build updated to use cached archives to avoid repeated downloads
ed9a6be is described below
commit ed9a6beda01f9edf6e41f7439c6d3a676cec8574
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Sun Feb 28 23:46:21 2021 -0800
RANGER-3196: Docker build updated to use cached archives to avoid repeated downloads
(cherry picked from commit 5c8db0ff6c760e0e27ce89d4e55820c13c4b362a)
---
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/.env | 2 +-
dev-support/ranger-docker/Dockerfile.ranger | 12 ++-
dev-support/ranger-docker/Dockerfile.ranger-base | 4 +-
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 7 +-
dev-support/ranger-docker/Dockerfile.ranger-hbase | 7 +-
dev-support/ranger-docker/Dockerfile.ranger-hive | 13 +--
dev-support/ranger-docker/Dockerfile.ranger-kafka | 7 +-
.../ranger-kafka-setup.sh => Dockerfile.ranger-zk} | 15 +--
dev-support/ranger-docker/README.md | 106 ++-------------------
.../ranger-docker/docker-compose.ranger-hadoop.yml | 9 +-
.../ranger-docker/docker-compose.ranger-hbase.yml | 8 +-
.../ranger-docker/docker-compose.ranger-hive.yml | 9 +-
.../ranger-docker/docker-compose.ranger-kafka.yml | 5 +-
.../ranger-docker/docker-compose.ranger.yml | 21 +++-
dev-support/ranger-docker/download-archives.sh | 52 ++++++++++
dev-support/ranger-docker/downloads/.gitignore | 1 +
dev-support/ranger-docker/scripts/hbase-site.xml | 8 +-
dev-support/ranger-docker/scripts/hive-site.xml | 3 +-
dev-support/ranger-docker/scripts/ranger-hadoop.sh | 4 +-
dev-support/ranger-docker/scripts/ranger-hbase.sh | 4 +-
.../scripts/ranger-hive-plugin-install.properties | 2 +-
dev-support/ranger-docker/scripts/ranger-hive.sh | 4 +-
.../scripts/ranger-kafka-service-dev_kafka.py | 2 +-
.../ranger-docker/scripts/ranger-kafka-setup.sh | 2 +
dev-support/ranger-docker/scripts/ranger-kafka.sh | 7 +-
dev-support/ranger-docker/scripts/ranger.sh | 4 +-
27 files changed, 156 insertions(+), 163 deletions(-)
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index d0a6bc7..e42d865 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -7,4 +7,5 @@
!dist/ranger-*-hive-plugin.tar.gz
!dist/ranger-*-hbase-plugin.tar.gz
!dist/ranger-*-kafka-plugin.tar.gz
+!downloads/*
!scripts/*
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
index 55420d8..1d45a62 100644
--- a/dev-support/ranger-docker/.env
+++ b/dev-support/ranger-docker/.env
@@ -5,7 +5,7 @@ BRANCH=ranger-2.2
POSTGRES_PASSWORD=rangerR0cks!
-RANGER_VERSION=3.0.0-SNAPSHOT
+RANGER_VERSION=2.2.0-SNAPSHOT
HADOOP_VERSION=3.3.0
HBASE_VERSION=2.2.6
HIVE_VERSION=3.1.2
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
index 6225595..8940014 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -18,7 +18,10 @@ FROM ranger-base:latest
ARG RANGER_VERSION
-COPY ./dist/version ${RANGER_DIST}/
+COPY ./dist/version /home/ranger/dist/
+COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
+COPY ./downloads/postgresql-42.2.16.jre7.jar /home/ranger/dist/
+
COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-admin-install.properties ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hdfs-service-dev_hdfs.py ${RANGER_SCRIPTS}/
@@ -27,15 +30,14 @@ COPY ./scripts/ranger-hive-service-dev_hive.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-hbase-service-dev_hbase.py ${RANGER_SCRIPTS}/
COPY ./scripts/ranger-kafka-service-dev_kafka.py ${RANGER_SCRIPTS}/
-COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /tmp/
-RUN tar xvfz /tmp/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} && \
+RUN tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} && \
ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin && \
- rm -f /tmp/ranger-${RANGER_VERSION}-admin.tar.gz && \
+ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz && \
cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties ${RANGER_HOME}/admin/install.properties && \
mkdir -p /var/run/ranger && \
mkdir -p /var/log/ranger && \
chown -R ranger:ranger ${RANGER_HOME}/admin/ /var/run/ranger/ /var/log/ranger/ && \
mkdir -p /usr/share/java/ && \
- wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar" -O /usr/share/java/postgresql.jar
+ mv /home/ranger/dist/postgresql-42.2.16.jre7.jar /usr/share/java/postgresql.jar
ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base
index c7fb75c..3fa657b 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-base
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -17,9 +17,9 @@
FROM ubuntu:20.04
-# Install curl, wget, tzdata, Python, Java, python-requests
+# Install tzdata, Python, Java, python-requests
RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
+ DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata \
python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
pip3 install apache-ranger && \
pip3 install requests
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
index 71ff1c2..f25bc0d 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -23,16 +23,17 @@ ARG RANGER_VERSION
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/
+COPY ./downloads/hadoop-${HADOOP_VERSION}.tar.gz /home/ranger/dist/
+
COPY ./scripts/ranger-hadoop-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-hadoop.sh /home/ranger/scripts/
COPY ./scripts/ranger-hadoop-mkdir.sh /home/ranger/scripts/
COPY ./scripts/ranger-hdfs-plugin-install.properties /home/ranger/scripts/
COPY ./scripts/ranger-yarn-plugin-install.properties /home/ranger/scripts/
-RUN curl https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz --output /tmp/hadoop-${HADOOP_VERSION}.tar.gz && \
- tar xvfz /tmp/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
+RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \
- rm -f /tmp/hadoop-${HADOOP_VERSION}.tar.gz && \
+ rm -f /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz && \
tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz --directory=/opt/ranger && \
ln -s /opt/ranger/ranger-${RANGER_VERSION}-hdfs-plugin /opt/ranger/ranger-hdfs-plugin && \
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase
index a0a6827..5a2f056 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hbase
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase
@@ -22,15 +22,16 @@ ARG RANGER_VERSION
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz /home/ranger/dist/
+COPY ./downloads/hbase-${HBASE_VERSION}-bin.tar.gz /home/ranger/dist/
+
COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/
COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/
COPY ./scripts/hbase-site.xml /home/ranger/scripts/
-RUN curl https://archive.apache.org/dist/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz --output /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
- tar xvfz /tmp/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
+RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \
- rm -f /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
+ rm -f /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz && \
tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz --directory=/opt/ranger && \
ln -s /opt/ranger/ranger-${RANGER_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive
index bd31629..6a27f3a 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hive
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hive
@@ -23,19 +23,20 @@ ARG RANGER_VERSION
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
+COPY ./downloads/apache-hive-${HIVE_VERSION}-bin.tar.gz /home/ranger/dist/
+COPY ./downloads/hadoop-${HIVE_HADOOP_VERSION}.tar.gz /home/ranger/dist/
+
COPY ./scripts/ranger-hive-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-hive.sh /home/ranger/scripts/
COPY ./scripts/ranger-hive-plugin-install.properties /home/ranger/scripts/
COPY ./scripts/hive-site.xml /home/ranger/scripts/
-RUN curl https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz --output /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
- tar xvfz /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
+RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
- rm -f /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
- curl https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION}/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --output /tmp/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
- tar xvfz /tmp/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --directory=/opt/ && \
+ rm -f /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
+ tar xvfz /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --directory=/opt/ && \
ln -s /opt/hadoop-${HIVE_HADOOP_VERSION} /opt/hadoop && \
- rm -f /tmp/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
+ rm -f /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
ln -s /opt/ranger/ranger-${RANGER_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka
index b09500b..83c41b2 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-kafka
+++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka
@@ -22,14 +22,15 @@ ARG RANGER_VERSION
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz /home/ranger/dist/
+COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/ranger/dist/
+
COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/
COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/
-RUN curl https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.12-${KAFKA_VERSION}.tgz --output /tmp/kafka_2.12-${KAFKA_VERSION}.tgz && \
- tar xvfz /tmp/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
+RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
- rm -f /tmp/kafka_2.12-${KAFKA_VERSION}.tgz && \
+ rm -f /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz --directory=/opt/ranger && \
ln -s /opt/ranger/ranger-${RANGER_VERSION}-kafka-plugin /opt/ranger/ranger-kafka-plugin && \
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz && \
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh b/dev-support/ranger-docker/Dockerfile.ranger-zk
old mode 100755
new mode 100644
similarity index 68%
copy from dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
copy to dev-support/ranger-docker/Dockerfile.ranger-zk
index 77cf465..2855b50
--- a/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
+++ b/dev-support/ranger-docker/Dockerfile.ranger-zk
@@ -1,5 +1,3 @@
-#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,15 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-cat <<EOF > /etc/ssh/ssh_config
-Host *
- StrictHostKeyChecking no
- UserKnownHostsFile=/dev/null
-EOF
-
-chown -R kafka:hadoop /opt/kafka/
-
-cd ${RANGER_HOME}/ranger-kafka-plugin
-./enable-kafka-plugin.sh
-
-echo "authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" >> ${KAFKA_HOME}/config/server.properties
+FROM zookeeper:3.5.9
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 1f8689a..6fb9659 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -23,116 +23,26 @@ Docker files in this folder create docker images and run them to build Apache Ra
## Usage
-1. Ensure that you have recent version of Docker installed from [docker.io](http://www.docker.io) (as of this writing: Engine 19.03, Compose 1.26.2).
+1. Ensure that you have recent version of Docker installed from [docker.io](http://www.docker.io) (as of this writing: Engine 20.10.5, Compose 1.28.5).
+ Make sure to configure docker with at least 6gb of memory.
2. Set this folder as your working directory.
3. Update environment variables in .env file, if necessary
-4. Using docker-compose is the simpler way to build and deploy Apache Ranger in containers.
+4. Execute following command to download necessary archives to setup Ranger/HDFS/Hive/HBase/Kafka services:
+ ./download-archives.sh
- 4.1. Execute following command to build Apache Ranger:
+5. Build and deploy Apache Ranger in containers using docker-compose
+
+ 5.1. Execute following command to build Apache Ranger:
docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up
Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
- 4.2. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Kafka and dependent services (Solr, DB) in containers:
+ 5.2. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Kafka and dependent services (Solr, DB) in containers:
docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml up -d
-5. Alternatively docker command can be used to build and deploy Apache Ranger.
-
- 5.1. Execute following command to build Docker image **ranger-base**:
-
- docker build -f Dockerfile.ranger-base -t ranger-base .
-
- This might take about 10 minutes to complete.
-
- 5.2. Execute following command to build Docker image **ranger-build**:
-
- docker build -f Dockerfile.ranger-build -t ranger-build .
-
- 5.3. Build Apache Ranger in a container with the following command:
-
- docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2:delegated -v $(pwd)/scripts:/home/ranger/scripts -v $(pwd)/../..:/home/ranger/src:delegated -v $(pwd)/dist:/home/ranger/dist --env-file ./.env ranger-build
-
- Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
-
- 5.4. Execute following command to build Docker image **ranger**:
-
- docker build -f Dockerfile.ranger --build-arg RANGER_VERSION=`cat dist/version` -t ranger .
-
- This might take about 10 minutes to complete.
-
- 5.5. Execute following command to build a Docker image **ranger-solr**:
-
- docker build -f Dockerfile.ranger-solr -t ranger-solr .
-
- 5.6. Execute following command to build a Docker image **ranger-db**:
-
- docker build -f Dockerfile.ranger-db -t ranger-db .
-
- 5.7. Execute following command to start a container that runs Solr for use by Ranger Admin:
-
- docker run --name ranger-solr --hostname ranger-solr.example.com -p 8983:8983 -d ranger-solr solr-precreate ranger_audits /opt/solr/server/solr/configsets/ranger_audits/
-
- 5.8. Execute following command to start a container that runs database for use by Ranger Admin:
-
- docker run --name ranger-db --hostname ranger-db.example.com --env-file ./.env -d ranger-db
-
- 5.9. Execute following command to install and run Ranger services in a container:
-
- docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr --env-file ./.env ranger
-
- This might take few minutes to complete.
-
- 5.10. Execute following command to build Docker image **ranger-hadoop**:
-
- docker build -f Dockerfile.ranger-hadoop --build-arg RANGER_VERSION=`cat dist/version` --build-arg HADOOP_VERSION=3.3.0 -t ranger-hadoop .
-
- This step includes downloading of Hadoop tar balls, and can take a while to complete.
-
- 5.11. Execute following command to install and run Ranger enabled HDFS in a container:
-
- docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 -p 8088:8088 --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hadoop
-
- This might take few minutes to complete.
-
- 5.12. Execute following command to build Docker image **ranger-hbase**:
-
- docker build -f Dockerfile.ranger-hbase --build-arg RANGER_VERSION=`cat dist/version` --build-arg HBASE_VERSION=2.2.6 -t ranger-hbase .
-
- This step includes downloading of HBase tar ball, and can take a while to complete.
-
- 5.13. Execute following command to install and run Ranger enabled HBase in a container:
-
- docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hbase
-
- This might take few minutes to complete.
-
- 5.14. Execute following command to build Docker image **ranger-kafka**:
-
- docker build -f Dockerfile.ranger-kafka --build-arg RANGER_VERSION=`cat dist/version` --build-arg KAFKA_VERSION=2.5.0 -t ranger-kafka .
-
- This step includes downloading of Kafka tar ball, and can take a while to complete.
-
- 5.15. Execute following command to install and run Ranger enabled Kafka in a container:
-
- docker run -it -d --name ranger-kafka --hostname ranger-kafka.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-kafka
-
- This might take few minutes to complete.
-
- 5.16. Execute following command to build Docker image **ranger-hive**:
-
- docker build -f Dockerfile.ranger-hive --build-arg RANGER_VERSION=`cat dist/version` --build-arg HIVE_VERSION=3.1.2 --build-arg HIVE_HADOOP_VERSION=3.1.0 -t ranger-hbase .
-
- This step includes downloading of Hive tar ball and Hadoop tar ball, and can take a while to complete.
-
- 5.17. Execute following command to install and run Ranger enabled Hive in a container:
-
- docker run -it -d --name ranger-hive --hostname ranger-hive.example.com --link ranger-hadoop:ranger-hadoop --link ranger-hbase:ranger-hbase --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hive
-
- This might take few minutes to complete.
-
6. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!)
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
index 54e5cb5..2ce346d 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
@@ -18,7 +18,14 @@ services:
- "9000:9000"
- "8088:8088"
depends_on:
- - ranger
+ ranger:
+ condition: service_started
+ healthcheck:
+ test: "hdfs dfs -ls /hbase"
+ interval: 1m30s
+ timeout: 10s
+ retries: 30
+ start_period: 40s
environment:
- HADOOP_VERSION
- RANGER_VERSION
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
index 46a9fee..0e890bb 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
@@ -15,11 +15,15 @@ services:
networks:
- ranger
ports:
- - "2181:16181"
+ - "16000:16000"
- "16010:16010"
- "16020:16020"
+ - "16030:16030"
depends_on:
- - ranger-hadoop
+ ranger-hadoop:
+ condition: service_healthy
+ ranger-zk:
+ condition: service_started
environment:
- HBASE_VERSION
- RANGER_VERSION
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hive.yml b/dev-support/ranger-docker/docker-compose.ranger-hive.yml
index d7c1797..5a2e22e 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-hive.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-hive.yml
@@ -18,9 +18,12 @@ services:
ports:
- "10000:10000"
depends_on:
- - ranger
- - ranger-hadoop
- - ranger-hbase
+ ranger:
+ condition: service_started
+ ranger-zk:
+ condition: service_started
+ ranger-hadoop:
+ condition: service_healthy
environment:
- HIVE_HADOOP_VERSION
- HIVE_VERSION
diff --git a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
index 5cf0b9a..6622b86 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
@@ -15,7 +15,10 @@ services:
networks:
- ranger
depends_on:
- - ranger
+ ranger:
+ condition: service_started
+ ranger-zk:
+ condition: service_started
environment:
- KAFKA_VERSION
- RANGER_VERSION
diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml
index db32cdb..59105c5 100644
--- a/dev-support/ranger-docker/docker-compose.ranger.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger.yml
@@ -16,14 +16,29 @@ services:
ports:
- "6080:6080"
depends_on:
- - ranger-base
- - ranger-db
- - ranger-solr
+ ranger-zk:
+ condition: service_started
+ ranger-db:
+ condition: service_started
+ ranger-solr:
+ condition: service_started
environment:
- RANGER_VERSION
command:
- /home/ranger/scripts/ranger.sh
+ ranger-zk:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-zk
+ image: ranger-zk
+ container_name: ranger-zk
+ hostname: ranger-zk.example.com
+ networks:
+ - ranger
+ ports:
+ - "2181:2181"
+
ranger-solr:
build:
context: .
diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh
new file mode 100755
index 0000000..e107be0
--- /dev/null
+++ b/dev-support/ranger-docker/download-archives.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Downloads HDFS/Hive/HBase/Kafka/.. archives to a local cache directory.
+# The downloaded archives will be used while building docker images that
+# run these services
+#
+
+
+#
+# source .env file to get versions to download
+#
+source .env
+
+
+downloadIfNotPresent() {
+ local fileName=$1
+ local urlBase=$2
+
+ if [ ! -f "downloads/${fileName}" ]
+ then
+ echo "downloading ${urlBase}/${fileName}.."
+
+ curl -L ${urlBase}/${fileName} --output downloads/${fileName}
+ else
+ echo "file already in cache: ${fileName}"
+ fi
+}
+
+downloadIfNotPresent hadoop-${HADOOP_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}
+downloadIfNotPresent hbase-${HBASE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hbase/${HBASE_VERSION}
+downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
+downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}
+downloadIfNotPresent hadoop-${HIVE_HADOOP_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION}
+downloadIfNotPresent postgresql-42.2.16.jre7.jar https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7
+
diff --git a/dev-support/ranger-docker/downloads/.gitignore b/dev-support/ranger-docker/downloads/.gitignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/dev-support/ranger-docker/downloads/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/dev-support/ranger-docker/scripts/hbase-site.xml b/dev-support/ranger-docker/scripts/hbase-site.xml
index ef4bc26..6789bf1 100644
--- a/dev-support/ranger-docker/scripts/hbase-site.xml
+++ b/dev-support/ranger-docker/scripts/hbase-site.xml
@@ -42,11 +42,11 @@
<value>true</value>
</property>
<property>
- <name>hbase.root.dir</name>
- <value>hdfs://ranger-hadoop:9000/hbase</value>
+ <name>hbase.rootdir</name>
+ <value>hdfs://ranger-hadoop.example.com:9000/hbase</value>
</property>
<property>
- <name>hbase.tmp.dir</name>
- <value>hdfs://ranger-hadoop:9000/hbase</value>
+ <name>hbase.zookeeper.quorum</name>
+ <value>ranger-zk.example.com</value>
</property>
</configuration>
diff --git a/dev-support/ranger-docker/scripts/hive-site.xml b/dev-support/ranger-docker/scripts/hive-site.xml
index 69386eb..55343a3 100644
--- a/dev-support/ranger-docker/scripts/hive-site.xml
+++ b/dev-support/ranger-docker/scripts/hive-site.xml
@@ -41,10 +41,9 @@
<value>false</value>
</property>
- <!-- NOTE: using zookeeper from ranger-hbase! -->
<property>
<name>hive.zookeeper.quorum</name>
- <value>ranger-hbase.example.com</value>
+ <value>ranger-zk.example.com</value>
</property>
<property>
<name>hive.zookeeper.client.port</name>
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
index 8e3ee28..98eb51b 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
@@ -48,5 +48,7 @@ then
su -c "${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh" hdfs
fi
+NAMENODE_PID=`ps -ef | grep -v grep | grep -i "org.apache.hadoop.hdfs.server.namenode.NameNode" | awk '{print $2}'`
+
# prevent the container from exiting
-/bin/bash
+tail --pid=$NAMENODE_PID -f /dev/null
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase.sh b/dev-support/ranger-docker/scripts/ranger-hbase.sh
index 7bca8f7..2092b24 100755
--- a/dev-support/ranger-docker/scripts/ranger-hbase.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hbase.sh
@@ -33,5 +33,7 @@ fi
su -c "${HBASE_HOME}/bin/start-hbase.sh" hbase
+HBASE_MASTER_PID=`ps -ef | grep -v grep | grep -i "org.apache.hadoop.hbase.master.HMaster" | awk '{print $2}'`
+
# prevent the container from exiting
-/bin/bash
+tail --pid=$HBASE_MASTER_PID -f /dev/null
diff --git a/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties
index b3e403c..0cca7c2 100644
--- a/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties
@@ -19,7 +19,7 @@ COMPONENT_INSTALL_DIR_NAME=/opt/hive
UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
CUSTOM_USER=hive
-CUSTOM_GROUP=hive
+CUSTOM_GROUP=hadoop
XAAUDIT.SOLR.IS_ENABLED=true
XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
diff --git a/dev-support/ranger-docker/scripts/ranger-hive.sh b/dev-support/ranger-docker/scripts/ranger-hive.sh
index 7057c28..0602ce2 100755
--- a/dev-support/ranger-docker/scripts/ranger-hive.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hive.sh
@@ -37,5 +37,7 @@ fi
su -c "${HIVE_HOME}/bin/hiveserver2" hive
+HIVESERVER2_PID=`ps -ef | grep -v grep | grep -i "org.apache.hive.service.server.HiveServer2" | awk '{print $2}'`
+
# prevent the container from exiting
-/bin/bash
+tail --pid=$HIVESERVER2_PID -f /dev/null
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py b/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
index 99a953c..9f6cd79 100644
--- a/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
@@ -3,6 +3,6 @@ from apache_ranger.client.ranger_client import RangerClient
ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!'))
-service = RangerService({'name': 'dev_kafka', 'type': 'kafka', 'configs': {'username':'kafka', 'password':'kafka', 'zookeeper.connect': 'ranger-kafka:2181'}})
+service = RangerService({'name': 'dev_kafka', 'type': 'kafka', 'configs': {'username':'kafka', 'password':'kafka', 'zookeeper.connect': 'ranger-zk.example.com:2181'}})
ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh b/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
index 77cf465..c6edce6 100755
--- a/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-setup.sh
@@ -27,4 +27,6 @@ chown -R kafka:hadoop /opt/kafka/
cd ${RANGER_HOME}/ranger-kafka-plugin
./enable-kafka-plugin.sh
+sed -i 's/localhost:2181/ranger-zk.example.com:2181/' ${KAFKA_HOME}/config/server.properties
+
echo "authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" >> ${KAFKA_HOME}/config/server.properties
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka.sh b/dev-support/ranger-docker/scripts/ranger-kafka.sh
index e5145f8..8be501c 100755
--- a/dev-support/ranger-docker/scripts/ranger-kafka.sh
+++ b/dev-support/ranger-docker/scripts/ranger-kafka.sh
@@ -31,9 +31,4 @@ then
touch ${KAFKA_HOME}/.setupDone
fi
-su -c "cd ${KAFKA_HOME} && ./bin/zookeeper-server-start.sh config/zookeeper.properties &" kafka
-sleep 30
-su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config ./bin/kafka-server-start.sh config/server.properties &" kafka
-
-# prevent the container from exiting
-/bin/bash
+su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config ./bin/kafka-server-start.sh config/server.properties" kafka
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh
index 0b62344..04ac7cb 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger.sh
@@ -45,5 +45,7 @@ then
python3 ${RANGER_SCRIPTS}/ranger-kafka-service-dev_kafka.py
fi
+RANGER_ADMIN_PID=`ps -ef | grep -v grep | grep -i "org.apache.ranger.server.tomcat.EmbeddedServer" | awk '{print $2}'`
+
# prevent the container from exiting
-/bin/bash
+tail --pid=$RANGER_ADMIN_PID -f /dev/null