You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/08/27 22:02:29 UTC
[ranger] branch master updated: RANGER-2971: Docker setup to run
Ranger enabled HBase (standalone)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new f5bcb82 RANGER-2971: Docker setup to run Ranger enabled HBase (standalone)
f5bcb82 is described below
commit f5bcb8232ef27cd5b3dfb3fc5a55a89787537347
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Wed Aug 26 19:24:05 2020 -0700
RANGER-2971: Docker setup to run Ranger enabled HBase (standalone)
---
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/Dockerfile.ranger | 52 +++++---------
.../{Dockerfile.ranger => Dockerfile.ranger-base} | 32 +++++----
dev-support/ranger-docker/Dockerfile.ranger-build | 26 ++-----
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 41 +----------
dev-support/ranger-docker/Dockerfile.ranger-hbase | 38 +++++++++++
dev-support/ranger-docker/README.md | 47 +++++++------
.../ranger-docker/docker-compose.ranger-base.yml | 12 ++++
.../ranger-docker/docker-compose.ranger-build.yml | 2 +
.../ranger-docker/docker-compose.ranger-hbase.yml | 22 ++++++
.../ranger-docker/docker-compose.ranger.yml | 1 +
dev-support/ranger-docker/scripts/ranger-hadoop.sh | 14 ----
.../scripts/ranger-hbase-plugin-install.properties | 79 ++++++++++++++++++++++
.../scripts/ranger-hbase-service-dev_hbase.py | 8 +++
.../ranger-docker/scripts/ranger-hbase-setup.sh | 30 ++++++++
.../scripts/{ranger-hadoop.sh => ranger-hbase.sh} | 31 ++-------
dev-support/ranger-docker/scripts/ranger.sh | 23 ++-----
17 files changed, 279 insertions(+), 180 deletions(-)
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index 5a236e9..3ffb780 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -4,4 +4,5 @@
!dist/ranger-*-admin.tar.gz
!dist/ranger-*-hdfs-plugin.tar.gz
!dist/ranger-*-hive-plugin.tar.gz
+!dist/ranger-*-hbase-plugin.tar.gz
!scripts/*
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
index 2711b35..fca32ae 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -14,43 +14,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:20.04
+FROM ranger-base:latest
-ENV RANGER_VERSION 3.0.0-SNAPSHOT
-# Install curl, wget, tzdata, Python, Java, python-requests
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
- pip3 install requests && \
- pip3 install apache-ranger && \
- pip install requests
+COPY ./dist/version ${RANGER_DIST}/
+COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-admin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-hdfs-service-dev_hdfs.py ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-hive-service-dev_hive.py ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-hbase-service-dev_hbase.py ${RANGER_SCRIPTS}/
-# Set environment variables
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-ENV RANGER_DIST /home/ranger/dist
-ENV RANGER_SCRIPTS /home/ranger/scripts
-ENV RANGER_HOME /opt/ranger
-
-# setup ranger group, and users
-RUN groupadd ranger && \
- useradd -g ranger -ms /bin/bash ranger && \
- useradd -g ranger -ms /bin/bash rangeradmin && \
- useradd -g ranger -ms /bin/bash rangerusersync && \
- useradd -g ranger -ms /bin/bash rangertagsync && \
- useradd -g ranger -ms /bin/bash rangerkms && \
- mkdir -p /home/ranger/dist && \
- mkdir -p /home/ranger/scripts && \
- mkdir -p /opt/ranger && \
- chown -R ranger:ranger /opt/ranger
-
-COPY ./dist/version /home/ranger/dist/
-COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
-COPY ./scripts/ranger.sh /home/ranger/scripts/
-COPY ./scripts/ranger-admin-install.properties /home/ranger/scripts/
-COPY ./scripts/ranger-hdfs-service-dev_hdfs.py /home/ranger/scripts/
-COPY ./scripts/ranger-hive-service-dev_hive.py /home/ranger/scripts/
+COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /tmp/
+RUN tar xvfz /tmp/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} && \
+ ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin && \
+ rm -f /tmp/ranger-${RANGER_VERSION}-admin.tar.gz && \
+ cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties ${RANGER_HOME}/admin/install.properties && \
+ mkdir -p /var/run/ranger && \
+ mkdir -p /var/log/ranger && \
+ chown -R ranger:ranger ${RANGER_HOME}/admin/ /var/run/ranger/ /var/log/ranger/ && \
+ mkdir -p /usr/share/java/ && \
+ wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar" -O /usr/share/java/postgresql.jar
ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger-base
similarity index 76%
copy from dev-support/ranger-docker/Dockerfile.ranger
copy to dev-support/ranger-docker/Dockerfile.ranger-base
index 2711b35..fff84eb 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -16,41 +16,45 @@
FROM ubuntu:20.04
+
ENV RANGER_VERSION 3.0.0-SNAPSHOT
+ENV HADOOP_VERSION 3.1.1
+ENV HIVE_VERSION 3.1.2
+ENV HBASE_VERSION 2.0.3
# Install curl, wget, tzdata, Python, Java, python-requests
RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping && \
+ DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
+ python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
python2 /tmp/get-pip.py && \
- pip3 install requests && \
pip3 install apache-ranger && \
+ pip3 install requests && \
pip install requests
# Set environment variables
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV RANGER_DIST /home/ranger/dist
ENV RANGER_SCRIPTS /home/ranger/scripts
ENV RANGER_HOME /opt/ranger
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
-# setup ranger group, and users
+# setup groups, users, directories
RUN groupadd ranger && \
useradd -g ranger -ms /bin/bash ranger && \
useradd -g ranger -ms /bin/bash rangeradmin && \
useradd -g ranger -ms /bin/bash rangerusersync && \
useradd -g ranger -ms /bin/bash rangertagsync && \
useradd -g ranger -ms /bin/bash rangerkms && \
+ groupadd hadoop && \
+ useradd -g hadoop -ms /bin/bash hdfs && \
+ useradd -g hadoop -ms /bin/bash hive && \
+ useradd -g hadoop -ms /bin/bash hbase && \
mkdir -p /home/ranger/dist && \
mkdir -p /home/ranger/scripts && \
- mkdir -p /opt/ranger && \
- chown -R ranger:ranger /opt/ranger
-
-COPY ./dist/version /home/ranger/dist/
-COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
-COPY ./scripts/ranger.sh /home/ranger/scripts/
-COPY ./scripts/ranger-admin-install.properties /home/ranger/scripts/
-COPY ./scripts/ranger-hdfs-service-dev_hdfs.py /home/ranger/scripts/
-COPY ./scripts/ranger-hive-service-dev_hive.py /home/ranger/scripts/
+ chown -R ranger:ranger /home/ranger && \
+ mkdir -p /opt/ranger && \
+ chown -R ranger:ranger /opt/ranger
-ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ]
+ENTRYPOINT [ "/bin/bash" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-build b/dev-support/ranger-docker/Dockerfile.ranger-build
index c5a11a0..56c6d5d 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-build
+++ b/dev-support/ranger-docker/Dockerfile.ranger-build
@@ -14,30 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:20.04
+FROM ranger-base:latest
-# Install curl, wget, tzdata, Python, Java, python-requests
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
- python python3 python3-pip openjdk-8-jdk bc iputils-ping git maven build-essential && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
- pip3 install requests && \
- pip install requests
+
+# Install necessary packages to build Ranger
+RUN apt-get update && apt-get -y install git maven build-essential
# Set environment variables
-ENV MAVEN_HOME /usr/share/maven
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-ENV RANGER_DIST /home/ranger/dist
-ENV RANGER_SCRIPTS /home/ranger/scripts
+ENV MAVEN_HOME /usr/share/maven
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/apache-maven/bin
# setup ranger group, and users
-RUN groupadd ranger && \
- useradd -g ranger -ms /bin/bash ranger && \
- mkdir -p /home/ranger/dist && \
- mkdir -p /home/ranger/scripts && \
- mkdir -p /home/ranger/git && \
+RUN mkdir -p /home/ranger/git && \
mkdir -p /home/ranger/.m2 && \
chown -R ranger:ranger /home/ranger
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
index 27af2b7..d6046af 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -14,26 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:20.04
-
-ENV RANGER_VERSION 3.0.0-SNAPSHOT
-ENV HADOOP_VERSION 3.1.1
-ENV HIVE_VERSION 3.1.2
-
-# Install curl, wget, tzdata, Python, Java, python-requests
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install vim sudo curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
- pip3 install requests && \
- pip install requests
-
-RUN groupadd hadoop && \
- useradd -g hadoop -ms /bin/bash hdfs && \
- useradd -g hadoop -ms /bin/bash hive && \
- mkdir -p /opt/ranger && \
- mkdir -p /home/ranger/dist && \
- mkdir -p /home/ranger/scripts
+FROM ranger-base:latest
COPY ./dist/version /home/ranger/dist/
@@ -51,30 +32,12 @@ RUN curl https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz && \
cp -f /home/ranger/scripts/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV RANGER_DIST /home/ranger/dist
-ENV RANGER_SCRIPTS /home/ranger/scripts
-ENV RANGER_HOME /opt/ranger
-
ENV HADOOP_HOME /opt/hadoop
ENV HADOOP_CONF_DIR /opt/hadoop/etc/hadoop
ENV HADOOP_HDFS_HOME /opt/hadoop
ENV HADOOP_MAPRED_HOME /opt/hadoop
ENV HADOOP_COMMON_HOME /opt/hadoop
ENV YARN_HOME /opt/hadoop
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin
-
-# COPY ./dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
-#
-# RUN curl https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz --output /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz &&
-# tar xvfz /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
-# ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
-# rm -f /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
-# tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
-# ln -s /opt/ranger/ranger-${RANGER_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
-# rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz
-# ENV HIVE_HOME /opt/hive
-# ENV HIVE_CONF_DIR /opt/hive/conf
-# ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin:/opt/hive/bin
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin
ENTRYPOINT [ "/home/ranger/scripts/ranger-hadoop.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase
new file mode 100644
index 0000000..a995250
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ranger-base:latest
+
+
+COPY ./dist/version /home/ranger/dist/
+COPY ./dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz /home/ranger/dist/
+COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/
+
+RUN curl https://archive.apache.org/dist/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz --output /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
+ tar xvfz /tmp/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
+ ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \
+ rm -f /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
+ tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz --directory=/opt/ranger && \
+ ln -s /opt/ranger/ranger-${RANGER_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \
+ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz && \
+ cp -f /home/ranger/scripts/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties
+
+ENV HBASE_HOME /opt/hbase
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hbase/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-hbase.sh" ]
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 42c866a..0fad420 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -34,55 +34,64 @@ deploy Apache Ranger and its dependent services in containers.
3. Using docker-compose is the simpler way to build and deploy Apache Ranger
in containers.
3.1. Execute following command to build Apache Ranger:
- docker-compose -f docker-compose.ranger-build.yml up
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up --remove-orphans
Time taken to complete the build might vary (upto an hour), depending on
status of ${HOME}/.m2 directory cache.
- 3.2. Execute following command to start Ranger and dependent services in containers:
- docker-compose -f docker-compose.ranger.yml up -d
-
- 3.2. Execute following command to start Ranger enabled Hadoop services (only HDFS for now) in a continer:
- docker-compose -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -d
-
+ 3.2. Execute following command to start Ranger, Ranger enabled HDFS, Ranger enabled HBase, and dependeny services (Solr, DB) in continers:
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -f docker-compose.ranger-hbase.yml -d
4. Alternatively docker command can be used to build and deploy Apache Ranger.
- 4.1. Execute following command to build Docker image **ranger-build**:
- docker build -f Dockerfile.ranger-build -t ranger-build .
+ 4.1. Execute following command to build Docker image **ranger-base**:
+ docker build -f Dockerfile.ranger-base -t ranger-base .
This might take about 10 minutes to complete.
- 4.2. Build Apache Ranger in a container with the following commands:
- docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2 -v $(pwd)/dist:/home/ranger/dist -e BRANCH=master -e PROFILE=all -e SKIPTESTS=true ranger-build
+ 4.2. Execute following command to build Docker image **ranger-build**:
+ docker build -f Dockerfile.ranger-build -t ranger-build .
+
+ 4.3. Build Apache Ranger in a container with the following command:
+ docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2 -v $(pwd)/dist:/home/ranger/dist -e BRANCH=ranger-2.1 -e PROFILE=all -e SKIPTESTS=true ranger-build
Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
- 4.3. Execute following command to build Docker image **ranger**:
+ 4.4. Execute following command to build Docker image **ranger**:
docker build -f Dockerfile.ranger -t ranger .
This might take about 10 minutes to complete.
- 4.4. Execute following command to build a Docker image **ranger-solr**:
+ 4.5. Execute following command to build a Docker image **ranger-solr**:
docker build -f Dockerfile.ranger-solr -t ranger-solr .
- 4.5. Execute following command to start a container that runs database for use by Ranger Admin:
+ 4.6. Execute following command to start a container that runs database for use by Ranger Admin:
docker run --name ranger-db --hostname ranger-db.example.com -e POSTGRES_PASSWORD='rangerR0cks!' -d postgres:12
- 4.6. Execute following command to start a container that runs Solr for use by Ranger Admin:
+ 4.7. Execute following command to start a container that runs Solr for use by Ranger Admin:
docker run --name ranger-solr --hostname ranger-solr.example.com -p 8983:8983 -d ranger-solr solr-precreate ranger_audits /opt/solr/server/solr/configsets/ranger_audits/
- 4.7. Execute following command to install and run Ranger services in a container:
+ 4.8. Execute following command to install and run Ranger services in a container:
docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr ranger
This might take few minutes to complete.
- 4.8. Execute following command to build Docker image **ranger-hadoop**:
+ 4.9. Execute following command to build Docker image **ranger-hadoop**:
docker build -f Dockerfile.ranger-hadoop -t ranger-hadoop .
This steps includes downloading of Hadoop tar balls, and can take a while to complete.
- 4.9. Execute following command to install and run Ranger enabled Hadoop services (only HDFS for now) in a container:
- docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
+ 4.10. Execute following command to install and run Ranger enabled HDFS in a container:
+ docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
+
+ This might take few minutes to complete.
+
+ 4.11. Execute following command to build Docker image **ranger-hbase**:
+ docker build -f Dockerfile.ranger-hbase -t ranger-hbase .
+
+ This steps includes downloading of HBase tar ball, and can take a while to complete.
+
+ 4.12. Execute following command to install and run Ranger enabled HBase in a container:
+ docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-hbase
This might take few minutes to complete.
diff --git a/dev-support/ranger-docker/docker-compose.ranger-base.yml b/dev-support/ranger-docker/docker-compose.ranger-base.yml
new file mode 100644
index 0000000..18e78db
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-base.yml
@@ -0,0 +1,12 @@
+version: '3'
+services:
+ ranger-base:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-base
+ image: ranger-base
+ networks:
+ - ranger
+
+networks:
+ ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger-build.yml b/dev-support/ranger-docker/docker-compose.ranger-build.yml
index c17361a..9572be1 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-build.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-build.yml
@@ -12,6 +12,8 @@ services:
volumes:
- ~/.m2:/home/ranger/.m2
- ./dist:/home/ranger/dist
+ depends_on:
+ - ranger-base
environment:
BRANCH: 'master'
SKIPTESTS: 'true'
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
new file mode 100644
index 0000000..81804aa
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
@@ -0,0 +1,22 @@
+version: '3'
+services:
+ ranger-hbase:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-hbase
+ image: ranger-hbase
+ container_name: ranger-hbase
+ hostname: ranger-hbase.example.com
+ stdin_open: true
+ tty: true
+ networks:
+ - ranger
+ ports:
+ - "2181:16181"
+ - "16010:16010"
+ - "16020:16020"
+ depends_on:
+ - ranger-hadoop
+
+networks:
+ ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml
index 2c6a3c9..787c2ef 100644
--- a/dev-support/ranger-docker/docker-compose.ranger.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger.yml
@@ -14,6 +14,7 @@ services:
ports:
- "6080:6080"
depends_on:
+ - ranger-base
- ranger-db
- ranger-solr
command:
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
index 9d7ebf0..8dc5cd4 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
@@ -36,19 +36,5 @@ fi
su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs
-# if [ ! -e ${HIVE_HOME}/.setupDone ]
-# then
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /user/hive/warehouse" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /user/hive/warehouse" hdfs
-#
-# su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive
-#
-# touch ${HIVE_HOME}/.setupDone
-# fi
-#
-# su -c "${HIVE_HOME}/bin/hiveserver2" hive
-
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
new file mode 100644
index 0000000..bc80a6d
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+POLICY_MGR_URL=http://ranger:6080
+REPOSITORY_NAME=dev_hbase
+COMPONENT_INSTALL_DIR_NAME=/opt/hbase
+
+CUSTOM_USER=hbase
+CUSTOM_GROUP=hadoop
+
+XAAUDIT.SUMMARY.ENABLE=true
+UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
+
+XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
+
+# Following properties are needed to get past installation script! Please don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hadoop
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hadoop/hbase/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hadoop/hbase/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/solr/spool
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=false
+XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
new file mode 100644
index 0000000..9294bf5
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
@@ -0,0 +1,8 @@
+from apache_ranger.model.ranger_service import RangerService
+from apache_ranger.client.ranger_client import RangerClient
+
+ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+
+service = RangerService(name='dev_hbase', type='hbase', configs={'username':'hbase', 'password':'hbase', 'hadoop.security.authentication': 'simple', 'hbase.security.authentication': 'simple', 'hadoop.security.authorization': 'true', 'hbase.zookeeper.property.clientPort': '16181', 'hbase.zookeeper.quorum': 'ranger-hbase', 'zookeeper.znode.parent': '/hbase'})
+
+ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
new file mode 100755
index 0000000..bc6d226
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "export JAVA_HOME=${JAVA_HOME}" >> ${HBASE_HOME}/conf/hbase-env.sh
+
+cat <<EOF > /etc/ssh/ssh_config
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile=/dev/null
+EOF
+
+chown -R hbase:hadoop /opt/hbase/
+
+cd ${RANGER_HOME}/ranger-hbase-plugin
+./enable-hbase-plugin.sh
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hbase.sh
similarity index 50%
copy from dev-support/ranger-docker/scripts/ranger-hadoop.sh
copy to dev-support/ranger-docker/scripts/ranger-hbase.sh
index 9d7ebf0..7bca8f7 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hbase.sh
@@ -18,37 +18,20 @@
service ssh start
-if [ ! -e ${HADOOP_HOME}/.setupDone ]
+if [ ! -e ${HBASE_HOME}/.setupDone ]
then
- su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs
- su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hdfs
- su -c "chmod 0600 ~/.ssh/authorized_keys" hdfs
+ su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hbase
+ su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hbase
+ su -c "chmod 0600 ~/.ssh/authorized_keys" hbase
echo "ssh" > /etc/pdsh/rcmd_default
- ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh
+ ${RANGER_SCRIPTS}/ranger-hbase-setup.sh
- su -c "${HADOOP_HOME}/bin/hdfs namenode -format" hdfs
-
- touch ${HADOOP_HOME}/.setupDone
+ touch ${HBASE_HOME}/.setupDone
fi
-su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
-su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs
-
-# if [ ! -e ${HIVE_HOME}/.setupDone ]
-# then
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /user/hive/warehouse" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /user/hive/warehouse" hdfs
-#
-# su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive
-#
-# touch ${HIVE_HOME}/.setupDone
-# fi
-#
-# su -c "${HIVE_HOME}/bin/hiveserver2" hive
+su -c "${HBASE_HOME}/bin/start-hbase.sh" hbase
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh
index bf61968..3076556 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger.sh
@@ -16,32 +16,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export RANGER_VERSION=`cat ${RANGER_DIST}/version`
-
-if [ -e ${RANGER_HOME}/admin ]
+if [ ! -e ${RANGER_HOME}/.setupDone ]
then
- SETUP_RANGER=false
-else
SETUP_RANGER=true
+else
+ SETUP_RANGER=false
fi
if [ "${SETUP_RANGER}" == "true" ]
then
- # Download PostgreSQL JDBC library
- wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar" -O /usr/share/java/postgresql.jar
-
- cd ${RANGER_HOME}
- tar xvfz ${RANGER_DIST}/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME}
- ln -s ranger-${RANGER_VERSION}-admin admin
- cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties admin/install.properties
+ su -c "cd ${RANGER_HOME}/admin && ./setup.sh" ranger
- cd ${RANGER_HOME}/admin
- ./setup.sh
+ touch ${RANGER_HOME}/.setupDone
fi
-cd ${RANGER_HOME}/admin
-./ews/ranger-admin-services.sh start
+su -c "cd ${RANGER_HOME}/admin && ./ews/ranger-admin-services.sh start" ranger
if [ "${SETUP_RANGER}" == "true" ]
then
@@ -50,6 +40,7 @@ then
python3 ${RANGER_SCRIPTS}/ranger-hdfs-service-dev_hdfs.py
python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py
+ python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py
fi
# prevent the container from exiting