You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/08/27 22:13:02 UTC
[ranger] branch ranger-2.1 updated (ab86a4c -> 5a16f9a)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a change to branch ranger-2.1
in repository https://gitbox.apache.org/repos/asf/ranger.git.
from ab86a4c RANGER-2966 : Upgrade POI to 4.1.2
new 84e0bbd RANGER-2973: NPE check in audit generation in RangerHDFSPlugin to avoid possible issues
new 5a16f9a RANGER-2971: Docker setup to run Ranger enabled HBase (standalone)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/Dockerfile.ranger | 52 +++++++---------------
.../{Dockerfile.ranger => Dockerfile.ranger-base} | 32 +++++++------
dev-support/ranger-docker/Dockerfile.ranger-build | 26 +++--------
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 41 +----------------
dev-support/ranger-docker/Dockerfile.ranger-hbase | 38 ++++++++++++++++
dev-support/ranger-docker/README.md | 45 +++++++++++--------
.../ranger-docker/docker-compose.ranger-base.yml | 12 +++++
.../ranger-docker/docker-compose.ranger-build.yml | 2 +
.../ranger-docker/docker-compose.ranger-hbase.yml | 22 +++++++++
.../ranger-docker/docker-compose.ranger.yml | 1 +
dev-support/ranger-docker/scripts/ranger-hadoop.sh | 14 ------
...ties => ranger-hbase-plugin-install.properties} | 17 ++++---
.../scripts/ranger-hbase-service-dev_hbase.py | 8 ++++
.../ranger-hbase-setup.sh} | 19 +++++---
.../ranger-hbase.sh} | 26 ++++++++---
dev-support/ranger-docker/scripts/ranger.sh | 23 +++-------
.../authorization/hadoop/RangerHdfsAuthorizer.java | 44 +++++++++---------
18 files changed, 226 insertions(+), 197 deletions(-)
copy dev-support/ranger-docker/{Dockerfile.ranger => Dockerfile.ranger-base} (76%)
create mode 100644 dev-support/ranger-docker/Dockerfile.ranger-hbase
create mode 100644 dev-support/ranger-docker/docker-compose.ranger-base.yml
create mode 100644 dev-support/ranger-docker/docker-compose.ranger-hbase.yml
copy dev-support/ranger-docker/scripts/{ranger-hdfs-plugin-install.properties => ranger-hbase-plugin-install.properties} (86%)
create mode 100644 dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
copy dev-support/ranger-docker/{Dockerfile.ranger-solr => scripts/ranger-hbase-setup.sh} (73%)
mode change 100644 => 100755
copy dev-support/ranger-docker/{Dockerfile.ranger-solr => scripts/ranger-hbase.sh} (63%)
mode change 100644 => 100755
[ranger] 02/02: RANGER-2971: Docker setup to run Ranger enabled
HBase (standalone)
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.1
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 5a16f9afffc0ba5cc3f9e16e23af8e67090112d4
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Wed Aug 26 19:24:05 2020 -0700
RANGER-2971: Docker setup to run Ranger enabled HBase (standalone)
(cherry picked from commit f5bcb8232ef27cd5b3dfb3fc5a55a89787537347)
---
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/Dockerfile.ranger | 52 +++++---------
.../{Dockerfile.ranger => Dockerfile.ranger-base} | 32 +++++----
dev-support/ranger-docker/Dockerfile.ranger-build | 26 ++-----
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 41 +----------
dev-support/ranger-docker/Dockerfile.ranger-hbase | 38 +++++++++++
dev-support/ranger-docker/README.md | 45 +++++++-----
.../ranger-docker/docker-compose.ranger-base.yml | 12 ++++
.../ranger-docker/docker-compose.ranger-build.yml | 2 +
.../ranger-docker/docker-compose.ranger-hbase.yml | 22 ++++++
.../ranger-docker/docker-compose.ranger.yml | 1 +
dev-support/ranger-docker/scripts/ranger-hadoop.sh | 14 ----
.../scripts/ranger-hbase-plugin-install.properties | 79 ++++++++++++++++++++++
.../scripts/ranger-hbase-service-dev_hbase.py | 8 +++
.../ranger-docker/scripts/ranger-hbase-setup.sh | 30 ++++++++
.../scripts/{ranger-hadoop.sh => ranger-hbase.sh} | 31 ++-------
dev-support/ranger-docker/scripts/ranger.sh | 23 ++-----
17 files changed, 278 insertions(+), 179 deletions(-)
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index 5a236e9..3ffb780 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -4,4 +4,5 @@
!dist/ranger-*-admin.tar.gz
!dist/ranger-*-hdfs-plugin.tar.gz
!dist/ranger-*-hive-plugin.tar.gz
+!dist/ranger-*-hbase-plugin.tar.gz
!scripts/*
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
index d57e384..fca32ae 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -14,43 +14,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:20.04
+FROM ranger-base:latest
-ENV RANGER_VERSION 2.1.0
-# Install curl, wget, tzdata, Python, Java, python-requests
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
- pip3 install requests && \
- pip3 install apache-ranger && \
- pip install requests
+COPY ./dist/version ${RANGER_DIST}/
+COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-admin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-hdfs-service-dev_hdfs.py ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-hive-service-dev_hive.py ${RANGER_SCRIPTS}/
+COPY ./scripts/ranger-hbase-service-dev_hbase.py ${RANGER_SCRIPTS}/
-# Set environment variables
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-ENV RANGER_DIST /home/ranger/dist
-ENV RANGER_SCRIPTS /home/ranger/scripts
-ENV RANGER_HOME /opt/ranger
-
-# setup ranger group, and users
-RUN groupadd ranger && \
- useradd -g ranger -ms /bin/bash ranger && \
- useradd -g ranger -ms /bin/bash rangeradmin && \
- useradd -g ranger -ms /bin/bash rangerusersync && \
- useradd -g ranger -ms /bin/bash rangertagsync && \
- useradd -g ranger -ms /bin/bash rangerkms && \
- mkdir -p /home/ranger/dist && \
- mkdir -p /home/ranger/scripts && \
- mkdir -p /opt/ranger && \
- chown -R ranger:ranger /opt/ranger
-
-COPY ./dist/version /home/ranger/dist/
-COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
-COPY ./scripts/ranger.sh /home/ranger/scripts/
-COPY ./scripts/ranger-admin-install.properties /home/ranger/scripts/
-COPY ./scripts/ranger-hdfs-service-dev_hdfs.py /home/ranger/scripts/
-COPY ./scripts/ranger-hive-service-dev_hive.py /home/ranger/scripts/
+COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /tmp/
+RUN tar xvfz /tmp/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} && \
+ ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin && \
+ rm -f /tmp/ranger-${RANGER_VERSION}-admin.tar.gz && \
+ cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties ${RANGER_HOME}/admin/install.properties && \
+ mkdir -p /var/run/ranger && \
+ mkdir -p /var/log/ranger && \
+ chown -R ranger:ranger ${RANGER_HOME}/admin/ /var/run/ranger/ /var/log/ranger/ && \
+ mkdir -p /usr/share/java/ && \
+ wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar" -O /usr/share/java/postgresql.jar
ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger-base
similarity index 76%
copy from dev-support/ranger-docker/Dockerfile.ranger
copy to dev-support/ranger-docker/Dockerfile.ranger-base
index d57e384..f461f74 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -16,41 +16,45 @@
FROM ubuntu:20.04
+
ENV RANGER_VERSION 2.1.0
+ENV HADOOP_VERSION 3.1.1
+ENV HIVE_VERSION 3.1.2
+ENV HBASE_VERSION 2.0.3
# Install curl, wget, tzdata, Python, Java, python-requests
RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping && \
+ DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
+ python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
python2 /tmp/get-pip.py && \
- pip3 install requests && \
pip3 install apache-ranger && \
+ pip3 install requests && \
pip install requests
# Set environment variables
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV RANGER_DIST /home/ranger/dist
ENV RANGER_SCRIPTS /home/ranger/scripts
ENV RANGER_HOME /opt/ranger
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
-# setup ranger group, and users
+# setup groups, users, directories
RUN groupadd ranger && \
useradd -g ranger -ms /bin/bash ranger && \
useradd -g ranger -ms /bin/bash rangeradmin && \
useradd -g ranger -ms /bin/bash rangerusersync && \
useradd -g ranger -ms /bin/bash rangertagsync && \
useradd -g ranger -ms /bin/bash rangerkms && \
+ groupadd hadoop && \
+ useradd -g hadoop -ms /bin/bash hdfs && \
+ useradd -g hadoop -ms /bin/bash hive && \
+ useradd -g hadoop -ms /bin/bash hbase && \
mkdir -p /home/ranger/dist && \
mkdir -p /home/ranger/scripts && \
- mkdir -p /opt/ranger && \
- chown -R ranger:ranger /opt/ranger
-
-COPY ./dist/version /home/ranger/dist/
-COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
-COPY ./scripts/ranger.sh /home/ranger/scripts/
-COPY ./scripts/ranger-admin-install.properties /home/ranger/scripts/
-COPY ./scripts/ranger-hdfs-service-dev_hdfs.py /home/ranger/scripts/
-COPY ./scripts/ranger-hive-service-dev_hive.py /home/ranger/scripts/
+ chown -R ranger:ranger /home/ranger && \
+ mkdir -p /opt/ranger && \
+ chown -R ranger:ranger /opt/ranger
-ENTRYPOINT [ "/home/ranger/scripts/ranger.sh" ]
+ENTRYPOINT [ "/bin/bash" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-build b/dev-support/ranger-docker/Dockerfile.ranger-build
index c5a11a0..56c6d5d 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-build
+++ b/dev-support/ranger-docker/Dockerfile.ranger-build
@@ -14,30 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:20.04
+FROM ranger-base:latest
-# Install curl, wget, tzdata, Python, Java, python-requests
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
- python python3 python3-pip openjdk-8-jdk bc iputils-ping git maven build-essential && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
- pip3 install requests && \
- pip install requests
+
+# Install necessary packages to build Ranger
+RUN apt-get update && apt-get -y install git maven build-essential
# Set environment variables
-ENV MAVEN_HOME /usr/share/maven
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-ENV RANGER_DIST /home/ranger/dist
-ENV RANGER_SCRIPTS /home/ranger/scripts
+ENV MAVEN_HOME /usr/share/maven
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/apache-maven/bin
# setup ranger group, and users
-RUN groupadd ranger && \
- useradd -g ranger -ms /bin/bash ranger && \
- mkdir -p /home/ranger/dist && \
- mkdir -p /home/ranger/scripts && \
- mkdir -p /home/ranger/git && \
+RUN mkdir -p /home/ranger/git && \
mkdir -p /home/ranger/.m2 && \
chown -R ranger:ranger /home/ranger
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
index e866ba7..d6046af 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -14,26 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:20.04
-
-ENV RANGER_VERSION 2.1.0
-ENV HADOOP_VERSION 3.1.1
-ENV HIVE_VERSION 3.1.2
-
-# Install curl, wget, tzdata, Python, Java, python-requests
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get -y install vim sudo curl wget tzdata python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
- pip3 install requests && \
- pip install requests
-
-RUN groupadd hadoop && \
- useradd -g hadoop -ms /bin/bash hdfs && \
- useradd -g hadoop -ms /bin/bash hive && \
- mkdir -p /opt/ranger && \
- mkdir -p /home/ranger/dist && \
- mkdir -p /home/ranger/scripts
+FROM ranger-base:latest
COPY ./dist/version /home/ranger/dist/
@@ -51,30 +32,12 @@ RUN curl https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz && \
cp -f /home/ranger/scripts/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV RANGER_DIST /home/ranger/dist
-ENV RANGER_SCRIPTS /home/ranger/scripts
-ENV RANGER_HOME /opt/ranger
-
ENV HADOOP_HOME /opt/hadoop
ENV HADOOP_CONF_DIR /opt/hadoop/etc/hadoop
ENV HADOOP_HDFS_HOME /opt/hadoop
ENV HADOOP_MAPRED_HOME /opt/hadoop
ENV HADOOP_COMMON_HOME /opt/hadoop
ENV YARN_HOME /opt/hadoop
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin
-
-# COPY ./dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
-#
-# RUN curl https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz --output /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz &&
-# tar xvfz /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
-# ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
-# rm -f /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
-# tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
-# ln -s /opt/ranger/ranger-${RANGER_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
-# rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz
-# ENV HIVE_HOME /opt/hive
-# ENV HIVE_CONF_DIR /opt/hive/conf
-# ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin:/opt/hive/bin
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin
ENTRYPOINT [ "/home/ranger/scripts/ranger-hadoop.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase
new file mode 100644
index 0000000..a995250
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ranger-base:latest
+
+
+COPY ./dist/version /home/ranger/dist/
+COPY ./dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz /home/ranger/dist/
+COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/
+
+RUN curl https://archive.apache.org/dist/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz --output /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
+ tar xvfz /tmp/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
+ ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \
+ rm -f /tmp/hbase-${HBASE_VERSION}-bin.tar.gz && \
+ tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz --directory=/opt/ranger && \
+ ln -s /opt/ranger/ranger-${RANGER_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \
+ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz && \
+ cp -f /home/ranger/scripts/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties
+
+ENV HBASE_HOME /opt/hbase
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hbase/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-hbase.sh" ]
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 938d8ac..0fad420 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -34,55 +34,64 @@ deploy Apache Ranger and its dependent services in containers.
3. Using docker-compose is the simpler way to build and deploy Apache Ranger
in containers.
3.1. Execute following command to build Apache Ranger:
- docker-compose -f docker-compose.ranger-build.yml up
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up --remove-orphans
Time taken to complete the build might vary (upto an hour), depending on
status of ${HOME}/.m2 directory cache.
- 3.2. Execute following command to start Ranger and dependent services in containers:
- docker-compose -f docker-compose.ranger.yml up -d
-
- 3.2. Execute following command to start Ranger enabled Hadoop services (only HDFS for now) in a continer:
- docker-compose -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -d
-
+ 3.2. Execute following command to start Ranger, Ranger enabled HDFS, Ranger enabled HBase, and dependeny services (Solr, DB) in continers:
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml up -f docker-compose.ranger-hbase.yml -d
4. Alternatively docker command can be used to build and deploy Apache Ranger.
- 4.1. Execute following command to build Docker image **ranger-build**:
- docker build -f Dockerfile.ranger-build -t ranger-build .
+ 4.1. Execute following command to build Docker image **ranger-base**:
+ docker build -f Dockerfile.ranger-base -t ranger-base .
This might take about 10 minutes to complete.
- 4.2. Build Apache Ranger in a container with the following command:
+ 4.2. Execute following command to build Docker image **ranger-build**:
+ docker build -f Dockerfile.ranger-build -t ranger-build .
+
+ 4.3. Build Apache Ranger in a container with the following command:
docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2 -v $(pwd)/dist:/home/ranger/dist -e BRANCH=ranger-2.1 -e PROFILE=all -e SKIPTESTS=true ranger-build
Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
- 4.3. Execute following command to build Docker image **ranger**:
+ 4.4. Execute following command to build Docker image **ranger**:
docker build -f Dockerfile.ranger -t ranger .
This might take about 10 minutes to complete.
- 4.4. Execute following command to build a Docker image **ranger-solr**:
+ 4.5. Execute following command to build a Docker image **ranger-solr**:
docker build -f Dockerfile.ranger-solr -t ranger-solr .
- 4.5. Execute following command to start a container that runs database for use by Ranger Admin:
+ 4.6. Execute following command to start a container that runs database for use by Ranger Admin:
docker run --name ranger-db --hostname ranger-db.example.com -e POSTGRES_PASSWORD='rangerR0cks!' -d postgres:12
- 4.6. Execute following command to start a container that runs Solr for use by Ranger Admin:
+ 4.7. Execute following command to start a container that runs Solr for use by Ranger Admin:
docker run --name ranger-solr --hostname ranger-solr.example.com -p 8983:8983 -d ranger-solr solr-precreate ranger_audits /opt/solr/server/solr/configsets/ranger_audits/
- 4.7. Execute following command to install and run Ranger services in a container:
+ 4.8. Execute following command to install and run Ranger services in a container:
docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr ranger
This might take few minutes to complete.
- 4.8. Execute following command to build Docker image **ranger-hadoop**:
+ 4.9. Execute following command to build Docker image **ranger-hadoop**:
docker build -f Dockerfile.ranger-hadoop -t ranger-hadoop .
This steps includes downloading of Hadoop tar balls, and can take a while to complete.
- 4.9. Execute following command to install and run Ranger enabled Hadoop services (only HDFS for now) in a container:
- docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
+ 4.10. Execute following command to install and run Ranger enabled HDFS in a container:
+ docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
+
+ This might take few minutes to complete.
+
+ 4.11. Execute following command to build Docker image **ranger-hbase**:
+ docker build -f Dockerfile.ranger-hbase -t ranger-hbase .
+
+ This steps includes downloading of HBase tar ball, and can take a while to complete.
+
+ 4.12. Execute following command to install and run Ranger enabled HBase in a container:
+ docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-hbase
This might take few minutes to complete.
diff --git a/dev-support/ranger-docker/docker-compose.ranger-base.yml b/dev-support/ranger-docker/docker-compose.ranger-base.yml
new file mode 100644
index 0000000..18e78db
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-base.yml
@@ -0,0 +1,12 @@
+version: '3'
+services:
+ ranger-base:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-base
+ image: ranger-base
+ networks:
+ - ranger
+
+networks:
+ ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger-build.yml b/dev-support/ranger-docker/docker-compose.ranger-build.yml
index a63b3dd..48decd5 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-build.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-build.yml
@@ -12,6 +12,8 @@ services:
volumes:
- ~/.m2:/home/ranger/.m2
- ./dist:/home/ranger/dist
+ depends_on:
+ - ranger-base
environment:
BRANCH: 'ranger-2.1'
SKIPTESTS: 'true'
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
new file mode 100644
index 0000000..81804aa
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
@@ -0,0 +1,22 @@
+version: '3'
+services:
+ ranger-hbase:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-hbase
+ image: ranger-hbase
+ container_name: ranger-hbase
+ hostname: ranger-hbase.example.com
+ stdin_open: true
+ tty: true
+ networks:
+ - ranger
+ ports:
+ - "2181:16181"
+ - "16010:16010"
+ - "16020:16020"
+ depends_on:
+ - ranger-hadoop
+
+networks:
+ ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml
index 2c6a3c9..787c2ef 100644
--- a/dev-support/ranger-docker/docker-compose.ranger.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger.yml
@@ -14,6 +14,7 @@ services:
ports:
- "6080:6080"
depends_on:
+ - ranger-base
- ranger-db
- ranger-solr
command:
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
index 9d7ebf0..8dc5cd4 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
@@ -36,19 +36,5 @@ fi
su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs
-# if [ ! -e ${HIVE_HOME}/.setupDone ]
-# then
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /user/hive/warehouse" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /user/hive/warehouse" hdfs
-#
-# su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive
-#
-# touch ${HIVE_HOME}/.setupDone
-# fi
-#
-# su -c "${HIVE_HOME}/bin/hiveserver2" hive
-
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
new file mode 100644
index 0000000..bc80a6d
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-plugin-install.properties
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+POLICY_MGR_URL=http://ranger:6080
+REPOSITORY_NAME=dev_hbase
+COMPONENT_INSTALL_DIR_NAME=/opt/hbase
+
+CUSTOM_USER=hbase
+CUSTOM_GROUP=hadoop
+
+XAAUDIT.SUMMARY.ENABLE=true
+UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
+
+XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
+
+# Following properties are needed to get past installation script! Please don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hadoop
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hadoop/hbase/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hadoop/hbase/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/solr/spool
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=false
+XAAUDIT.HDFS.HDFS_DIR=hdfs://localhost:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hbase/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
new file mode 100644
index 0000000..9294bf5
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
@@ -0,0 +1,8 @@
+from apache_ranger.model.ranger_service import RangerService
+from apache_ranger.client.ranger_client import RangerClient
+
+ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+
+service = RangerService(name='dev_hbase', type='hbase', configs={'username':'hbase', 'password':'hbase', 'hadoop.security.authentication': 'simple', 'hbase.security.authentication': 'simple', 'hadoop.security.authorization': 'true', 'hbase.zookeeper.property.clientPort': '16181', 'hbase.zookeeper.quorum': 'ranger-hbase', 'zookeeper.znode.parent': '/hbase'})
+
+ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
new file mode 100755
index 0000000..bc6d226
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "export JAVA_HOME=${JAVA_HOME}" >> ${HBASE_HOME}/conf/hbase-env.sh
+
+cat <<EOF > /etc/ssh/ssh_config
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile=/dev/null
+EOF
+
+chown -R hbase:hadoop /opt/hbase/
+
+cd ${RANGER_HOME}/ranger-hbase-plugin
+./enable-hbase-plugin.sh
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hbase.sh
similarity index 50%
copy from dev-support/ranger-docker/scripts/ranger-hadoop.sh
copy to dev-support/ranger-docker/scripts/ranger-hbase.sh
index 9d7ebf0..7bca8f7 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hbase.sh
@@ -18,37 +18,20 @@
service ssh start
-if [ ! -e ${HADOOP_HOME}/.setupDone ]
+if [ ! -e ${HBASE_HOME}/.setupDone ]
then
- su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs
- su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hdfs
- su -c "chmod 0600 ~/.ssh/authorized_keys" hdfs
+ su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hbase
+ su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hbase
+ su -c "chmod 0600 ~/.ssh/authorized_keys" hbase
echo "ssh" > /etc/pdsh/rcmd_default
- ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh
+ ${RANGER_SCRIPTS}/ranger-hbase-setup.sh
- su -c "${HADOOP_HOME}/bin/hdfs namenode -format" hdfs
-
- touch ${HADOOP_HOME}/.setupDone
+ touch ${HBASE_HOME}/.setupDone
fi
-su -c "${HADOOP_HOME}/sbin/start-dfs.sh" hdfs
-su -c "${HADOOP_HOME}/sbin/start-yarn.sh" hdfs
-
-# if [ ! -e ${HIVE_HOME}/.setupDone ]
-# then
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir /user/hive/warehouse" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /tmp" hdfs
-# su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod g+w /user/hive/warehouse" hdfs
-#
-# su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive
-#
-# touch ${HIVE_HOME}/.setupDone
-# fi
-#
-# su -c "${HIVE_HOME}/bin/hiveserver2" hive
+su -c "${HBASE_HOME}/bin/start-hbase.sh" hbase
# prevent the container from exiting
/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh
index bf61968..3076556 100755
--- a/dev-support/ranger-docker/scripts/ranger.sh
+++ b/dev-support/ranger-docker/scripts/ranger.sh
@@ -16,32 +16,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export RANGER_VERSION=`cat ${RANGER_DIST}/version`
-
-if [ -e ${RANGER_HOME}/admin ]
+if [ ! -e ${RANGER_HOME}/.setupDone ]
then
- SETUP_RANGER=false
-else
SETUP_RANGER=true
+else
+ SETUP_RANGER=false
fi
if [ "${SETUP_RANGER}" == "true" ]
then
- # Download PostgreSQL JDBC library
- wget "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7/postgresql-42.2.16.jre7.jar" -O /usr/share/java/postgresql.jar
-
- cd ${RANGER_HOME}
- tar xvfz ${RANGER_DIST}/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME}
- ln -s ranger-${RANGER_VERSION}-admin admin
- cp -f ${RANGER_SCRIPTS}/ranger-admin-install.properties admin/install.properties
+ su -c "cd ${RANGER_HOME}/admin && ./setup.sh" ranger
- cd ${RANGER_HOME}/admin
- ./setup.sh
+ touch ${RANGER_HOME}/.setupDone
fi
-cd ${RANGER_HOME}/admin
-./ews/ranger-admin-services.sh start
+su -c "cd ${RANGER_HOME}/admin && ./ews/ranger-admin-services.sh start" ranger
if [ "${SETUP_RANGER}" == "true" ]
then
@@ -50,6 +40,7 @@ then
python3 ${RANGER_SCRIPTS}/ranger-hdfs-service-dev_hdfs.py
python3 ${RANGER_SCRIPTS}/ranger-hive-service-dev_hive.py
+ python3 ${RANGER_SCRIPTS}/ranger-hbase-service-dev_hbase.py
fi
# prevent the container from exiting
[ranger] 01/02: RANGER-2973: NPE check in audit generation in
RangerHDFSPlugin to avoid possible issues
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.1
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 84e0bbd28ff000a3d018f9a939dc41c7991e3aac
Author: Ramesh Mani <ra...@gmail.com>
AuthorDate: Thu Aug 27 12:56:23 2020 -0700
RANGER-2973: NPE check in audit generation in RangerHDFSPlugin to avoid possible issues
(cherry picked from commit 84c39cbfe3f014d89feabb27748ffe3618fa7d2c)
---
.../authorization/hadoop/RangerHdfsAuthorizer.java | 44 +++++++++++-----------
1 file changed, 23 insertions(+), 21 deletions(-)
diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
index b7ace87..d8bcac7 100644
--- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
+++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
@@ -914,32 +914,34 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler {
LOG.debug("==> RangerHdfsAuditHandler.logAudit(" + result + ")");
}
- if(! isAuditEnabled && result.getIsAudited()) {
- isAuditEnabled = true;
- }
+ if (result != null) {
+ if(! isAuditEnabled && result.getIsAudited()) {
+ isAuditEnabled = true;
+ }
- if (auditEvent == null) {
- auditEvent = super.getAuthzEvents(result);
- }
+ if (auditEvent == null) {
+ auditEvent = super.getAuthzEvents(result);
+ }
- if (auditEvent != null) {
- RangerAccessRequest request = result.getAccessRequest();
- RangerAccessResource resource = request.getResource();
- String resourcePath = resource != null ? resource.getAsString() : null;
+ if (auditEvent != null) {
+ RangerAccessRequest request = result.getAccessRequest();
+ RangerAccessResource resource = request.getResource();
+ String resourcePath = resource != null ? resource.getAsString() : null;
- // Overwrite fields in original auditEvent
- auditEvent.setEventTime(request.getAccessTime() != null ? request.getAccessTime() : new Date());
- auditEvent.setAccessType(request.getAction());
- auditEvent.setResourcePath(this.pathToBeValidated);
- auditEvent.setResultReason(resourcePath);
+ // Overwrite fields in original auditEvent
+ auditEvent.setEventTime(request.getAccessTime() != null ? request.getAccessTime() : new Date());
+ auditEvent.setAccessType(request.getAction());
+ auditEvent.setResourcePath(this.pathToBeValidated);
+ auditEvent.setResultReason(resourcePath);
- auditEvent.setAccessResult((short) (result.getIsAllowed() ? 1 : 0));
- auditEvent.setPolicyId(result.getPolicyId());
- auditEvent.setPolicyVersion(result.getPolicyVersion());
+ auditEvent.setAccessResult((short) (result.getIsAllowed() ? 1 : 0));
+ auditEvent.setPolicyId(result.getPolicyId());
+ auditEvent.setPolicyVersion(result.getPolicyVersion());
- Set<String> tags = getTags(request);
- if (tags != null) {
- auditEvent.setTags(tags);
+ Set<String> tags = getTags(request);
+ if (tags != null) {
+ auditEvent.setTags(tags);
+ }
}
}