You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2021/03/02 22:30:15 UTC
[ranger] branch master updated: RANGER-3185: Docker setup to run
Ranger enabled HiveServer2
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 2595bc3 RANGER-3185: Docker setup to run Ranger enabled HiveServer2
2595bc3 is described below
commit 2595bc32f9494c6ef364ec27f450e2721424bd64
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Fri Feb 26 22:19:03 2021 -0800
RANGER-3185: Docker setup to run Ranger enabled HiveServer2
---
dev-support/ranger-docker/.env | 2 +
dev-support/ranger-docker/Dockerfile.ranger-base | 7 +-
...ranger-hadoop-mkdir.sh => Dockerfile.ranger-db} | 21 ++----
dev-support/ranger-docker/Dockerfile.ranger-hive | 48 ++++++++++++++
dev-support/ranger-docker/README.md | 40 +++++++----
.../init_postgres.sh} | 23 ++++---
.../ranger-docker/docker-compose.ranger-hive.yml | 30 +++++++++
.../ranger-docker/docker-compose.ranger.yml | 7 +-
dev-support/ranger-docker/scripts/hive-site.xml | 53 +++++++++++++++
.../scripts/ranger-admin-install.properties | 2 +-
.../ranger-docker/scripts/ranger-hadoop-mkdir.sh | 15 ++++-
.../scripts/ranger-hbase-service-dev_hbase.py | 4 +-
.../scripts/ranger-hdfs-service-dev_hdfs.py | 4 +-
.../scripts/ranger-hive-plugin-install.properties | 77 ++++++++++++++++++++++
.../scripts/ranger-hive-service-dev_hive.py | 4 +-
...ranger-hadoop-mkdir.sh => ranger-hive-setup.sh} | 38 +++++++----
.../{ranger-hadoop-mkdir.sh => ranger-hive.sh} | 36 ++++++----
.../scripts/ranger-kafka-service-dev_kafka.py | 4 +-
.../scripts/ranger-yarn-service-dev_yarn.py | 4 +-
distro/src/main/assembly/hive-agent.xml | 3 +
kms/scripts/install.properties | 2 +-
ranger-util/pom.xml | 2 +-
security-admin/scripts/install.properties | 2 +-
23 files changed, 341 insertions(+), 87 deletions(-)
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
index d280c10..6df1245 100644
--- a/dev-support/ranger-docker/.env
+++ b/dev-support/ranger-docker/.env
@@ -8,4 +8,6 @@ POSTGRES_PASSWORD=rangerR0cks!
RANGER_VERSION=3.0.0-SNAPSHOT
HADOOP_VERSION=3.3.0
HBASE_VERSION=2.2.6
+HIVE_VERSION=3.1.2
+HIVE_HADOOP_VERSION=3.1.0
KAFKA_VERSION=2.5.0
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base
index a09fbda..c7fb75c 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-base
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -20,12 +20,9 @@ FROM ubuntu:20.04
# Install curl, wget, tzdata, Python, Java, python-requests
RUN apt-get update && \
DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
- python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
- curl https://bootstrap.pypa.io/get-pip.py --output /tmp/get-pip.py && \
- python2 /tmp/get-pip.py && \
+ python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh && \
pip3 install apache-ranger && \
- pip3 install requests && \
- pip install requests
+ pip3 install requests
# Set environment variables
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh b/dev-support/ranger-docker/Dockerfile.ranger-db
old mode 100755
new mode 100644
similarity index 50%
copy from dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
copy to dev-support/ranger-docker/Dockerfile.ranger-db
index ab9911a..f33e813
--- a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
+++ b/dev-support/ranger-docker/Dockerfile.ranger-db
@@ -1,5 +1,3 @@
-#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/kafka
+FROM postgres:12
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
-${HADOOP_HOME}/bin/hdfs dfs -chown hdfs:hadoop /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -chown yarn:hadoop /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -chown kafka:hadoop /ranger/audit/kafka
+# Copy DB init script
+USER 0
+RUN mkdir -p /docker-entrypoint-initdb.d
+COPY config/init_postgres.sh /docker-entrypoint-initdb.d/
+RUN chown -R postgres:postgres /docker-entrypoint-initdb.d/
+ENV POSTGRES_PASSWORD rangerR0cks!
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive
new file mode 100644
index 0000000..bd31629
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hive
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ranger-base:latest
+
+ARG HIVE_VERSION
+ARG HIVE_HADOOP_VERSION
+ARG RANGER_VERSION
+
+
+COPY ./dist/version /home/ranger/dist/
+COPY ./dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
+COPY ./scripts/ranger-hive-setup.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hive.sh /home/ranger/scripts/
+COPY ./scripts/ranger-hive-plugin-install.properties /home/ranger/scripts/
+COPY ./scripts/hive-site.xml /home/ranger/scripts/
+
+RUN curl https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz --output /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
+ tar xvfz /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
+ ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
+ rm -f /tmp/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
+ curl https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION}/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --output /tmp/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
+ tar xvfz /tmp/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --directory=/opt/ && \
+ ln -s /opt/hadoop-${HIVE_HADOOP_VERSION} /opt/hadoop && \
+ rm -f /tmp/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
+ tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
+ ln -s /opt/ranger/ranger-${RANGER_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
+ rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-hive-plugin.tar.gz && \
+ cp -f /home/ranger/scripts/ranger-hive-plugin-install.properties /opt/ranger/ranger-hive-plugin/install.properties
+
+ENV HIVE_HOME /opt/hive
+ENV HADOOP_HOME /opt/hadoop
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ]
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 19508ed..1f8689a 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -39,7 +39,7 @@ Docker files in this folder create docker images and run them to build Apache Ra
4.2. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Kafka and dependent services (Solr, DB) in containers:
- docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml up -d
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml up -d
5. Alternatively docker command can be used to build and deploy Apache Ranger.
@@ -69,54 +69,70 @@ Docker files in this folder create docker images and run them to build Apache Ra
docker build -f Dockerfile.ranger-solr -t ranger-solr .
- 5.6. Execute following command to start a container that runs database for use by Ranger Admin:
+ 5.6. Execute following command to build a Docker image **ranger-db**:
- docker run --name ranger-db --hostname ranger-db.example.com --env-file ./.env -d postgres:12
+ docker build -f Dockerfile.ranger-db -t ranger-db .
5.7. Execute following command to start a container that runs Solr for use by Ranger Admin:
docker run --name ranger-solr --hostname ranger-solr.example.com -p 8983:8983 -d ranger-solr solr-precreate ranger_audits /opt/solr/server/solr/configsets/ranger_audits/
- 5.8. Execute following command to install and run Ranger services in a container:
+ 5.8. Execute following command to start a container that runs database for use by Ranger Admin:
+
+ docker run --name ranger-db --hostname ranger-db.example.com --env-file ./.env -d ranger-db
+
+ 5.9. Execute following command to install and run Ranger services in a container:
docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr --env-file ./.env ranger
This might take few minutes to complete.
- 5.9. Execute following command to build Docker image **ranger-hadoop**:
+ 5.10. Execute following command to build Docker image **ranger-hadoop**:
- docker build -f Dockerfile.ranger-hadoop --build-arg RANGER_VERSION=`cat dist/version` --build-arg HADOOP_VERSION=3.1.1 -t ranger-hadoop .
+ docker build -f Dockerfile.ranger-hadoop --build-arg RANGER_VERSION=`cat dist/version` --build-arg HADOOP_VERSION=3.3.0 -t ranger-hadoop .
This step includes downloading of Hadoop tar balls, and can take a while to complete.
- 5.10. Execute following command to install and run Ranger enabled HDFS in a container:
+ 5.11. Execute following command to install and run Ranger enabled HDFS in a container:
docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 -p 8088:8088 --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hadoop
This might take few minutes to complete.
- 5.11. Execute following command to build Docker image **ranger-hbase**:
+ 5.12. Execute following command to build Docker image **ranger-hbase**:
- docker build -f Dockerfile.ranger-hbase --build-arg RANGER_VERSION=`cat dist/version` --build-arg HBASE_VERSION=2.0.3 -t ranger-hbase .
+ docker build -f Dockerfile.ranger-hbase --build-arg RANGER_VERSION=`cat dist/version` --build-arg HBASE_VERSION=2.2.6 -t ranger-hbase .
This step includes downloading of HBase tar ball, and can take a while to complete.
- 5.12. Execute following command to install and run Ranger enabled HBase in a container:
+ 5.13. Execute following command to install and run Ranger enabled HBase in a container:
docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hbase
This might take few minutes to complete.
- 5.13. Execute following command to build Docker image **ranger-kafka**:
+ 5.14. Execute following command to build Docker image **ranger-kafka**:
docker build -f Dockerfile.ranger-kafka --build-arg RANGER_VERSION=`cat dist/version` --build-arg KAFKA_VERSION=2.5.0 -t ranger-kafka .
This step includes downloading of Kafka tar ball, and can take a while to complete.
- 5.14. Execute following command to install and run Ranger enabled Kafka in a container:
+ 5.15. Execute following command to install and run Ranger enabled Kafka in a container:
docker run -it -d --name ranger-kafka --hostname ranger-kafka.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-kafka
This might take few minutes to complete.
+ 5.16. Execute following command to build Docker image **ranger-hive**:
+
+ docker build -f Dockerfile.ranger-hive --build-arg RANGER_VERSION=`cat dist/version` --build-arg HIVE_VERSION=3.1.2 --build-arg HIVE_HADOOP_VERSION=3.1.0 -t ranger-hbase .
+
+ This step includes downloading of Hive tar ball and Hadoop tar ball, and can take a while to complete.
+
+ 5.17. Execute following command to install and run Ranger enabled Hive in a container:
+
+ docker run -it -d --name ranger-hive --hostname ranger-hive.example.com --link ranger-hadoop:ranger-hadoop --link ranger-hbase:ranger-hbase --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hive
+
+ This might take few minutes to complete.
+
6. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!)
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh b/dev-support/ranger-docker/config/init_postgres.sh
old mode 100755
new mode 100644
similarity index 51%
copy from dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
copy to dev-support/ranger-docker/config/init_postgres.sh
index ab9911a..2a739d1
--- a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
+++ b/dev-support/ranger-docker/config/init_postgres.sh
@@ -16,16 +16,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/kafka
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
-${HADOOP_HOME}/bin/hdfs dfs -chown hdfs:hadoop /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -chown yarn:hadoop /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -chown kafka:hadoop /ranger/audit/kafka
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+ CREATE USER rangeradmin WITH PASSWORD 'rangerR0cks!';
+ CREATE DATABASE ranger;
+ GRANT ALL PRIVILEGES ON DATABASE ranger TO rangeradmin;
+
+ CREATE USER hive WITH PASSWORD 'rangerR0cks!';
+ CREATE DATABASE hive;
+ GRANT ALL PRIVILEGES ON DATABASE hive TO hive;
+EOSQL
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hive.yml b/dev-support/ranger-docker/docker-compose.ranger-hive.yml
new file mode 100644
index 0000000..d7c1797
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-hive.yml
@@ -0,0 +1,30 @@
+version: '3'
+services:
+ ranger-hive:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-hive
+ args:
+ - HIVE_HADOOP_VERSION=${HIVE_HADOOP_VERSION}
+ - HIVE_VERSION=${HIVE_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
+ image: ranger-hive
+ container_name: ranger-hive
+ hostname: ranger-hive.example.com
+ stdin_open: true
+ tty: true
+ networks:
+ - ranger
+ ports:
+ - "10000:10000"
+ depends_on:
+ - ranger
+ - ranger-hadoop
+ - ranger-hbase
+ environment:
+ - HIVE_HADOOP_VERSION
+ - HIVE_VERSION
+ - RANGER_VERSION
+
+networks:
+ ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml
index 089a51b..db32cdb 100644
--- a/dev-support/ranger-docker/docker-compose.ranger.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger.yml
@@ -41,13 +41,14 @@ services:
- /opt/solr/server/solr/configsets/ranger_audits/
ranger-db:
- image: postgres:12
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-db
+ image: ranger-db
container_name: ranger-db
hostname: ranger-db.example.com
networks:
- ranger
- environment:
- POSTGRES_PASSWORD: rangerR0cks!
networks:
ranger:
diff --git a/dev-support/ranger-docker/scripts/hive-site.xml b/dev-support/ranger-docker/scripts/hive-site.xml
new file mode 100644
index 0000000..69386eb
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/hive-site.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:postgresql://ranger-db/hive</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>org.postgresql.Driver</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>hive</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>rangerR0cks!</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.doAs</name>
+ <value>false</value>
+ </property>
+
+ <!-- NOTE: using zookeeper from ranger-hbase! -->
+ <property>
+ <name>hive.zookeeper.quorum</name>
+ <value>ranger-hbase.example.com</value>
+ </property>
+ <property>
+ <name>hive.zookeeper.client.port</name>
+ <value>2181</value>
+ </property>
+</configuration>
diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install.properties b/dev-support/ranger-docker/scripts/ranger-admin-install.properties
index 86019c2..dc43306 100644
--- a/dev-support/ranger-docker/scripts/ranger-admin-install.properties
+++ b/dev-support/ranger-docker/scripts/ranger-admin-install.properties
@@ -17,7 +17,7 @@
# This file provides a list of the deployment variables for the Policy Manager Web Application
#
-PYTHON_COMMAND_INVOKER=python
+PYTHON_COMMAND_INVOKER=python3
RANGER_ADMIN_LOG_DIR=/var/log/ranger
RANGER_PID_DIR_PATH=/var/run/ranger
DB_FLAVOR=POSTGRES
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
index ab9911a..de16245 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
@@ -16,16 +16,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
+# setup directories for Ranger audits
${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseRegional
${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/kafka
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hiveServer2
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
${HADOOP_HOME}/bin/hdfs dfs -chown hdfs:hadoop /ranger/audit/hdfs
${HADOOP_HOME}/bin/hdfs dfs -chown yarn:hadoop /ranger/audit/yarn
${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseMaster
${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseRegional
${HADOOP_HOME}/bin/hdfs dfs -chown kafka:hadoop /ranger/audit/kafka
+${HADOOP_HOME}/bin/hdfs dfs -chown hive:hadoop /ranger/audit/hiveServer2
+
+# setup directories for HBase
+${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
+${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
+
+# setup directories for Hive
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/hive/warehouse
+${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /tmp/hive
+${HADOOP_HOME}/bin/hdfs dfs -chown -R hive:hadoop /tmp/hive /user/hive
+${HADOOP_HOME}/bin/hdfs dfs -chmod 777 /tmp/hive
diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
index 9294bf5..80a3d44 100644
--- a/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
+++ b/dev-support/ranger-docker/scripts/ranger-hbase-service-dev_hbase.py
@@ -1,8 +1,8 @@
from apache_ranger.model.ranger_service import RangerService
from apache_ranger.client.ranger_client import RangerClient
-ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!'))
-service = RangerService(name='dev_hbase', type='hbase', configs={'username':'hbase', 'password':'hbase', 'hadoop.security.authentication': 'simple', 'hbase.security.authentication': 'simple', 'hadoop.security.authorization': 'true', 'hbase.zookeeper.property.clientPort': '16181', 'hbase.zookeeper.quorum': 'ranger-hbase', 'zookeeper.znode.parent': '/hbase'})
+service = RangerService({'name': 'dev_hbase', 'type': 'hbase', 'configs': {'username':'hbase', 'password':'hbase', 'hadoop.security.authentication': 'simple', 'hbase.security.authentication': 'simple', 'hadoop.security.authorization': 'true', 'hbase.zookeeper.property.clientPort': '16181', 'hbase.zookeeper.quorum': 'ranger-hbase', 'zookeeper.znode.parent': '/hbase'}})
ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-hdfs-service-dev_hdfs.py b/dev-support/ranger-docker/scripts/ranger-hdfs-service-dev_hdfs.py
index 27d20b7..f1b138e 100644
--- a/dev-support/ranger-docker/scripts/ranger-hdfs-service-dev_hdfs.py
+++ b/dev-support/ranger-docker/scripts/ranger-hdfs-service-dev_hdfs.py
@@ -1,8 +1,8 @@
from apache_ranger.model.ranger_service import RangerService
from apache_ranger.client.ranger_client import RangerClient
-ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!'))
-service = RangerService(name='dev_hdfs', type='hdfs', configs={'username':'hdfs', 'password':'hdfs', 'fs.default.name': 'hdfs://ranger-hadoop:9000', 'hadoop.security.authentication': 'simple', 'hadoop.security.authorization': 'true'})
+service = RangerService({'name': 'dev_hdfs', 'type': 'hdfs', 'configs': {'username':'hdfs', 'password':'hdfs', 'fs.default.name': 'hdfs://ranger-hadoop:9000', 'hadoop.security.authentication': 'simple', 'hadoop.security.authorization': 'true'}})
ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties b/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties
new file mode 100644
index 0000000..b3e403c
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/ranger-hive-plugin-install.properties
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+POLICY_MGR_URL=http://ranger:6080
+REPOSITORY_NAME=dev_hive
+COMPONENT_INSTALL_DIR_NAME=/opt/hive
+UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
+
+CUSTOM_USER=hive
+CUSTOM_GROUP=hive
+
+XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
+
+# Following properties are needed to get past installation script! Please don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hive
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hive/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hive/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
diff --git a/dev-support/ranger-docker/scripts/ranger-hive-service-dev_hive.py b/dev-support/ranger-docker/scripts/ranger-hive-service-dev_hive.py
index 36a871c..c2bd7d7 100644
--- a/dev-support/ranger-docker/scripts/ranger-hive-service-dev_hive.py
+++ b/dev-support/ranger-docker/scripts/ranger-hive-service-dev_hive.py
@@ -1,8 +1,8 @@
from apache_ranger.model.ranger_service import RangerService
from apache_ranger.client.ranger_client import RangerClient
-ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!'))
-service = RangerService(name='dev_hive', type='hive', configs={'username':'hive', 'password':'hive', 'jdbc.driverClassName': 'org.apache.hive.jdbc.HiveDriver', 'jdbc.url': 'jdfb:hive2://ranger-hadoop:10000', 'hadoop.security.authorization': 'true'})
+service = RangerService({'name': 'dev_hive', 'type': 'hive', 'configs': {'username':'hive', 'password':'hive', 'jdbc.driverClassName': 'org.apache.hive.jdbc.HiveDriver', 'jdbc.url': 'jdfb:hive2://ranger-hadoop:10000', 'hadoop.security.authorization': 'true'}})
ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh b/dev-support/ranger-docker/scripts/ranger-hive-setup.sh
similarity index 51%
copy from dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
copy to dev-support/ranger-docker/scripts/ranger-hive-setup.sh
index ab9911a..27b2323 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hive-setup.sh
@@ -16,16 +16,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/kafka
+echo "export JAVA_HOME=${JAVA_HOME}" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
-${HADOOP_HOME}/bin/hdfs dfs -chown hdfs:hadoop /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -chown yarn:hadoop /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -chown kafka:hadoop /ranger/audit/kafka
+cat <<EOF > /etc/ssh/ssh_config
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile=/dev/null
+EOF
+
+cat <<EOF > ${HADOOP_HOME}/etc/hadoop/core-site.xml
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://ranger-hadoop:9000</value>
+ </property>
+</configuration>
+EOF
+
+cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hive-site.xml
+cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hiveserver2-site.xml
+su -c "${HIVE_HOME}/bin/schematool -dbType postgres -initSchema" hive
+
+mkdir -p /opt/hive/logs
+chown -R hive:hadoop /opt/hive/
+chmod g+w /opt/hive/logs
+
+cd ${RANGER_HOME}/ranger-hive-plugin
+./enable-hive-plugin.sh
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh b/dev-support/ranger-docker/scripts/ranger-hive.sh
similarity index 51%
copy from dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
copy to dev-support/ranger-docker/scripts/ranger-hive.sh
index ab9911a..7057c28 100755
--- a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
+++ b/dev-support/ranger-docker/scripts/ranger-hive.sh
@@ -16,16 +16,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-${HADOOP_HOME}/bin/hdfs dfs -mkdir /hbase
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/kafka
-
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /hbase
-${HADOOP_HOME}/bin/hdfs dfs -chown hdfs:hadoop /ranger/audit/hdfs
-${HADOOP_HOME}/bin/hdfs dfs -chown yarn:hadoop /ranger/audit/yarn
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseMaster
-${HADOOP_HOME}/bin/hdfs dfs -chown hbase:hadoop /ranger/audit/hbaseRegional
-${HADOOP_HOME}/bin/hdfs dfs -chown kafka:hadoop /ranger/audit/kafka
+service ssh start
+
+if [ ! -e ${HIVE_HOME}/.setupDone ]
+then
+ su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs
+ su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hdfs
+ su -c "chmod 0600 ~/.ssh/authorized_keys" hdfs
+
+ su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" yarn
+ su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" yarn
+ su -c "chmod 0600 ~/.ssh/authorized_keys" yarn
+
+ echo "ssh" > /etc/pdsh/rcmd_default
+
+ ${RANGER_SCRIPTS}/ranger-hive-setup.sh
+
+ touch ${HIVE_HOME}/.setupDone
+fi
+
+su -c "${HIVE_HOME}/bin/hiveserver2" hive
+
+# prevent the container from exiting
+/bin/bash
diff --git a/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py b/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
index 2274d32..99a953c 100644
--- a/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
+++ b/dev-support/ranger-docker/scripts/ranger-kafka-service-dev_kafka.py
@@ -1,8 +1,8 @@
from apache_ranger.model.ranger_service import RangerService
from apache_ranger.client.ranger_client import RangerClient
-ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!'))
-service = RangerService(name='dev_kafka', type='kafka', configs={'username':'kafka', 'password':'kafka', 'zookeeper.connect': 'ranger-kafka:2181'})
+service = RangerService({'name': 'dev_kafka', 'type': 'kafka', 'configs': {'username':'kafka', 'password':'kafka', 'zookeeper.connect': 'ranger-kafka:2181'}})
ranger_client.create_service(service)
diff --git a/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py b/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py
index b3e8a28..8f1ba71 100644
--- a/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py
+++ b/dev-support/ranger-docker/scripts/ranger-yarn-service-dev_yarn.py
@@ -1,8 +1,8 @@
from apache_ranger.model.ranger_service import RangerService
from apache_ranger.client.ranger_client import RangerClient
-ranger_client = RangerClient('http://ranger:6080', 'admin', 'rangerR0cks!')
+ranger_client = RangerClient('http://ranger:6080', ('admin', 'rangerR0cks!'))
-service = RangerService(name='dev_yarn', type='yarn', configs={'username':'yarn', 'password':'yarn', 'yarn.url': 'http://ranger-hadoop:8088'})
+service = RangerService({'name': 'dev_yarn', 'type': 'yarn', 'configs': {'username':'yarn', 'password':'yarn', 'yarn.url': 'http://ranger-hadoop:8088'}})
ranger_client.create_service(service)
diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml
index f2b1f22..41d130c 100644
--- a/distro/src/main/assembly/hive-agent.xml
+++ b/distro/src/main/assembly/hive-agent.xml
@@ -53,6 +53,7 @@
<directoryMode>755</directoryMode>
<fileMode>644</fileMode>
<includes>
+ <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include>
<include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
<include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include>
<include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include>
@@ -91,6 +92,8 @@
<include>commons-cli:commons-cli</include>
<include>commons-collections:commons-collections</include>
<include>org.apache.commons:commons-configuration2:jar:${commons.configuration.version}</include>
+ <include>org.apache.commons:commons-lang3:jar:${commons.lang3.version}</include>
+ <include>org.apache.commons:commons-compress:jar:${commons.compress.version}</include>
<include>commons-io:commons-io:jar:${commons.io.version}</include>
<include>commons-lang:commons-lang</include>
<include>commons-logging:commons-logging:jar:${commons.logging.version}</include>
diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties
index 137a729..4935536 100755
--- a/kms/scripts/install.properties
+++ b/kms/scripts/install.properties
@@ -21,7 +21,7 @@
# Uncomment the below if the DBA steps need to be run separately
#setup_mode=SeparateDBA
-PYTHON_COMMAND_INVOKER=python
+PYTHON_COMMAND_INVOKER=python3
#DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA
DB_FLAVOR=MYSQL
diff --git a/ranger-util/pom.xml b/ranger-util/pom.xml
index 3fb81c1..6a4b581 100644
--- a/ranger-util/pom.xml
+++ b/ranger-util/pom.xml
@@ -46,7 +46,7 @@
<phase>generate-sources</phase>
<configuration>
<target>
- <exec executable="python" failonerror="true">
+ <exec executable="python3" failonerror="true">
<arg value="${project.basedir}/src/scripts/saveVersion.py"/>
<arg value="${project.version}"/>
<arg value="${ranger.version.shortname}"/>
diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties
index d300de1..6cde15d 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -21,7 +21,7 @@
# Uncomment the below if the DBA steps need to be run separately
#setup_mode=SeparateDBA
-PYTHON_COMMAND_INVOKER=python
+PYTHON_COMMAND_INVOKER=python3
#DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA
DB_FLAVOR=MYSQL