You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by gu...@apache.org on 2022/07/05 06:30:31 UTC
[bigtop] branch master updated: BIGTOP-2971. RPM packaging for Ranger. (#929)
This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 47ee254a BIGTOP-2971. RPM packaging for Ranger. (#929)
47ee254a is described below
commit 47ee254af1fb7ff131bdc36bed12d66b85ac63b9
Author: Kengo Seki <se...@apache.org>
AuthorDate: Tue Jul 5 15:30:26 2022 +0900
BIGTOP-2971. RPM packaging for Ranger. (#929)
This PR is based on #326 submitted by @Xingwd (much appreciated!) and
includes the following updates:
* Upgrade Ranger version to 2.2.0
* Change the download site from ASF to GitHub, because the source
tarball distributed on the former is unnecessarily large
* Add package dependencies to bigtop.bom
* Remove the assembly:assembly target from the mvn command since it
doesn't work in recent releases
* Add the version info of other components to the mvn command
* Fix the lib paths in install_ranger.sh and the RPM spec file
* Add the #BIGTOP_PATCH_FILES and #BIGTOP_PATCH_COMMANDS comments to the
RPM spec so that the patches provided by Bigtop are applied during
building RPM
* Add patches for addressing RANGER-3373 and RANGER-3459
* Remove symlink to non-existent ojdbc6.jar in install_ranger.sh
* Replace the hard-coded paths to /usr/lib/rpm/redhat in the RPM spec
with %{_rpmconfigdir} (cf. BIGTOP-1344)
* Define debug_package=nil in the RPM spec to avoid build failure on
Fedora 35
---
.../src/common/ranger/do-component-build | 35 ++
.../src/common/ranger/install_ranger.sh | 155 ++++++++
.../src/common/ranger/patch0-RANGER-3373.diff | 33 ++
.../src/common/ranger/patch1-RANGER-3459.diff | 193 ++++++++++
.../src/common/ranger/patch2-update-solr.diff | 22 ++
bigtop-packages/src/rpm/ranger/SPECS/ranger.spec | 426 +++++++++++++++++++++
bigtop.bom | 11 +-
7 files changed, 874 insertions(+), 1 deletion(-)
diff --git a/bigtop-packages/src/common/ranger/do-component-build b/bigtop-packages/src/common/ranger/do-component-build
new file mode 100644
index 00000000..eff4bc49
--- /dev/null
+++ b/bigtop-packages/src/common/ranger/do-component-build
@@ -0,0 +1,35 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+. `dirname $0`/bigtop.bom
+
+
+mvn clean compile package install -DskipTests -Drat.skip \
+ -Dhadoop.version=${HADOOP_VERSION} \
+ -Dhbase.version=${HBASE_VERSION} \
+ -Dhive.version=${HIVE_VERSION} \
+ -Dkafka.version=${KAFKA_VERSION} \
+ -Dsolr.version=${SOLR_VERSION} \
+ -Dzookeeper.version=${ZOOKEEPER_VERSION} \
+ "$@"
+
+mkdir build
+for f in target/ranger*.tar.gz
+do
+ tar -C build -xf ${f}
+done
diff --git a/bigtop-packages/src/common/ranger/install_ranger.sh b/bigtop-packages/src/common/ranger/install_ranger.sh
new file mode 100644
index 00000000..481956ec
--- /dev/null
+++ b/bigtop-packages/src/common/ranger/install_ranger.sh
@@ -0,0 +1,155 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to ranger dist.dir
+ --prefix=PREFIX path to install into
+ --component=rangerComponentName Ranger component name [admin|hdfs-plugin|yarn-plugin|hive-plugin|hbase-plugin|kafka-plugin|atlas-plugin|...|usersync|kms|tagsync]
+ Optional options:
+ --comp-dir=DIR path to install ranger comp [/usr/lib/ranger/admin]
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'build-dir:' \
+ -l 'prefix:' \
+ -l 'doc-dir:' \
+ -l 'comp-dir:' \
+ -l 'component:' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --component)
+ COMPONENT=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --comp-dir)
+ COMP_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR COMPONENT ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+
+RANGER_HOME=${RANGER_HOME:-/usr/lib/ranger}
+ETC_DIR=${ETC_DIR:-/etc/ranger}
+ADMIN_CONF_DIR=${CONF_DIR:-${ETC_DIR}/admin/conf.dist}
+USERSYNC_CONF_DIR=${CONF_DIR:-${ETC_DIR}/usersync/conf.dist}
+KMS_CONF_DIR=${CONF_DIR:-${ETC_DIR}/kms/conf.dist}
+TAGSYNC_CONF_DIR=${CONF_DIR:-${ETC_DIR}/tagsync/conf.dist}
+
+if [ "${COMP_DIR}" == "" ]
+then
+ COMP_DIR=ranger-${COMPONENT}
+fi
+
+# Create the required directories.
+install -d -m 0755 ${PREFIX}/usr/lib/$COMP_DIR
+
+install -d -m 0755 ${PREFIX}/$ETC_DIR/{admin,usersync,kms,tagsync}
+
+install -d -m 0755 ${PREFIX}/var/{log,run}/ranger/{admin,usersync,kms,tagsync}
+
+
+# Copy artifacts to the appropriate Linux locations.
+cp -r ${BUILD_DIR}/ranger-*-${COMPONENT}/* ${PREFIX}/usr/lib/${COMP_DIR}/
+
+
+if [[ "${COMPONENT}" = "admin" ]]
+then
+cp -a ${BUILD_DIR}/ranger-*-${COMPONENT}/ews/webapp/WEB-INF/classes/conf.dist ${PREFIX}/${ADMIN_CONF_DIR}
+ln -s /etc/ranger/admin/conf ${PREFIX}/usr/lib/${COMP_DIR}/conf
+ln -s /usr/lib/${COMP_DIR}/conf ${PREFIX}/usr/lib/${COMP_DIR}/ews/webapp/WEB-INF/classes/conf
+ln -s /usr/lib/${COMP_DIR}/ews/start-ranger-admin.sh ${PREFIX}/usr/lib/${COMP_DIR}/ews/ranger-admin-start
+ln -s /usr/lib/${COMP_DIR}/ews/stop-ranger-admin.sh ${PREFIX}/usr/lib/${COMP_DIR}/ews/ranger-admin-stop
+fi
+
+if [[ "${COMPONENT}" = "usersync" ]]
+then
+echo "usersync"
+cp -a ${BUILD_DIR}/ranger-*-${COMPONENT}/conf.dist ${PREFIX}/${USERSYNC_CONF_DIR}
+ln -s /etc/ranger/usersync/conf ${PREFIX}/usr/lib/${COMP_DIR}/conf
+ln -s /usr/lib/${COMP_DIR}/start.sh ${PREFIX}/usr/lib/${COMP_DIR}/ranger-usersync-start
+ln -s /usr/lib/${COMP_DIR}/stop.sh ${PREFIX}/usr/lib/${COMP_DIR}/ranger-usersync-stop
+fi
+
+if [[ "${COMPONENT}" = "kms" ]]
+then
+echo "kms"
+cp -a ${BUILD_DIR}/ranger-*-${COMPONENT}/ews/webapp/WEB-INF/classes/conf.dist ${PREFIX}/${KMS_CONF_DIR}
+ln -s /etc/ranger/kms/conf ${PREFIX}//usr/lib/${COMP_DIR}/conf
+ln -s /usr/lib/${COMP_DIR}/conf ${PREFIX}/usr/lib/${COMP_DIR}/ews/webapp/WEB-INF/classes/conf
+fi
+
+if [[ "${COMPONENT}" = "tagsync" ]]
+then
+echo "tagsync"
+cp -a ${BUILD_DIR}/ranger-*-${COMPONENT}/conf.dist ${PREFIX}/${TAGSYNC_CONF_DIR}
+ln -s /etc/ranger/tagsync/conf ${PREFIX}/usr/lib/${COMP_DIR}/conf
+fi
+
+# For other Components
+if [[ "${COMPONENT}" = "hive-plugin" || "${COMPONENT}" = "hbase-plugin" || "${COMPONENT}" = "storm-plugin" || "${COMPONENT}" = "hdfs-plugin" || "${COMPONENT}" = "yarn-plugin" || "${COMPONENT}" = "kafka-plugin" || "${COMPONENT}" = "atlas-plugin" || "${COMPONENT}" = "knox-plugin" ]]
+then
+ RANGER_COMPONENT=${COMPONENT}
+ [[ "${COMPONENT}" = "hdfs-plugin" ]] && RANGER_COMPONENT="hadoop"
+ [[ "${COMPONENT}" = "yarn-plugin" ]] && RANGER_COMPONENT="hadoop"
+ [[ "${COMPONENT}" = "storm-plugin" ]] && RANGER_COMPONENT="storm"
+ [[ "${COMPONENT}" = "hbase-plugin" ]] && RANGER_COMPONENT="hbase"
+ [[ "${COMPONENT}" = "hive-plugin" ]] && RANGER_COMPONENT="hive"
+ [[ "${COMPONENT}" = "kafka-plugin" ]] && RANGER_COMPONENT="kafka"
+ [[ "${COMPONENT}" = "atlas-plugin" ]] && RANGER_COMPONENT="atlas"
+ [[ "${COMPONENT}" = "knox-plugin" ]] && RANGER_COMPONENT="knox"
+ install -d -m 0755 ${PREFIX}/usr/lib/${RANGER_COMPONENT}/lib
+ cp -r $BUILD_DIR/ranger-*-${COMPONENT}/lib/* ${PREFIX}/usr/lib/${RANGER_COMPONENT}/lib/
+fi
diff --git a/bigtop-packages/src/common/ranger/patch0-RANGER-3373.diff b/bigtop-packages/src/common/ranger/patch0-RANGER-3373.diff
new file mode 100644
index 00000000..f649d430
--- /dev/null
+++ b/bigtop-packages/src/common/ranger/patch0-RANGER-3373.diff
@@ -0,0 +1,33 @@
+diff --git a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java
+index 9be691485..bdbd711d1 100644
+--- a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java
++++ b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/RangerAuthorizationCoprocessor.java
+@@ -788,7 +788,7 @@ public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) t
+ Throwable var3 = null;
+
+ try {
+- if(!admin.tableExists(AccessControlLists.ACL_TABLE_NAME)) {
++ if(!admin.tableExists(PermissionStorage.ACL_TABLE_NAME)) {
+ createACLTable(admin);
+ }
+ } catch (Throwable var12) {
+@@ -812,8 +812,8 @@ public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) t
+ }
+
+ private static void createACLTable(Admin admin) throws IOException {
+- ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(AccessControlLists.ACL_LIST_FAMILY).setMaxVersions(1).setInMemory(true).setBlockCacheEnabled(true).setBlocksize(8192).setBloomFilterType(BloomType.NONE).setScope(0).build();
+- TableDescriptor td = TableDescriptorBuilder.newBuilder(AccessControlLists.ACL_TABLE_NAME).addColumnFamily(cfd).build();
++ ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(PermissionStorage.ACL_LIST_FAMILY).setMaxVersions(1).setInMemory(true).setBlockCacheEnabled(true).setBlocksize(8192).setBloomFilterType(BloomType.NONE).setScope(0).build();
++ TableDescriptor td = TableDescriptorBuilder.newBuilder(PermissionStorage.ACL_TABLE_NAME).addColumnFamily(cfd).build();
+ admin.createTable(td);
+ }
+
+@@ -1403,7 +1403,7 @@ public void getUserPermissions(RpcController controller, AccessControlProtos.Get
+ });
+ if (_userUtils.isSuperUser(user)) {
+ perms.add(new UserPermission(_userUtils.getUserAsString(user),
+- Permission.newBuilder(AccessControlLists.ACL_TABLE_NAME).withActions(Action.values()).build()));
++ Permission.newBuilder(PermissionStorage.ACL_TABLE_NAME).withActions(Action.values()).build()));
+ }
+ }
+ response = AccessControlUtil.buildGetUserPermissionsResponse(perms);
diff --git a/bigtop-packages/src/common/ranger/patch1-RANGER-3459.diff b/bigtop-packages/src/common/ranger/patch1-RANGER-3459.diff
new file mode 100644
index 00000000..8537e84d
--- /dev/null
+++ b/bigtop-packages/src/common/ranger/patch1-RANGER-3459.diff
@@ -0,0 +1,193 @@
+From b63924c717b1a394d693814d02a282a0aaa7dc97 Mon Sep 17 00:00:00 2001
+From: Viktor Somogyi-Vass <vi...@gmail.com>
+Date: Mon, 20 Sep 2021 17:23:29 +0200
+Subject: [PATCH] RANGER-3459: Upgrade Kafka dependency to 2.8
+
+Co-authored-by: Andras Katona <ak...@cloudera.com>
+---
+ .../kafka/authorizer/KafkaRangerAuthorizerGSSTest.java | 8 +++++---
+ .../authorizer/KafkaRangerAuthorizerSASLSSLTest.java | 8 +++++---
+ .../kafka/authorizer/KafkaRangerAuthorizerTest.java | 8 +++++---
+ .../kafka/authorizer/KafkaRangerTopicCreationTest.java | 8 +++++---
+ pom.xml | 2 +-
+ 5 files changed, 21 insertions(+), 13 deletions(-)
+
+diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
+index fe600b1eaf..e82de18495 100644
+--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
++++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
+@@ -30,6 +30,7 @@
+ import java.util.Properties;
+ import java.util.concurrent.Future;
+
++import kafka.server.KafkaServer;
+ import org.apache.curator.test.InstanceSpec;
+ import org.apache.curator.test.TestingServer;
+ import org.apache.hadoop.security.UserGroupInformation;
+@@ -43,6 +44,7 @@
+ import org.apache.kafka.clients.producer.RecordMetadata;
+ import org.apache.kafka.common.PartitionInfo;
+ import org.apache.kafka.common.config.SaslConfigs;
++import org.apache.kafka.common.utils.Time;
+ import org.apache.kerby.kerberos.kerb.server.SimpleKdcServer;
+ import org.junit.Assert;
+ import org.junit.Test;
+@@ -50,7 +52,7 @@
+ import org.slf4j.LoggerFactory;
+
+ import kafka.server.KafkaConfig;
+-import kafka.server.KafkaServerStartable;
++import scala.Some;
+
+ /**
+ * A simple test that starts a Kafka broker, creates "test" and "dev" topics,
+@@ -69,7 +71,7 @@
+ public class KafkaRangerAuthorizerGSSTest {
+ private final static Logger LOG = LoggerFactory.getLogger(KafkaRangerAuthorizerGSSTest.class);
+
+- private static KafkaServerStartable kafkaServer;
++ private static KafkaServer kafkaServer;
+ private static TestingServer zkServer;
+ private static int port;
+ private static Path tempDir;
+@@ -140,7 +142,7 @@ public static void setup() throws Exception {
+ UserGroupInformation.createUserForTesting("kafka/localhost@kafka.apache.org", new String[] {"IT"});
+
+ KafkaConfig config = new KafkaConfig(props);
+- kafkaServer = new KafkaServerStartable(config);
++ kafkaServer = new KafkaServer(config, Time.SYSTEM, new Some<String>("KafkaRangerAuthorizerGSSTest"), false);
+ kafkaServer.startup();
+
+ // Create some topics
+diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java
+index 6f4538549b..b25d1fdd10 100644
+--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java
++++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java
+@@ -27,6 +27,7 @@
+ import java.util.Properties;
+ import java.util.concurrent.Future;
+
++import kafka.server.KafkaServer;
+ import org.apache.commons.io.FileUtils;
+ import org.apache.curator.test.TestingServer;
+ import org.apache.hadoop.security.UserGroupInformation;
+@@ -40,11 +41,12 @@
+ import org.apache.kafka.clients.producer.RecordMetadata;
+ import org.apache.kafka.common.config.SaslConfigs;
+ import org.apache.kafka.common.config.SslConfigs;
++import org.apache.kafka.common.utils.Time;
+ import org.junit.Assert;
+ import org.junit.Test;
+
+ import kafka.server.KafkaConfig;
+-import kafka.server.KafkaServerStartable;
++import scala.Some;
+
+ /**
+ * A simple test that starts a Kafka broker, creates "test" and "dev" topics, sends a message to them and consumes it. We also plug in a
+@@ -63,7 +65,7 @@
+ @org.junit.Ignore("Causing JVM to abort on some platforms")
+ public class KafkaRangerAuthorizerSASLSSLTest {
+
+- private static KafkaServerStartable kafkaServer;
++ private static KafkaServer kafkaServer;
+ private static TestingServer zkServer;
+ private static int port;
+ private static String serviceKeystorePath;
+@@ -139,7 +141,7 @@ public static void setup() throws Exception {
+ UserGroupInformation.createUserForTesting("alice", new String[] {"IT"});
+
+ KafkaConfig config = new KafkaConfig(props);
+- kafkaServer = new KafkaServerStartable(config);
++ kafkaServer = new KafkaServer(config, Time.SYSTEM, new Some<String>("KafkaRangerAuthorizerSASLSSLTest"), false);
+ kafkaServer.startup();
+
+ // Create some topics
+diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java
+index 113ac36869..d24ee1e570 100644
+--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java
++++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java
+@@ -29,6 +29,7 @@
+ import java.util.Properties;
+ import java.util.concurrent.Future;
+
++import kafka.server.KafkaServer;
+ import org.apache.commons.io.FileUtils;
+ import org.apache.curator.test.TestingServer;
+ import org.apache.hadoop.security.UserGroupInformation;
+@@ -41,11 +42,12 @@
+ import org.apache.kafka.clients.producer.ProducerRecord;
+ import org.apache.kafka.clients.producer.RecordMetadata;
+ import org.apache.kafka.common.config.SslConfigs;
++import org.apache.kafka.common.utils.Time;
+ import org.junit.Assert;
+ import org.junit.Test;
+
+ import kafka.server.KafkaConfig;
+-import kafka.server.KafkaServerStartable;
++import scala.Some;
+
+ /**
+ * A simple test that starts a Kafka broker, creates "test" and "dev" topics, sends a message to them and consumes it. We also plug in a
+@@ -65,7 +67,7 @@
+ */
+ public class KafkaRangerAuthorizerTest {
+
+- private static KafkaServerStartable kafkaServer;
++ private static KafkaServer kafkaServer;
+ private static TestingServer zkServer;
+ private static int port;
+ private static String serviceKeystorePath;
+@@ -133,7 +135,7 @@ public static void setup() throws Exception {
+ UserGroupInformation.createUserForTesting(serviceDN, new String[] {"IT"});
+
+ KafkaConfig config = new KafkaConfig(props);
+- kafkaServer = new KafkaServerStartable(config);
++ kafkaServer = new KafkaServer(config, Time.SYSTEM, new Some<String>("KafkaRangerAuthorizerTest"), false);
+ kafkaServer.startup();
+
+ // Create some topics
+diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerTopicCreationTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerTopicCreationTest.java
+index a12817ebab..201064970b 100644
+--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerTopicCreationTest.java
++++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerTopicCreationTest.java
+@@ -18,7 +18,7 @@
+ package org.apache.ranger.authorization.kafka.authorizer;
+
+ import kafka.server.KafkaConfig;
+-import kafka.server.KafkaServerStartable;
++import kafka.server.KafkaServer;
+ import org.apache.curator.test.InstanceSpec;
+ import org.apache.curator.test.TestingServer;
+ import org.apache.hadoop.security.UserGroupInformation;
+@@ -29,10 +29,12 @@
+ import org.apache.kafka.clients.admin.CreateTopicsResult;
+ import org.apache.kafka.clients.admin.NewTopic;
+ import org.apache.kafka.common.KafkaFuture;
++import org.apache.kafka.common.utils.Time;
+ import org.apache.kerby.kerberos.kerb.server.SimpleKdcServer;
+ import org.junit.Test;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
++import scala.Some;
+
+ import java.io.File;
+ import java.net.ServerSocket;
+@@ -49,7 +51,7 @@
+ public class KafkaRangerTopicCreationTest {
+ private final static Logger LOG = LoggerFactory.getLogger(KafkaRangerTopicCreationTest.class);
+
+- private static KafkaServerStartable kafkaServer;
++ private static KafkaServer kafkaServer;
+ private static TestingServer zkServer;
+ private static int port;
+ private static Path tempDir;
+@@ -121,7 +123,7 @@ public static void setup() throws Exception {
+ UserGroupInformation.createUserForTesting("kafka/localhost@kafka.apache.org", new String[] {"IT"});
+
+ KafkaConfig config = new KafkaConfig(props);
+- kafkaServer = new KafkaServerStartable(config);
++ kafkaServer = new KafkaServer(config, Time.SYSTEM, new Some<String>("KafkaRangerTopicCreationTest"), false);
+ kafkaServer.startup();
+ }
+
diff --git a/bigtop-packages/src/common/ranger/patch2-update-solr.diff b/bigtop-packages/src/common/ranger/patch2-update-solr.diff
new file mode 100644
index 00000000..d5b21e41
--- /dev/null
+++ b/bigtop-packages/src/common/ranger/patch2-update-solr.diff
@@ -0,0 +1,22 @@
+diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java
+index 200d03712..0147df423 100644
+--- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java
++++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java
+@@ -428,7 +428,7 @@ public void prepare(ResponseBuilder rb) throws IOException {
+ if (httpServletRequest == null) {
+ SolrCore solrCore = rb.req.getCore();
+ StringBuilder builder = new StringBuilder("Unable to locate HttpServletRequest");
+- if (solrCore != null && !solrCore.getSolrConfig().getBool("requestDispatcher/requestParsers/@addHttpRequestToContext", true)) {
++ if (solrCore != null && !solrCore.getSolrConfig().get("requestDispatcher/requestParsers/@addHttpRequestToContext").boolVal(true)) {
+ builder.append(", ensure requestDispatcher/requestParsers/@addHttpRequestToContext is set to true in solrconfig.xml");
+ }
+ throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED, builder.toString());
+@@ -732,7 +732,7 @@ private final String getUserName(SolrQueryRequest req) {
+ HttpServletRequest httpServletRequest = (HttpServletRequest) req.getContext().get("httpRequest");
+ if (httpServletRequest == null) {
+ StringBuilder builder = new StringBuilder("Unable to locate HttpServletRequest");
+- if (solrCore != null && !solrCore.getSolrConfig().getBool("requestDispatcher/requestParsers/@addHttpRequestToContext", true)) {
++ if (solrCore != null && !solrCore.getSolrConfig().get("requestDispatcher/requestParsers/@addHttpRequestToContext").boolVal(true)) {
+ builder.append(", ensure requestDispatcher/requestParsers/@addHttpRequestToContext is set to true in solrconfig.xml");
+ }
+ throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED, builder.toString());
diff --git a/bigtop-packages/src/rpm/ranger/SPECS/ranger.spec b/bigtop-packages/src/rpm/ranger/SPECS/ranger.spec
new file mode 100644
index 00000000..dad58f79
--- /dev/null
+++ b/bigtop-packages/src/rpm/ranger/SPECS/ranger.spec
@@ -0,0 +1,426 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+%undefine _missing_build_ids_terminate_build
+
+%define ranger_name ranger
+%define etc_ranger /etc/%{ranger_name}
+%define ranger_home /usr/lib/%{ranger_name}
+%define ranger_user_home /var/lib/%{ranger_name}
+%define config_ranger %{etc_ranger}/conf
+%define config_ranger_dist %{config_ranger}.dist
+
+%define usr_lib_ranger /usr/lib/%{ranger_name}
+%define var_log_ranger /var/log/%{ranger_name}
+%define var_run_ranger /var/run/%{ranger_name}
+%define usr_bin /usr/bin
+%define man_dir %{ranger_home}/man
+%define ranger_services ranger-admin ranger-usersync ranger-tagsync ranger-kms
+%define ranger_dist build
+
+%define hadoop_home /usr/lib/hadoop
+%define hive_home /usr/lib/hive
+%define knox_home /usr/lib/knox
+%define storm_home /usr/lib/storm
+%define hbase_home /usr/lib/hbase
+%define kafka_home /usr/lib/kafka
+%define atlas_home /usr/lib/atlas
+
+
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+%define __os_install_post \
+ %{_rpmconfigdir}/brp-compress ; \
+ %{_rpmconfigdir}/brp-strip-static-archive %{__strip} ; \
+ %{_rpmconfigdir}/brp-strip-comment-note %{__strip} %{__objdump} ; \
+ /usr/lib/rpm/brp-python-bytecompile ; \
+ %{nil}
+
+%define doc_ranger %{_docdir}/%{ranger_name}-%{ranger_version}
+%define alternatives_cmd alternatives
+%global initd_dir %{_sysconfdir}/rc.d/init.d
+
+%endif
+
+%if %{?suse_version:1}0
+
+# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed
+%if 0%{suse_version} > 1130
+%define suse_check \# Define an empty suse_check for compatibility with older sles
+%endif
+
+%define doc_ranger %{_docdir}/%{ranger_name}
+%define alternatives_cmd update-alternatives
+
+%global initd_dir %{_sysconfdir}/rc.d
+
+%define __os_install_post \
+ %{suse_check} ; \
+ /usr/lib/rpm/brp-compress ; \
+ %{nil}
+
+%endif
+
+
+%if 0%{?mgaversion}
+%define alternatives_cmd update-alternatives
+%global initd_dir %{_sysconfdir}/rc.d/init.d
+%endif
+
+# Even though we split the RPM into arch and noarch, it still will build and install
+# the entirety of hadoop. Defining this tells RPM not to fail the build
+# when it notices that we didn't package most of the installed files.
+%define _unpackaged_files_terminate_build 0
+
+# RPM searches perl files for dependancies and this breaks for non packaged perl lib
+# like thrift so disable this
+%define _use_internal_dependency_generator 0
+
+# Disable debuginfo package
+%define debug_package %{nil}
+
+Name: %{ranger_name}
+Version: %{ranger_base_version}
+Release: %{ranger_release}
+Summary: Ranger is a security framework for securing Hadoop data
+License: Apache License v2.0
+URL: http://ranger.apache.org/
+Group: Development/Libraries
+Buildroot: %{_topdir}/INSTALL/%{ranger_name}-%{version}
+Source0: release-%{ranger_name}-%{ranger_base_version}.tar.gz
+Source1: do-component-build
+Source2: install_%{ranger_name}.sh
+#BIGTOP_PATCH_FILES
+Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
+Requires: psmisc
+# Sadly, Sun/Oracle JDK in RPM form doesn't provide libjvm.so, which means we have
+# to set AutoReq to no in order to minimize confusion. Not ideal, but seems to work.
+# I wish there was a way to disable just one auto dependency (libjvm.so)
+AutoReq: no
+
+
+%if %{?suse_version:1}0
+# Required for init scripts
+Requires: sh-utils, insserv
+%endif
+
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+# Required for init scripts
+Requires: sh-utils, redhat-lsb
+%endif
+
+%if 0%{?mgaversion}
+Requires: chkconfig, xinetd-simple-services, zlib, initscripts
+%endif
+
+%description
+Ranger is a framework to secure hadoop data
+
+%package admin
+Summary: Web Interface for Ranger
+Group: System/Daemons
+Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
+Requires: psmisc
+# Sadly, Sun/Oracle JDK in RPM form doesn't provide libjvm.so, which means we have
+# to set AutoReq to no in order to minimize confusion. Not ideal, but seems to work.
+# I wish there was a way to disable just one auto dependency (libjvm.so)
+AutoReq: no
+
+
+%if %{?suse_version:1}0
+# Required for init scripts
+Requires: sh-utils, insserv
+%endif
+
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+# Required for init scripts
+Requires: sh-utils, redhat-lsb
+%endif
+
+%if 0%{?mgaversion}
+Requires: chkconfig, xinetd-simple-services, zlib, initscripts
+%endif
+
+%description admin
+Ranger-admin is admin component associated with the Ranger framework
+
+%package usersync
+Summary: Synchronize User/Group information from Corporate LD/AD or Unix
+Group: System/Daemons
+Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
+Requires: psmisc
+# Sadly, Sun/Oracle JDK in RPM form doesn't provide libjvm.so, which means we have
+# to set AutoReq to no in order to minimize confusion. Not ideal, but seems to work.
+# I wish there was a way to disable just one auto dependency (libjvm.so)
+AutoReq: no
+
+
+%if %{?suse_version:1}0
+# Required for init scripts
+Requires: sh-utils, insserv
+%endif
+
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+# Required for init scripts
+Requires: sh-utils, redhat-lsb
+%endif
+
+%if 0%{?mgaversion}
+Requires: chkconfig, xinetd-simple-services, zlib, initscripts
+%endif
+
+%description usersync
+Ranger-usersync is user/group synchronization component associated with the Ranger framework
+
+%package kms
+Summary: Key Management Server
+Group: System/Daemons
+Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
+Requires: psmisc
+# Sadly, Sun/Oracle JDK in RPM form doesn't provide libjvm.so, which means we have
+# to set AutoReq to no in order to minimize confusion. Not ideal, but seems to work.
+# I wish there was a way to disable just one auto dependency (libjvm.so)
+AutoReq: no
+
+
+%if %{?suse_version:1}0
+# Required for init scripts
+Requires: sh-utils, insserv
+%endif
+
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+# Required for init scripts
+Requires: sh-utils, redhat-lsb
+%endif
+
+%if 0%{?mgaversion}
+Requires: chkconfig, xinetd-simple-services, zlib, initscripts
+%endif
+
+%description kms
+Ranger-kms is key management server component associated with the Ranger framework
+
+
+%package tagsync
+Summary: Tag Synchronizer
+Group: System/Daemons
+Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
+Requires: psmisc
+# Sadly, Sun/Oracle JDK in RPM form doesn't provide libjvm.so, which means we have
+# to set AutoReq to no in order to minimize confusion. Not ideal, but seems to work.
+# I wish there was a way to disable just one auto dependency (libjvm.so)
+AutoReq: no
+%if %{?suse_version:1}0
+# Required for init scripts
+Requires: sh-utils, insserv
+%endif
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+# Required for init scripts
+Requires: sh-utils, redhat-lsb
+%endif
+%if 0%{?mgaversion}
+Requires: chkconfig, xinetd-simple-services, zlib, initscripts
+%endif
+%description tagsync
+Ranger-tagsync is tag synchronizer component associated with the Ranger framework
+
+%package hdfs-plugin
+Summary: ranger plugin for hdfs
+Group: System/Daemons
+
+%description hdfs-plugin
+Ranger HDFS plugnin component runs within namenode to provoide enterprise security using ranger framework
+
+%package yarn-plugin
+Summary: ranger plugin for yarn
+Group: System/Daemons
+
+%description yarn-plugin
+Ranger YARN plugnin component runs within namenode to provoide enterprise security using ranger framework
+
+%package hive-plugin
+Summary: ranger plugin for hive
+Group: System/Daemons
+
+%description hive-plugin
+Ranger Hive plugnin component runs within hiveserver2 to provoide enterprise security using ranger framework
+
+%package hbase-plugin
+Summary: ranger plugin for hbase
+Group: System/Daemons
+
+%description hbase-plugin
+Ranger HBASE plugnin component runs within master and regional servers as co-processor to provoide enterprise security using ranger framework
+
+%package knox-plugin
+Summary: ranger plugin for knox
+Group: System/Daemons
+
+%description knox-plugin
+Ranger KNOX plugnin component runs within knox proxy server to provoide enterprise security using ranger framework
+
+%package storm-plugin
+Summary: ranger plugin for storm
+Group: System/Daemons
+
+%description storm-plugin
+Ranger STORM plugnin component runs within storm to provoide enterprise security using ranger framework
+
+%package kafka-plugin
+Summary: ranger plugin for kafka
+Group: System/Daemons
+
+%description kafka-plugin
+Ranger KAFKA plugnin component runs within namenode to provoide enterprise security using ranger framework
+
+%package atlas-plugin
+Summary: ranger plugin for atlas
+Group: System/Daemons
+
+%description atlas-plugin
+Ranger ATLAS plugnin component runs within namenode to provoide enterprise security using ranger framework
+
+%prep
+%setup -q -n %{ranger_name}-release-%{ranger_name}-%{ranger_base_version}
+
+#BIGTOP_PATCH_COMMANDS
+
+%build
+bash %{SOURCE1}
+
+%clean
+%__rm -rf $RPM_BUILD_ROOT
+
+#########################
+#### INSTALL SECTION ####
+#########################
+%install
+%__rm -rf $RPM_BUILD_ROOT
+echo
+for comp in admin usersync kms tagsync hdfs-plugin yarn-plugin hive-plugin hbase-plugin knox-plugin storm-plugin kafka-plugin atlas-plugin
+do
+ env RANGER_VERSION=%{ranger_base_version} /bin/bash %{SOURCE2} \
+ --prefix=$RPM_BUILD_ROOT \
+ --build-dir=%{ranger_dist} \
+ --component=${comp} \
+ --doc-dir=$RPM_BUILD_ROOT/%{doc_ranger}
+echo;echo
+done
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
+
+%pre admin
+getent group ranger >/dev/null || groupadd -r ranger
+getent passwd ranger >/dev/null || useradd -c "Ranger" -s /bin/bash -g ranger -m -d /var/lib/%{ranger_name} ranger 2> /dev/null || :
+
+%pre usersync
+getent group ranger >/dev/null || groupadd -r ranger
+getent passwd ranger >/dev/null || useradd -c "Ranger" -s /bin/bash -g ranger -m -d /var/lib/%{ranger_name} ranger 2> /dev/null || :
+
+%pre kms
+getent group ranger >/dev/null || groupadd -r ranger
+getent passwd ranger >/dev/null || useradd -c "Ranger" -s /bin/bash -g ranger -m -d /var/lib/%{ranger_name} ranger 2> /dev/null || :
+
+%pre tagsync
+getent group ranger >/dev/null || groupadd -r ranger
+getent passwd ranger >/dev/null || useradd -c "Ranger" -s /bin/bash -g ranger -m -d /var/lib/%{ranger_name} ranger 2> /dev/null || :
+
+%post usersync
+if [ -f %{usr_lib_ranger}-usersync/native/credValidator.uexe ]; then
+ chmod u+s %{usr_lib_ranger}-usersync/native/credValidator.uexe
+fi
+
+%preun
+
+%postun
+
+#######################
+#### FILES SECTION ####
+#######################
+%files admin
+%defattr(-,root,root,755)
+%attr(0775,ranger,ranger) %{var_run_ranger}/admin
+%attr(0775,ranger,ranger) %{var_log_ranger}/admin
+%{usr_lib_ranger}-admin
+%config(noreplace) /etc/ranger/admin/conf.dist
+
+%files usersync
+%defattr(-,root,root,755)
+%attr(0775,ranger,ranger) %{var_run_ranger}/usersync
+%attr(0775,ranger,ranger) %{var_log_ranger}/usersync
+%{usr_lib_ranger}-usersync
+%attr(750,root,ranger) %{usr_lib_ranger}-usersync/native/credValidator.uexe
+%config(noreplace) /etc/ranger/usersync/conf.dist
+
+%files kms
+%defattr(-,root,root,755)
+%attr(0775,ranger,ranger) %{var_run_ranger}/kms
+%attr(0775,ranger,ranger) %{var_log_ranger}/kms
+%{usr_lib_ranger}-kms
+%config(noreplace) /etc/ranger/kms/conf.dist
+
+%files tagsync
+%defattr(-,root,root,755)
+%attr(0775,ranger,ranger) %{var_run_ranger}/tagsync
+%attr(0775,ranger,ranger) %{var_log_ranger}/tagsync
+%{usr_lib_ranger}-tagsync
+%config(noreplace) /etc/ranger/tagsync/conf.dist
+
+%files hdfs-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-hdfs-plugin
+%{hadoop_home}/lib
+
+%files yarn-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-yarn-plugin
+%{hadoop_home}/lib
+
+%files hive-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-hive-plugin
+%{hive_home}/lib
+
+%files hbase-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-hbase-plugin
+%{hbase_home}/lib
+
+%files knox-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-knox-plugin
+%{knox_home}/lib
+
+%files storm-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-storm-plugin
+%{storm_home}/lib
+
+%files kafka-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-kafka-plugin
+%{kafka_home}/lib
+
+%files atlas-plugin
+%defattr(-,root,root,755)
+%{usr_lib_ranger}-atlas-plugin
+%{atlas_home}/lib
diff --git a/bigtop.bom b/bigtop.bom
index f6f1b6b9..215cf305 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -115,7 +115,7 @@ bigtop {
'bigtop-jsvc':['hadoop'],
zookeeper:['hadoop', 'hbase', 'kafka'],
hadoop:['hbase', 'hive', 'tez', 'sqoop',
- 'oozie', 'solr', 'spark',
+ 'oozie', 'solr', 'spark', 'ranger',
'phoenix', 'alluxio', 'ycsb', 'zeppelin'
],
hbase:['phoenix', 'ycsb', 'hive'],
@@ -383,5 +383,14 @@ bigtop {
url { site = "https://github.com/elastic/kibana/archive"
archive = site }
}
+ 'ranger' {
+ name = 'ranger'
+ relNotes = 'Apache Ranger'
+ version { base = '2.2.0'; pkg = base; release = 1 }
+ tarball { destination = "release-$name-${version.base}.tar.gz"
+ source = destination }
+ url { site = "https://github.com/apache/ranger/archive/refs/tags"
+ archive = site }
+ }
}
}