You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2020/12/02 18:27:07 UTC
[ranger] branch master updated: RANGER-3096: Kafka upgrade to 2.5.0
and Scala to 2.12
This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new cb702d6 RANGER-3096: Kafka upgrade to 2.5.0 and Scala to 2.12
cb702d6 is described below
commit cb702d6cc24e7eff633e9de37acbcbaad48ee5fc
Author: Andras Katona <ak...@cloudera.com>
AuthorDate: Tue Dec 1 14:04:18 2020 +0100
RANGER-3096: Kafka upgrade to 2.5.0 and Scala to 2.12
Signed-off-by: Ramesh Mani <rm...@cloudera.com>
---
dev-support/ranger-docker/.env | 2 +-
dev-support/ranger-docker/Dockerfile.ranger-kafka | 8 ++++----
dev-support/ranger-docker/README.md | 2 +-
.../authorizer/KafkaRangerAuthorizerGSSTest.java | 7 ++++++-
.../KafkaRangerAuthorizerSASLSSLTest.java | 23 ++++++++++++++++------
.../authorizer/KafkaRangerAuthorizerTest.java | 21 ++++++++++++++------
.../kafka/authorizer/KafkaTestUtils.java | 23 +++++++++-------------
plugin-schema-registry/pom.xml | 4 +---
pom.xml | 6 +++---
tagsync/pom.xml | 6 ++++++
10 files changed, 63 insertions(+), 39 deletions(-)
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
index 5968960..d280c10 100644
--- a/dev-support/ranger-docker/.env
+++ b/dev-support/ranger-docker/.env
@@ -8,4 +8,4 @@ POSTGRES_PASSWORD=rangerR0cks!
RANGER_VERSION=3.0.0-SNAPSHOT
HADOOP_VERSION=3.3.0
HBASE_VERSION=2.2.6
-KAFKA_VERSION=2.4.0
+KAFKA_VERSION=2.5.0
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka
index 72610d5..b09500b 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-kafka
+++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka
@@ -26,10 +26,10 @@ COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/
COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/
COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/
-RUN curl https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz --output /tmp/kafka_2.11-${KAFKA_VERSION}.tgz && \
- tar xvfz /tmp/kafka_2.11-${KAFKA_VERSION}.tgz --directory=/opt/ && \
- ln -s /opt/kafka_2.11-${KAFKA_VERSION} /opt/kafka && \
- rm -f /tmp/kafka_2.11-${KAFKA_VERSION}.tgz && \
+RUN curl https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.12-${KAFKA_VERSION}.tgz --output /tmp/kafka_2.12-${KAFKA_VERSION}.tgz && \
+ tar xvfz /tmp/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
+ ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
+ rm -f /tmp/kafka_2.12-${KAFKA_VERSION}.tgz && \
tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz --directory=/opt/ranger && \
ln -s /opt/ranger/ranger-${RANGER_VERSION}-kafka-plugin /opt/ranger/ranger-kafka-plugin && \
rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz && \
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 190b6db..19508ed 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -109,7 +109,7 @@ Docker files in this folder create docker images and run them to build Apache Ra
5.13. Execute following command to build Docker image **ranger-kafka**:
- docker build -f Dockerfile.ranger-kafka --build-arg RANGER_VERSION=`cat dist/version` --build-arg KAFKA_VERSION=2.4.0 -t ranger-kafka .
+ docker build -f Dockerfile.ranger-kafka --build-arg RANGER_VERSION=`cat dist/version` --build-arg KAFKA_VERSION=2.5.0 -t ranger-kafka .
This step includes downloading of Kafka tar ball, and can take a while to complete.
diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
index 0d3665d..fe600b1 100644
--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
+++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
@@ -42,6 +42,7 @@ import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kerby.kerberos.kerb.server.SimpleKdcServer;
import org.junit.Assert;
import org.junit.Test;
@@ -143,7 +144,11 @@ public class KafkaRangerAuthorizerGSSTest {
kafkaServer.startup();
// Create some topics
- KafkaTestUtils.createSomeTopics(zkServer.getConnectString());
+ final Properties adminProps = new Properties();
+ adminProps.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
+ adminProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ adminProps.put(SaslConfigs.SASL_MECHANISM, "GSSAPI");
+ KafkaTestUtils.createSomeTopics(adminProps);
}
private static void configureKerby(String baseDir) throws Exception {
diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java
index 4bcf078..6f45385 100644
--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java
+++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerSASLSSLTest.java
@@ -38,6 +38,7 @@ import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.config.SslConfigs;
import org.junit.Assert;
import org.junit.Test;
@@ -125,11 +126,11 @@ public class KafkaRangerAuthorizerSASLSSLTest {
props.put("offsets.topic.replication.factor", (short) 1);
props.put("offsets.topic.num.partitions", 1);
- props.put("ssl.keystore.location", serviceKeystorePath);
- props.put("ssl.keystore.password", "sspass");
- props.put("ssl.key.password", "skpass");
- props.put("ssl.truststore.location", truststorePath);
- props.put("ssl.truststore.password", "security");
+ props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, serviceKeystorePath);
+ props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "sspass");
+ props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "skpass");
+ props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststorePath);
+ props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
// Plug in Apache Ranger authorizer
props.put("authorizer.class.name", "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer");
@@ -142,7 +143,17 @@ public class KafkaRangerAuthorizerSASLSSLTest {
kafkaServer.startup();
// Create some topics
- KafkaTestUtils.createSomeTopics(zkServer.getConnectString());
+ final Properties adminProps = new Properties();
+ adminProps.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
+ adminProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL");
+ adminProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ // ssl
+ adminProps.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, serviceKeystorePath);
+ adminProps.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "sspass");
+ adminProps.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "skpass");
+ adminProps.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststorePath);
+ adminProps.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
+ KafkaTestUtils.createSomeTopics(adminProps);
}
@org.junit.AfterClass
diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java
index a042fc7..113ac36 100644
--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java
+++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerTest.java
@@ -115,11 +115,11 @@ public class KafkaRangerAuthorizerTest {
props.put("controlled.shutdown.enable", Boolean.TRUE.toString());
// Enable SSL
props.put("listeners", "SSL://localhost:" + port);
- props.put("ssl.keystore.location", serviceKeystorePath);
- props.put("ssl.keystore.password", "sspass");
- props.put("ssl.key.password", "skpass");
- props.put("ssl.truststore.location", truststorePath);
- props.put("ssl.truststore.password", "security");
+ props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, serviceKeystorePath);
+ props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "sspass");
+ props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "skpass");
+ props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststorePath);
+ props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
props.put("security.inter.broker.protocol", "SSL");
props.put("ssl.client.auth", "required");
props.put("offsets.topic.replication.factor", (short) 1);
@@ -137,7 +137,16 @@ public class KafkaRangerAuthorizerTest {
kafkaServer.startup();
// Create some topics
- KafkaTestUtils.createSomeTopics(zkServer.getConnectString());
+ final Properties adminProps = new Properties();
+ adminProps.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
+ adminProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL");
+ // ssl
+ adminProps.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, serviceKeystorePath);
+ adminProps.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "sspass");
+ adminProps.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "skpass");
+ adminProps.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststorePath);
+ adminProps.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
+ KafkaTestUtils.createSomeTopics(adminProps);
}
@org.junit.AfterClass
diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaTestUtils.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaTestUtils.java
index b703f95..dc82770 100644
--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaTestUtils.java
+++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaTestUtils.java
@@ -27,10 +27,12 @@ import java.security.KeyStore;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
+import java.util.Arrays;
import java.util.Date;
import java.util.Properties;
-import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.asn1.x500.style.RFC4519Style;
import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
@@ -39,12 +41,6 @@ import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import org.bouncycastle.operator.ContentSigner;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-import kafka.admin.RackAwareMode;
-import kafka.zk.AdminZkClient;
-import kafka.zk.KafkaZkClient;
-import kafka.zookeeper.ZooKeeperClient;
-import scala.Option;
-
public final class KafkaTestUtils {
public static String createAndStoreKey(String subjectName, String issuerName, BigInteger serial, String keystorePassword,
@@ -82,13 +78,12 @@ public final class KafkaTestUtils {
}
- static void createSomeTopics(String zkConnectString) {
- ZooKeeperClient zookeeperClient = new ZooKeeperClient(zkConnectString, 30000, 30000,
- 1, Time.SYSTEM, "kafka.server", "SessionExpireListener", Option.empty());
- try (KafkaZkClient kafkaZkClient = new KafkaZkClient(zookeeperClient, false, Time.SYSTEM)) {
- AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
- adminZkClient.createTopic("test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
- adminZkClient.createTopic("dev", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
+ static void createSomeTopics(Properties adminProps) {
+ try (AdminClient adminClient = AdminClient.create(adminProps)) {
+ adminClient.createTopics(Arrays.asList(
+ new NewTopic("test", 1, (short) 1),
+ new NewTopic("dev", 1, (short) 1)
+ ));
}
}
}
diff --git a/plugin-schema-registry/pom.xml b/plugin-schema-registry/pom.xml
index 28e8b7e..4a597d5 100644
--- a/plugin-schema-registry/pom.xml
+++ b/plugin-schema-registry/pom.xml
@@ -33,8 +33,6 @@
<properties>
<hadoop.version>3.1.1</hadoop.version>
- <kafka.version>2.1.0</kafka.version>
- <kafkaArtifact>kafka_2.11</kafkaArtifact>
<jersey.version>2.22.1</jersey.version>
<junit.version>4.5</junit.version>
<schema.registry.version>0.9.1</schema.registry.version>
@@ -124,7 +122,7 @@
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>${kafkaArtifact}</artifactId>
+ <artifactId>kafka_${scala.binary.version}</artifactId>
<version>${kafka.version}</version>
<exclusions>
<exclusion>
diff --git a/pom.xml b/pom.xml
index cd60038..6f7114c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -152,7 +152,7 @@
<jsr250.version>1.0</jsr250.version>
<jsr305.version>1.3.9</jsr305.version>
<junit.version>4.12</junit.version>
- <kafka.version>2.4.0</kafka.version>
+ <kafka.version>2.5.0</kafka.version>
<kerby.version>1.0.0</kerby.version>
<knox.gateway.version>1.4.0</knox.gateway.version>
<kylin.version>2.6.4</kylin.version>
@@ -174,8 +174,8 @@
<poi.version>4.1.2</poi.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<protobuf-java.version>2.5.0</protobuf-java.version>
- <scala.version>2.11.12</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.12.10</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
<scala.xml.version>1.0.4</scala.xml.version>
<security-agent-install-dir>hadoop-security/plugins</security-agent-install-dir>
<servlet.api.version>2.5</servlet.api.version>
diff --git a/tagsync/pom.xml b/tagsync/pom.xml
index d3afd07..eb79078 100644
--- a/tagsync/pom.xml
+++ b/tagsync/pom.xml
@@ -118,6 +118,12 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
<version>${atlas.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>