You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2026/02/11 19:26:03 UTC
(ranger) 01/01: RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer
This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch RANGER-5482-patch
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 54f64efe3da00622cbf821a0b3a4e3c5046a1604
Author: Ramesh Mani <rm...@apache.org>
AuthorDate: Wed Feb 11 11:24:15 2026 -0800
RANGER-5482:Create Ranger Audit Server with SOLR and HDFS as audit consumer
---
.gitignore | 6 +
.../audit/provider/AuditProviderFactory.java | 2 +
.../ranger/audit/provider/BaseAuditHandler.java | 96 +--
.../audit/utils/AbstractRangerAuditWriter.java | 18 +-
.../ranger/audit/utils/RangerJSONAuditWriter.java | 4 +
agents-audit/dest-auditserver/pom.xml | 63 ++
.../destination/RangerAuditServerDestination.java | 749 ++++++++++++++++++++
.../audit/destination/SolrAuditDestination.java | 4 +-
agents-audit/pom.xml | 1 +
.../plugin/audit/RangerDefaultAuditHandler.java | 29 +-
.../plugin/policyengine/RangerAccessResult.java | 10 +
.../audit/TestRangerDefaultAuditHandler.java | 6 +-
dev-support/ranger-docker/.dockerignore | 3 +
.../Dockerfile.ranger-audit-consumer-hdfs | 85 +++
.../Dockerfile.ranger-audit-consumer-solr | 86 +++
.../ranger-docker/Dockerfile.ranger-audit-server | 86 +++
dev-support/ranger-docker/README.md | 5 +
.../docker-compose.ranger-audit-server.yml | 168 +++++
.../ranger-docker/docker-compose.ranger-kafka.yml | 2 +-
.../scripts/admin/create-ranger-services.py | 23 +-
.../audit-server/ranger-audit-consumer-hdfs.sh | 138 ++++
.../audit-server/ranger-audit-consumer-solr.sh | 137 ++++
.../scripts/audit-server/ranger-audit-server.sh | 137 ++++
.../audit-server/service-check-functions.sh | 76 ++
.../ranger-docker/scripts/hadoop/hdfs-site.xml | 12 +
.../hadoop/ranger-hdfs-plugin-install.properties | 10 +-
.../hive/ranger-hive-plugin-install.properties | 10 +-
.../scripts/kafka/ranger-kafka-setup.sh | 61 +-
.../ranger-docker/scripts/kafka/ranger-kafka.sh | 11 +-
.../ranger-docker/scripts/kdc/entrypoint.sh | 15 +-
distro/pom.xml | 30 +
distro/src/main/assembly/hdfs-agent.xml | 1 +
distro/src/main/assembly/hive-agent.xml | 1 +
.../main/assembly/ranger-audit-consumer-hdfs.xml | 95 +++
.../main/assembly/ranger-audit-consumer-solr.xml | 93 +++
.../main/assembly/ranger-audit-server-service.xml | 93 +++
hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 5 +
hdfs-agent/conf/ranger-hdfs-audit.xml | 19 +-
hdfs-agent/pom.xml | 5 +
hdfs-agent/scripts/install.properties | 4 +
.../hadoop/RangerHdfsAuditHandler.java | 6 +-
hive-agent/conf/ranger-hive-audit-changes.cfg | 5 +
hive-agent/conf/ranger-hive-audit.xml | 19 +-
hive-agent/pom.xml | 5 +
hive-agent/scripts/install.properties | 4 +
pom.xml | 27 +
ranger-audit-server/pom.xml | 460 ++++++++++++
ranger-audit-server/ranger-audit-common/pom.xml | 127 ++++
.../ranger/audit/consumer/kafka/AuditConsumer.java | 78 +++
.../audit/consumer/kafka/AuditConsumerBase.java | 101 +++
.../audit/consumer/kafka/AuditConsumerFactory.java | 38 +
.../kafka/AuditConsumerRebalanceListener.java | 119 ++++
.../consumer/kafka/AuditConsumerRegistry.java | 143 ++++
.../apache/ranger/audit/server/AuditConfig.java | 90 +++
.../ranger/audit/server/AuditServerConstants.java | 99 +++
.../apache/ranger/audit/server/EmbeddedServer.java | 780 +++++++++++++++++++++
.../ranger/audit/utils/AuditMessageQueueUtils.java | 348 +++++++++
.../audit/utils/AuditServerLogFormatter.java | 148 ++++
.../ranger/audit/utils/AuditServerUtils.java | 286 ++++++++
.../ranger-audit-consumer-hdfs/pom.xml | 345 +++++++++
.../scripts/start-consumer-hdfs.sh | 164 +++++
.../scripts/stop-consumer-hdfs.sh | 72 ++
.../audit/consumer/HdfsConsumerApplication.java | 59 ++
.../ranger/audit/consumer/HdfsConsumerManager.java | 205 ++++++
.../audit/consumer/kafka/AuditHDFSConsumer.java | 497 +++++++++++++
.../audit/consumer/kafka/AuditRouterHDFS.java | 354 ++++++++++
.../apache/ranger/audit/rest/HealthCheckREST.java | 112 +++
.../ranger/audit/server/HdfsConsumerConfig.java | 146 ++++
.../src/main/resources/conf/core-site.xml | 39 ++
.../src/main/resources/conf/hdfs-site.xml | 52 ++
.../src/main/resources/conf/logback.xml | 71 ++
.../conf/ranger-audit-consumer-hdfs-site.xml | 156 +++++
.../src/main/webapp/WEB-INF/applicationContext.xml | 39 ++
.../src/main/webapp/WEB-INF/web.xml | 60 ++
.../ranger-audit-consumer-solr/pom.xml | 336 +++++++++
.../scripts/start-consumer-solr.sh | 163 +++++
.../scripts/stop-consumer-solr.sh | 72 ++
.../audit/consumer/SolrConsumerApplication.java | 65 ++
.../ranger/audit/consumer/SolrConsumerManager.java | 210 ++++++
.../audit/consumer/kafka/AuditSolrConsumer.java | 457 ++++++++++++
.../apache/ranger/audit/rest/HealthCheckREST.java | 112 +++
.../ranger/audit/server/SolrConsumerConfig.java | 71 ++
.../src/main/resources/conf/logback.xml | 76 ++
.../conf/ranger-audit-consumer-solr-site.xml | 204 ++++++
.../src/main/webapp/WEB-INF/applicationContext.xml | 39 ++
.../src/main/webapp/WEB-INF/web.xml | 60 ++
.../ranger-audit-server-service/pom.xml | 383 ++++++++++
.../scripts/start-audit-server.sh | 162 +++++
.../scripts/stop-audit-server.sh | 72 ++
.../ranger/audit/producer/AuditDestinationMgr.java | 102 +++
.../audit/producer/kafka/AuditMessageQueue.java | 277 ++++++++
.../ranger/audit/producer/kafka/AuditProducer.java | 132 ++++
.../audit/producer/kafka/AuditRecoveryManager.java | 222 ++++++
.../audit/producer/kafka/AuditRecoveryRetry.java | 362 ++++++++++
.../audit/producer/kafka/AuditRecoveryWriter.java | 319 +++++++++
.../org/apache/ranger/audit/rest/AuditREST.java | 219 ++++++
.../ranger/audit/security/AuditAuthEntryPoint.java | 52 ++
.../audit/security/AuditDelegationTokenFilter.java | 316 +++++++++
.../ranger/audit/security/AuditJwtAuthFilter.java | 196 ++++++
.../ranger/audit/security/FilterChainWrapper.java | 159 +++++
.../ranger/audit/security/NullServletContext.java | 240 +++++++
.../audit/server/AuditServerApplication.java | 61 ++
.../ranger/audit/server/AuditServerConfig.java | 71 ++
.../src/main/resources/conf/logback.xml | 71 ++
.../resources/conf/ranger-audit-server-site.xml | 359 ++++++++++
.../src/main/webapp/WEB-INF/applicationContext.xml | 54 ++
.../webapp/WEB-INF/security-applicationContext.xml | 63 ++
.../src/main/webapp/WEB-INF/web.xml | 73 ++
ranger-audit-server/scripts/README.md | 549 +++++++++++++++
ranger-audit-server/scripts/start-all-services.sh | 78 +++
ranger-audit-server/scripts/stop-all-services.sh | 66 ++
111 files changed, 14074 insertions(+), 102 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5bd45411b..92e70d43f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,9 @@ winpkg/target
.python-version
/security-admin/src/main/webapp/react-webapp/node_modules
**/target
+
+# Runtime logs and process files
+logs/
+*.log
+*.pid
+catalina.out
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index f6e63c23d..2fca68627 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -439,6 +439,8 @@ private AuditHandler getProviderFromConfig(Properties props, String propPrefix,
provider = createDestination("org.apache.ranger.audit.provider.kafka.KafkaAuditProvider");
} else if (providerName.equalsIgnoreCase("log4j")) {
provider = createDestination("org.apache.ranger.audit.destination.Log4JAuditDestination");
+ } else if (providerName.equalsIgnoreCase("auditserver")) {
+ provider = createDestination("org.apache.ranger.audit.destination.RangerAuditServerDestination");
} else if (providerName.equalsIgnoreCase("batch")) {
provider = getAuditProvider(props, propPrefix, consumer);
} else if (providerName.equalsIgnoreCase("async")) {
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index 32ea8b9c3..5a23b37e5 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -74,16 +74,16 @@ public abstract class BaseAuditHandler implements AuditHandler {
int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
long lastErrorLogMS;
- long totalCount;
- long totalSuccessCount;
- long totalFailedCount;
- long totalStashedCount;
- long totalDeferredCount;
- long lastIntervalCount;
- long lastIntervalSuccessCount;
- long lastIntervalFailedCount;
- long lastStashedCount;
- long lastDeferredCount;
+ AtomicLong totalCount = new AtomicLong(0);
+ AtomicLong totalSuccessCount = new AtomicLong(0);
+ AtomicLong totalFailedCount = new AtomicLong(0);
+ AtomicLong totalStashedCount = new AtomicLong(0);
+ AtomicLong totalDeferredCount = new AtomicLong(0);
+ AtomicLong lastIntervalCount = new AtomicLong(0);
+ AtomicLong lastIntervalSuccessCount = new AtomicLong(0);
+ AtomicLong lastIntervalFailedCount = new AtomicLong(0);
+ AtomicLong lastStashedCount = new AtomicLong(0);
+ AtomicLong lastDeferredCount = new AtomicLong(0);
boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
long lastStatusLogTime = System.currentTimeMillis();
@@ -237,61 +237,51 @@ public String getFinalPath() {
}
public long addTotalCount(int count) {
- totalCount += count;
-
- return totalCount;
+ return totalCount.addAndGet(count);
}
public long addSuccessCount(int count) {
- totalSuccessCount += count;
-
- return totalSuccessCount;
+ return totalSuccessCount.addAndGet(count);
}
public long addFailedCount(int count) {
- totalFailedCount += count;
-
- return totalFailedCount;
+ return totalFailedCount.addAndGet(count);
}
public long addStashedCount(int count) {
- totalStashedCount += count;
-
- return totalStashedCount;
+ return totalStashedCount.addAndGet(count);
}
public long addDeferredCount(int count) {
- totalDeferredCount += count;
-
- return totalDeferredCount;
+ return totalDeferredCount.addAndGet(count);
}
public long getTotalCount() {
- return totalCount;
+ return totalCount.get();
}
public long getTotalSuccessCount() {
- return totalSuccessCount;
+ return totalSuccessCount.get();
}
public long getTotalFailedCount() {
- return totalFailedCount;
+ return totalFailedCount.get();
}
public long getTotalStashedCount() {
- return totalStashedCount;
+ return totalStashedCount.get();
}
public long getLastStashedCount() {
- return lastStashedCount;
+ return lastStashedCount.get();
}
public long getTotalDeferredCount() {
- return totalDeferredCount;
+ return totalDeferredCount.get();
}
public long getLastDeferredCount() {
- return lastDeferredCount;
+ return lastDeferredCount.get();
}
public boolean isStatusLogEnabled() {
@@ -312,21 +302,27 @@ public void logStatus() {
lastStatusLogTime = currTime;
nextStatusLogTime = currTime + statusLogIntervalMS;
- long diffCount = totalCount - lastIntervalCount;
- long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
- long diffFailed = totalFailedCount - lastIntervalFailedCount;
- long diffStashed = totalStashedCount - lastStashedCount;
- long diffDeferred = totalDeferredCount - lastDeferredCount;
+ long currentTotalCount = totalCount.get();
+ long currentSuccessCount = totalSuccessCount.get();
+ long currentFailedCount = totalFailedCount.get();
+ long currentStashedCount = totalStashedCount.get();
+ long currentDeferredCount = totalDeferredCount.get();
+
+ long diffCount = currentTotalCount - lastIntervalCount.get();
+ long diffSuccess = currentSuccessCount - lastIntervalSuccessCount.get();
+ long diffFailed = currentFailedCount - lastIntervalFailedCount.get();
+ long diffStashed = currentStashedCount - lastStashedCount.get();
+ long diffDeferred = currentDeferredCount - lastDeferredCount.get();
if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 && diffStashed == 0 && diffDeferred == 0) {
return;
}
- lastIntervalCount = totalCount;
- lastIntervalSuccessCount = totalSuccessCount;
- lastIntervalFailedCount = totalFailedCount;
- lastStashedCount = totalStashedCount;
- lastDeferredCount = totalDeferredCount;
+ lastIntervalCount.set(currentTotalCount);
+ lastIntervalSuccessCount.set(currentSuccessCount);
+ lastIntervalFailedCount.set(currentFailedCount);
+ lastStashedCount.set(currentStashedCount);
+ lastDeferredCount.set(currentDeferredCount);
if (statusLogEnabled) {
String finalPath = "";
@@ -475,6 +471,12 @@ public void logFailedEventJSON(Collection<String> events, Throwable excp) {
}
private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) {
+ long currentTotalCount = totalCount.get();
+ long currentTotalSuccessCount = totalSuccessCount.get();
+ long currentTotalFailedCount = totalFailedCount.get();
+ long currentTotalStashedCount = totalStashedCount.get();
+ long currentTotalDeferredCount = totalDeferredCount.get();
+
String msg = "Audit Status Log: name="
+ getName()
+ finalPath
@@ -489,14 +491,14 @@ private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, lon
+ (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
: "")
+ ", totalEvents="
- + totalCount
- + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount)
+ + currentTotalCount
+ + (currentTotalSuccessCount > 0 ? (", totalSuccessCount=" + currentTotalSuccessCount)
: "")
- + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount)
+ + (currentTotalFailedCount > 0 ? (", totalFailedCount=" + currentTotalFailedCount)
: "")
- + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount)
+ + (currentTotalStashedCount > 0 ? (", totalStashedCount=" + currentTotalStashedCount)
: "")
- + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
+ + (currentTotalDeferredCount > 0 ? (", totalDeferredCount=" + currentTotalDeferredCount)
: "");
LOG.info(msg);
}
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
index df707d101..d1ac21051 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
@@ -133,13 +133,15 @@ public void createFileSystemFolders() throws Exception {
String defaultPath = fullPath;
+ fileSystemScheme = getFileSystemScheme();
+
conf = createConfiguration();
URI uri = URI.create(fullPath);
- fileSystem = FileSystem.get(uri, conf);
- auditPath = new Path(fullPath);
- fileSystemScheme = getFileSystemScheme();
+ fileSystem = FileSystem.get(uri, conf);
+
+ auditPath = new Path(fullPath);
logger.info("Checking whether log file exists. {} Path={}, UGI={}", fileSystemScheme, fullPath, MiscUtil.getUGILoginUser());
@@ -195,6 +197,9 @@ public void createParents(Path pathLogfile, FileSystem fileSystem) throws Except
if (parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) {
fileSystem.mkdirs(parentPath);
+ logger.info("Successfully created parent folder: {}", parentPath);
+ } else {
+ logger.info("Parent folder already exists or not required: {}", parentPath);
}
}
@@ -308,14 +313,17 @@ public PrintWriter createWriter() throws Exception {
if (!appendMode) {
// Create the file to write
- logger.info("Creating new log file. auditPath = {}", fullPath);
-
createFileSystemFolders();
+ logger.info("Creating new log file. fullPath = {}", fullPath);
+
ostream = fileSystem.create(auditPath);
+ logger.info("Successfully created {} output stream for file: {}", fileSystemScheme, fullPath);
}
logWriter = new PrintWriter(ostream);
isHFlushCapableStream = ostream.hasCapability(StreamCapabilities.HFLUSH);
+
+ logger.info("{} audit writer initialized successfully. File: {}, HFlush capable: {}", fileSystemScheme, fullPath, isHFlushCapableStream);
}
logger.debug("<== AbstractRangerAuditWriter.createWriter()");
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
index ab796b81d..c8b98fa75 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
@@ -102,9 +102,13 @@ public synchronized boolean logJSON(final Collection<String> events) throws Exce
} else {
out1 = getLogFileStream();
+ logger.debug("Writing {} audit events to HDFS file: {}", events.size(), currentFileName);
+
for (String event : events) {
out1.println(event);
}
+
+ logger.debug("Successfully wrote {} audit events to HDFS", events.size());
}
return out1;
diff --git a/agents-audit/dest-auditserver/pom.xml b/agents-audit/dest-auditserver/pom.xml
new file mode 100644
index 000000000..c489e1da9
--- /dev/null
+++ b/agents-audit/dest-auditserver/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+ <artifactId>ranger-audit-dest-auditserver</artifactId>
+ <packaging>jar</packaging>
+ <name>Ranger Audit Destination - auditserver</name>
+ <description>Ranger Audit Destination - auditserver</description>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <securesm.version>1.2</securesm.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <!-- Test -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java
new file mode 100644
index 000000000..572b3c430
--- /dev/null
+++ b/agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java
@@ -0,0 +1,749 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.destination;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.KerberosCredentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.config.Lookup;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.protocol.HttpContext;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+public class RangerAuditServerDestination extends AuditDestination {
+ public static final String PROP_AUDITSERVER_URL = "xasecure.audit.destination.auditserver.url";
+ public static final String PROP_AUDITSERVER_USER_NAME = "xasecure.audit.destination.auditserver.username";
+ public static final String PROP_AUDITSERVER_USER_PASSWORD = "xasecure.audit.destination.auditserver.password";
+ public static final String PROP_AUDITSERVER_AUTH_TYPE = "xasecure.audit.destination.auditserver.authentication.type";
+ public static final String PROP_AUDITSERVER_JWT_TOKEN = "xasecure.audit.destination.auditserver.jwt.token";
+ public static final String PROP_AUDITSERVER_JWT_TOKEN_FILE = "xasecure.audit.destination.auditserver.jwt.token.file";
+ public static final String PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS = "xasecure.audit.destination.auditserver.connection.timeout.ms";
+ public static final String PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS = "xasecure.audit.destination.auditserver.read.timeout.ms";
+ public static final String PROP_AUDITSERVER_MAX_CONNECTION = "xasecure.audit.destination.auditserver.max.connections";
+ public static final String PROP_AUDITSERVER_MAX_CONNECTION_PER_HOST = "xasecure.audit.destination.auditserver.max.connections.per.host";
+ public static final String PROP_AUDITSERVER_VALIDATE_INACTIVE_MS = "xasecure.audit.destination.auditserver.validate.inactivity.ms";
+ public static final String PROP_AUDITSERVER_POOL_RETRY_COUNT = "xasecure.audit.destination.auditserver.pool.retry.count";
+ public static final String GSON_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
+ public static final String REST_ACCEPTED_MIME_TYPE_JSON = "application/json";
+ public static final String REST_CONTENT_TYPE_MIME_TYPE_JSON = "application/json";
+ public static final String REST_HEADER_ACCEPT = "Accept";
+ public static final String REST_HEADER_CONTENT_TYPE = "Content-type";
+ public static final String REST_HEADER_AUTHORIZATION = "Authorization";
+ public static final String REST_RELATIVE_PATH_POST = "/api/audit/post";
+
+ // Authentication types
+ public static final String AUTH_TYPE_KERBEROS = "kerberos";
+ public static final String AUTH_TYPE_BASIC = "basic";
+ public static final String AUTH_TYPE_JWT = "jwt";
+
+ private static final Logger LOG = LoggerFactory.getLogger(RangerAuditServerDestination.class);
+ private static final String PROTOCOL_HTTPS = "https";
+ private volatile CloseableHttpClient httpClient;
+ private volatile Gson gsonBuilder;
+ private String httpURL;
+ private String authType;
+ private String jwtToken;
+
+ @Override
+ public void init(Properties props, String propPrefix) {
+ LOG.info("==> RangerAuditServerDestination:init()");
+ super.init(props, propPrefix);
+
+ this.authType = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_AUTH_TYPE);
+ if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) {
+ LOG.info("JWT authentication configured....");
+ initJwtToken();
+ } else if (StringUtils.isEmpty(authType)) {
+ // Authentication priority: JWT → Kerberos → Basic
+ try {
+ if (StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN)) ||
+ StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE))) {
+ this.authType = AUTH_TYPE_JWT;
+ initJwtToken();
+ } else if (isKerberosAuthenticated()) {
+ this.authType = AUTH_TYPE_KERBEROS;
+ } else if (StringUtils.isNotEmpty(MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME))) {
+ this.authType = AUTH_TYPE_BASIC;
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to auto-detect authentication type", e);
+ }
+ }
+
+ LOG.info("Audit destination authentication type: {}", authType);
+
+ if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) {
+ preAuthenticateKerberos();
+ }
+
+ this.httpClient = buildHTTPClient();
+ this.gsonBuilder = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create();
+ LOG.info("<== RangerAuditServerDestination:init()");
+ }
+
+ private void initJwtToken() {
+ LOG.info("==> RangerAuditServerDestination:initJwtToken()");
+
+ this.jwtToken = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN);
+ if (StringUtils.isEmpty(jwtToken)) {
+ String jwtTokenFile = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_JWT_TOKEN_FILE);
+ if (StringUtils.isNotEmpty(jwtTokenFile)) {
+ try {
+ this.jwtToken = readJwtTokenFromFile(jwtTokenFile);
+ LOG.info("JWT token loaded from file: {}", jwtTokenFile);
+ } catch (Exception e) {
+ LOG.error("Failed to read JWT token from file: {}", jwtTokenFile, e);
+ }
+ }
+ }
+
+ if (StringUtils.isEmpty(jwtToken)) {
+ LOG.warn("JWT authentication configured but no token found. Configure {} or {}", PROP_AUDITSERVER_JWT_TOKEN, PROP_AUDITSERVER_JWT_TOKEN_FILE);
+ } else {
+ LOG.info("JWT authentication initialized successfully");
+ }
+
+ LOG.info("<== RangerAuditServerDestination:initJwtToken()");
+ }
+
+ private String readJwtTokenFromFile(String tokenFile) throws IOException {
+ InputStream in = null;
+ try {
+ in = getFileInputStream(tokenFile);
+ if (in != null) {
+ String token = IOUtils.toString(in, Charset.defaultCharset()).trim();
+ return token;
+ } else {
+ throw new IOException("Unable to read JWT token file: " + tokenFile);
+ }
+ } finally {
+ close(in, tokenFile);
+ }
+ }
+
+ /**
+ * This method proactively obtains the TGT and service ticket for the audit server
+ * during initialization, so they are cached and ready when the first audit event arrives.
+ */
+ private void preAuthenticateKerberos() {
+ LOG.info("==> RangerAuditServerDestination:preAuthenticateKerberos()");
+
+ try {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+
+ if (ugi == null) {
+ LOG.warn("No UserGroupInformation available for Kerberos pre-authentication");
+ return;
+ }
+
+ if (!ugi.hasKerberosCredentials()) {
+ LOG.warn("User {} does not have Kerberos credentials for pre-authentication", ugi.getUserName());
+ return;
+ }
+
+ LOG.info("Pre-authenticating Kerberos for user: {}, authMethod: {}", ugi.getUserName(), ugi.getAuthenticationMethod());
+
+ ugi.checkTGTAndReloginFromKeytab();
+ LOG.debug("TGT verified and refreshed if needed for user: {}", ugi.getUserName());
+
+ // Get the audit server URL to determine the target hostname for service ticket
+ String auditServerUrl = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL);
+
+ if (StringUtils.isNotEmpty(auditServerUrl)) {
+ try {
+ URI uri = new URI(auditServerUrl);
+ String hostname = uri.getHost();
+
+ LOG.info("Pre-fetching Kerberos service ticket for HTTP/{}", hostname);
+
+ ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
+ LOG.debug("Kerberos security context initialized for audit server: {}", hostname);
+ return null;
+ });
+
+ LOG.info("Kerberos pre-authentication completed successfully. Service ticket cached for HTTP/{}", hostname);
+ } catch (URISyntaxException e) {
+ LOG.warn("Invalid audit server URL format: {}. Skipping service ticket pre-fetch", auditServerUrl, e);
+ } catch (Exception e) {
+ LOG.warn("Failed to pre-fetch service ticket for audit server: {}. First request may need to obtain ticket", auditServerUrl, e);
+ }
+ } else {
+ LOG.warn("Audit server URL not configured. Cannot pre-fetch service ticket");
+ }
+ } catch (Exception e) {
+ LOG.warn("Kerberos pre-authentication failed. First request will retry authentication", e);
+ }
+
+ LOG.info("<== RangerAuditServerDestination:preAuthenticateKerberos()");
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("==> RangerAuditServerDestination.stop() called..");
+ logStatus();
+ if (httpClient != null) {
+ try {
+ httpClient.close();
+ } catch (IOException ioe) {
+ LOG.error("Error while closing httpclient in RangerAuditServerDestination!", ioe);
+ } finally {
+ httpClient = null;
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ boolean ret = false;
+ try {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ if (httpClient == null) {
+ httpClient = buildHTTPClient();
+ if (httpClient == null) {
+ // HTTP Server is still not initialized. So need return error
+ addDeferredCount(events.size());
+ return ret;
+ }
+ }
+ ret = logAsBatch(events);
+ } catch (Throwable t) {
+ addDeferredCount(events.size());
+ logError("Error sending audit to HTTP Server", t);
+ }
+ return ret;
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+ private boolean logAsBatch(Collection<AuditEventBase> events) {
+ boolean batchSuccess = false;
+ int totalEvents = events.size();
+
+ LOG.debug("==> logAsBatch() Sending batch of {} events to Audit Server....", totalEvents);
+
+ batchSuccess = sendBatch(events);
+ if (batchSuccess) {
+ addSuccessCount(totalEvents);
+ } else {
+ LOG.error("Failed to send batch of {} events", totalEvents);
+ addFailedCount(totalEvents);
+ }
+
+ LOG.debug("<== logAsBatch() Batch processing complete: {}/{} events sent successfully", batchSuccess ? totalEvents : 0, totalEvents);
+
+ return batchSuccess;
+ }
+
+ private boolean sendBatch(Collection<AuditEventBase> events) {
+ boolean ret = false;
+ Map<String, String> queryParams = new HashMap<>();
+ try {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ LOG.debug("Sending audit batch of {} events as user: {}", events.size(), ugi.getUserName());
+
+ PrivilegedExceptionAction<Map<?, ?>> action =
+ () -> executeHttpBatchRequest(REST_RELATIVE_PATH_POST, queryParams, events, Map.class);
+ Map<?, ?> response = executeAction(action, ugi);
+
+ if (response != null) {
+ LOG.info("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), response);
+ ret = true;
+ } else {
+ LOG.error("Received null response from audit server for batch of {} events", events.size());
+ ret = false;
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to send audit batch of {} events to {}. Error: {}", events.size(), httpURL, e.getMessage(), e);
+
+ // Log additional context for authentication errors
+ if (e.getMessage() != null && e.getMessage().contains("401")) {
+ LOG.error("Authentication failure detected. Verify Kerberos credentials are valid and audit server is reachable.");
+ }
+ ret = false;
+ }
+
+ return ret;
+ }
+
+ public <T> T executeHttpBatchRequest(String relativeUrl, Map<String, String> params,
+ Collection<AuditEventBase> events, Class<T> clazz) throws Exception {
+ T finalResponse = postAndParse(httpURL + relativeUrl, params, events, clazz);
+ return finalResponse;
+ }
+
+ public <T> T executeHttpRequestPOST(String relativeUrl, Map<String, String> params, Object obj, Class<T> clazz) throws Exception {
+ LOG.debug("==> RangerAuditServerDestination().executeHttpRequestPOST()");
+
+ T finalResponse = postAndParse(httpURL + relativeUrl, params, obj, clazz);
+
+ LOG.debug("<== RangerAuditServerDestination().executeHttpRequestPOST()");
+ return finalResponse;
+ }
+
+ public <T> T postAndParse(String url, Map<String, String> queryParams, Object obj, Class<T> clazz) throws Exception {
+ HttpPost httpPost = new HttpPost(buildURI(url, queryParams));
+ StringEntity entity = new StringEntity(gsonBuilder.toJson(obj));
+ httpPost.setEntity(entity);
+ return executeAndParseResponse(httpPost, clazz, queryParams);
+ }
+
+ synchronized CloseableHttpClient buildHTTPClient() {
+ Lookup<AuthSchemeProvider> authRegistry = RegistryBuilder.<AuthSchemeProvider>create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, true)).build();
+ HttpClientBuilder clientBuilder = HttpClients.custom().setDefaultAuthSchemeRegistry(authRegistry);
+ String userName = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_NAME);
+ String passWord = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_USER_PASSWORD);
+ int clientConnTimeOutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_CONN_TIMEOUT_MS, 1000);
+ int clientReadTimeOutMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_CLIENT_READ_TIMEOUT_MS, 1000);
+ int maxConnections = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_CONNECTION, 10);
+ int maxConnectionsPerHost = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_MAX_CONNECTION_PER_HOST, 10);
+ int validateAfterInactivityMs = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_VALIDATE_INACTIVE_MS, 1000);
+ int poolRetryCount = MiscUtil.getIntProperty(props, PROP_AUDITSERVER_POOL_RETRY_COUNT, 5);
+ httpURL = MiscUtil.getStringProperty(props, PROP_AUDITSERVER_URL);
+
+ LOG.info("Building HTTP client for audit destination: url={}, connTimeout={}ms, readTimeout={}ms, maxConn={}", httpURL, clientConnTimeOutMs, clientReadTimeOutMs, maxConnections);
+
+ try {
+ if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) {
+ // JWT authentication - token will be added to request headers directly
+ LOG.info("HTTP client configured for JWT Bearer token authentication");
+ } else {
+ // Kerberos or Basic authentication - use credentials provider
+ BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+
+ if (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType) || isKerberosAuthenticated()) {
+ credentialsProvider.setCredentials(AuthScope.ANY, new KerberosCredentials(null));
+ LOG.info("HTTP client configured for Kerberos authentication (SPNEGO)....");
+ try {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ LOG.info("Current Kerberos principal: {}, authMethod: {}, hasKerberosCredentials: {}", ugi.getUserName(), ugi.getAuthenticationMethod(), ugi.hasKerberosCredentials());
+ } catch (Exception e) {
+ LOG.warn("Failed to get current UGI details", e);
+ }
+ } else if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType) || (StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(passWord))) {
+ credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, passWord));
+ LOG.info("HTTP client configured for basic authentication with username: {}", userName);
+ } else {
+ LOG.warn("No authentication credentials configured for HTTP audit destination");
+ }
+
+ clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+ }
+ } catch (Exception excp) {
+ LOG.error("Exception while configuring authentication credentials. Audits may fail to send!", excp);
+ }
+
+ PoolingHttpClientConnectionManager connectionManager;
+
+ KeyManager[] kmList = getKeyManagers();
+ TrustManager[] tmList = getTrustManagers();
+ SSLContext sslContext = getSSLContext(kmList, tmList);
+
+ if (sslContext != null) {
+ SSLContext.setDefault(sslContext);
+ }
+
+ boolean isSSL = (httpURL != null && httpURL.contains("https://"));
+ if (isSSL) {
+ SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
+ Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create().register(PROTOCOL_HTTPS, sslsf).build();
+
+ connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+
+ clientBuilder.setSSLSocketFactory(sslsf);
+ } else {
+ connectionManager = new PoolingHttpClientConnectionManager();
+ }
+
+ connectionManager.setMaxTotal(maxConnections);
+ connectionManager.setDefaultMaxPerRoute(maxConnectionsPerHost);
+ connectionManager.setValidateAfterInactivity(validateAfterInactivityMs);
+
+ RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
+ .setCookieSpec(CookieSpecs.DEFAULT)
+ .setConnectTimeout(clientConnTimeOutMs)
+ .setSocketTimeout(clientReadTimeOutMs);
+
+ // Configure authentication based on auth type
+ if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) {
+ // JWT doesn't use HttpClient's authentication mechanism - token is added to headers
+ LOG.info("RequestConfig configured for JWT authentication....");
+ } else {
+ // For Kerberos and Basic auth, enable HttpClient's authentication mechanism
+ requestConfigBuilder.setAuthenticationEnabled(true);
+
+ try {
+ // For Kerberos authentication, specify SPNEGO as target auth scheme
+ if (isKerberosAuthenticated() || (AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType))) {
+ requestConfigBuilder.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.SPNEGO));
+ LOG.info("Configured SPNEGO as target authentication scheme for audit HTTP client");
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to check Kerberos authentication status for request config", e);
+ }
+ }
+
+ RequestConfig customizedRequestConfig = requestConfigBuilder.build();
+
+ CloseableHttpClient httpClient = clientBuilder.setConnectionManager(connectionManager).setRetryHandler(new AuditHTTPRetryHandler(poolRetryCount, true)).setDefaultRequestConfig(customizedRequestConfig).build();
+
+ return httpClient;
+ }
+
+ boolean isKerberosAuthenticated() throws Exception {
+ boolean status = false;
+ try {
+ UserGroupInformation loggedInUser = UserGroupInformation.getLoginUser();
+ boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+ boolean hasKerberosCredentials = loggedInUser.hasKerberosCredentials();
+ UserGroupInformation.AuthenticationMethod loggedInUserAuthMethod = loggedInUser.getAuthenticationMethod();
+
+ status = isSecurityEnabled && hasKerberosCredentials && loggedInUserAuthMethod.equals(UserGroupInformation.AuthenticationMethod.KERBEROS);
+ } catch (IOException e) {
+ throw new Exception("Failed to get authentication details.", e);
+ }
+ return status;
+ }
+
+ private void close(InputStream str, String filename) {
+ if (str != null) {
+ try {
+ str.close();
+ } catch (IOException excp) {
+ LOG.error("Error while closing file: [{}]", filename, excp);
+ }
+ }
+ }
+
+ private <T> PrivilegedExceptionAction<T> getPrivilegedAction(String relativeUrl, Map<String, String> queryParams, Object postParam, Class<T> clazz) {
+ return () -> executeHttpRequestPOST(relativeUrl, queryParams, postParam, clazz);
+ }
+
+ private <T extends HttpRequestBase, R> R executeAndParseResponse(T request, Class<R> responseClazz, Map<String, String> queryParams) throws Exception {
+ R ret = null;
+ CloseableHttpClient client = getCloseableHttpClient();
+
+ if (client != null) {
+ addCommonHeaders(request, queryParams);
+ // Create an HttpClientContext to maintain authentication state across potential retries
+ HttpClientContext context = HttpClientContext.create();
+ try (CloseableHttpResponse response = client.execute(request, context)) {
+ ret = parseResponse(response, responseClazz);
+ }
+ } else {
+ LOG.error("Cannot process request as Audit HTTPClient is null...");
+ }
+ return ret;
+ }
+
+ private <T extends HttpRequestBase> T addCommonHeaders(T t, Map<String, String> queryParams) {
+ t.addHeader(REST_HEADER_ACCEPT, REST_ACCEPTED_MIME_TYPE_JSON);
+ t.setHeader(REST_HEADER_CONTENT_TYPE, REST_CONTENT_TYPE_MIME_TYPE_JSON);
+
+ // Add JWT Bearer token if JWT authentication is configured
+ if (AUTH_TYPE_JWT.equalsIgnoreCase(authType) && StringUtils.isNotEmpty(jwtToken)) {
+ t.setHeader(REST_HEADER_AUTHORIZATION, "Bearer " + jwtToken);
+ LOG.debug("Added JWT Bearer token to request Authorization header");
+ }
+
+ return t;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <R> R parseResponse(HttpResponse response, Class<R> clazz) throws Exception {
+ R type = null;
+
+ if (response == null) {
+ String responseError = "Received NULL response from server";
+ LOG.error(responseError);
+ throw new Exception(responseError);
+ }
+
+ int httpStatus = response.getStatusLine().getStatusCode();
+
+ if (httpStatus == HttpStatus.SC_OK) {
+ InputStream responseInputStream = response.getEntity().getContent();
+ if (clazz.equals(String.class)) {
+ type = (R) IOUtils.toString(responseInputStream, Charset.defaultCharset());
+ } else {
+ type = gsonBuilder.fromJson(new InputStreamReader(responseInputStream), clazz);
+ }
+ responseInputStream.close();
+ } else {
+ String responseBody = "";
+ try {
+ if (response.getEntity() != null && response.getEntity().getContent() != null) {
+ responseBody = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset());
+ }
+ } catch (Exception e) {
+ LOG.debug("Failed to read error response body", e);
+ }
+
+ String error = String.format("Request failed with HTTP status %d. Response body: %s", httpStatus, responseBody);
+ if (httpStatus == HttpStatus.SC_UNAUTHORIZED) {
+ LOG.error("{} - Authentication failed. Verify Kerberos credentials are valid and properly configured.", error);
+ } else {
+ LOG.error(error);
+ }
+ throw new Exception(error);
+ }
+ return type;
+ }
+
+ private <T> T executeAction(PrivilegedExceptionAction<T> action, UserGroupInformation owner) throws Exception {
+ T ret = null;
+ if (owner != null) {
+ ret = owner.doAs(action);
+ } else {
+ ret = action.run();
+ }
+ return ret;
+ }
+
+ private URI buildURI(String url, Map<String, String> queryParams) throws URISyntaxException {
+ URIBuilder builder = new URIBuilder(url);
+ if (queryParams != null) {
+ for (Map.Entry<String, String> param : queryParams.entrySet()) {
+ builder.addParameter(param.getKey(), param.getValue());
+ }
+ }
+ return builder.build();
+ }
+
+ private CloseableHttpClient getCloseableHttpClient() {
+ CloseableHttpClient client = httpClient;
+ if (client == null) {
+ synchronized (this) {
+ client = httpClient;
+
+ if (client == null) {
+ client = buildHTTPClient();
+ httpClient = client;
+ }
+ }
+ }
+ return client;
+ }
+
+ private KeyManager[] getKeyManagers() {
+ KeyManager[] kmList = null;
+ String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL);
+ String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS;
+ String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE);
+ String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias);
+ if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) {
+ InputStream in = null;
+ try {
+ in = getFileInputStream(keyStoreFile);
+
+ if (in != null) {
+ String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE);
+ keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT;
+ KeyStore keyStore = KeyStore.getInstance(keyStoreType);
+
+ keyStore.load(in, keyStoreFilepwd.toCharArray());
+
+ KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
+
+ keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
+
+ kmList = keyManagerFactory.getKeyManagers();
+ } else {
+ LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore :{}", e.getMessage(), e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification ", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL Keystore Files", e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL Keystore Files", e);
+ } catch (UnrecoverableKeyException e) {
+ LOG.error("Unable to recover the key from keystore", e);
+ } finally {
+ close(in, keyStoreFile);
+ }
+ }
+ return kmList;
+ }
+
+ private TrustManager[] getTrustManagers() {
+ TrustManager[] tmList = null;
+ String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL);
+ String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS;
+ String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE);
+ String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias);
+ if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) {
+ InputStream in = null;
+ try {
+ in = getFileInputStream(trustStoreFile);
+
+ if (in != null) {
+ String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE);
+ trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT;
+ KeyStore trustStore = KeyStore.getInstance(trustStoreType);
+
+ trustStore.load(in, trustStoreFilepwd.toCharArray());
+
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
+
+ trustManagerFactory.init(trustStore);
+
+ tmList = trustManagerFactory.getTrustManagers();
+ } else {
+ LOG.error("Unable to obtain truststore from file [{}]", trustStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment :{}", e.getMessage(), e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification :{}", e.getMessage(), e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL TrustStore File:{}", trustStoreFile, e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL TrustStore Files :{}", trustStoreFile, e);
+ } finally {
+ close(in, trustStoreFile);
+ }
+ }
+ return tmList;
+ }
+
+ private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) {
+ SSLContext sslContext = null;
+ try {
+ sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
+ if (sslContext != null) {
+ sslContext.init(kmList, tmList, new SecureRandom());
+ }
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is not available in the environment", e);
+ } catch (KeyManagementException e) {
+ LOG.error("Unable to initialise the SSLContext", e);
+ }
+ return sslContext;
+ }
+
+ private InputStream getFileInputStream(String fileName) throws IOException {
+ InputStream in = null;
+ if (StringUtils.isNotEmpty(fileName)) {
+ File file = new File(fileName);
+ if (file != null && file.exists()) {
+ in = new FileInputStream(file);
+ } else {
+ in = ClassLoader.getSystemResourceAsStream(fileName);
+ }
+ }
+ return in;
+ }
+
+ private class AuditHTTPRetryHandler extends StandardHttpRequestRetryHandler {
+ public AuditHTTPRetryHandler(Integer poolRetryCount, boolean isIdempotentRequest) {
+ super(poolRetryCount, isIdempotentRequest);
+ }
+
+ @Override
+ public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
+ LOG.debug("==> AuditHTTPRetryHandler.retryRequest {} Execution Count = {}", exception.getMessage(), executionCount);
+
+ boolean ret = super.retryRequest(exception, executionCount, context);
+
+ LOG.debug("<== AuditHTTPRetryHandler.retryRequest(): ret= {}", ret);
+
+ return ret;
+ }
+ }
+}
diff --git a/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index 81a576a22..61ec23692 100644
--- a/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -487,9 +487,9 @@ private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collecti
if (kerberosUser != null) {
// execute the privileged action as the given keytab user
- final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG);
+ final KerberosAction<UpdateResponse> kerberosAction = new KerberosAction<>(kerberosUser, action, LOG);
- ret = (UpdateResponse) kerberosAction.execute();
+ ret = kerberosAction.execute();
} else {
ret = action.run();
}
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index 250bb582e..a25edde92 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -31,6 +31,7 @@
<!-- Sub modules -->
<modules>
<module>core</module>
+ <module>dest-auditserver</module>
<module>dest-cloudwatch</module>
<module>dest-es</module>
<module>dest-hdfs</module>
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
index a9fb1863a..6e20ebe7a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
@@ -20,6 +20,7 @@
package org.apache.ranger.plugin.audit;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.ranger.audit.model.AuthzAuditEvent;
@@ -138,7 +139,7 @@ public AuthzAuditEvent getAuthzEvents(RangerAccessResult result) {
ret.setDatasets(getDatasets(request));
ret.setProjects(getProjects(request));
ret.setDatasetIds(getDatasetIds(request));
- ret.setAdditionalInfo(getAdditionalInfo(request));
+ ret.setAdditionalInfo(getAdditionalInfo(request, result));
ret.setClusterName(request.getClusterName());
ret.setZoneName(result.getZoneName());
ret.setAgentHostname(restUtils.getAgentHostname());
@@ -230,16 +231,28 @@ public final Set<Long> getDatasetIds(RangerAccessRequest request) {
return gdsResult != null ? gdsResult.getDatasetIds() : null;
}
- public String getAdditionalInfo(RangerAccessRequest request) {
- if (StringUtils.isBlank(request.getRemoteIPAddress()) && CollectionUtils.isEmpty(request.getForwardedAddresses())) {
- return null;
+ public String getAdditionalInfo(RangerAccessRequest request, RangerAccessResult result) {
+ String ret = org.apache.commons.lang3.StringUtils.EMPTY;
+ Map<String, String> addInfomap = new HashMap<>();
+
+ if (!CollectionUtils.isEmpty(request.getForwardedAddresses())) {
+ addInfomap.put("forwarded-ip-addresses", "[" + org.apache.commons.lang3.StringUtils.join(request.getForwardedAddresses(), ", ") + "]");
}
- Map<String, String> addInfomap = new HashMap<>();
- addInfomap.put("forwarded-ip-addresses", "[" + StringUtils.join(request.getForwardedAddresses(), ", ") + "]");
- addInfomap.put("remote-ip-address", request.getRemoteIPAddress());
+ if (org.apache.commons.lang3.StringUtils.isNotEmpty(request.getRemoteIPAddress())) {
+ addInfomap.put("remote-ip-address", request.getRemoteIPAddress());
+ }
- return JsonUtils.mapToJson(addInfomap);
+ String serviceType = result.getServiceTypeName();
+ if (org.apache.commons.lang3.StringUtils.isNotEmpty(serviceType)) {
+ addInfomap.put("serviceType", serviceType);
+ }
+
+ if (MapUtils.isNotEmpty(addInfomap)) {
+ ret = JsonUtils.mapToJson(addInfomap);
+ }
+
+ return ret;
}
protected final Set<String> getTags(RangerAccessRequest request) {
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
index 45441dac0..32a0aaab7 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
@@ -259,6 +259,16 @@ public int getServiceType() {
return ret;
}
+ public String getServiceTypeName() {
+ String ret = null;
+
+ if (serviceDef != null && serviceDef.getName() != null) {
+ ret = serviceDef.getName();
+ }
+
+ return ret;
+ }
+
public Map<String, Object> getAdditionalInfo() {
return additionalInfo;
}
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java
index acdad81e0..54a63c42d 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/audit/TestRangerDefaultAuditHandler.java
@@ -286,15 +286,17 @@ public void test07_getDatasetsAndProjects_fromContext() {
public void test08_getAdditionalInfo_nullAndNonNull() {
RangerDefaultAuditHandler handler = new RangerDefaultAuditHandler();
RangerAccessRequestImpl req = new RangerAccessRequestImpl();
+ RangerAccessResult res = new RangerAccessResult(0, "svc", null, req);
req.setRemoteIPAddress(null);
req.setForwardedAddresses(new ArrayList<>());
- Assertions.assertNull(handler.getAdditionalInfo(req));
+ Assertions.assertNull(handler.getAdditionalInfo(req, res));
RangerAccessRequestImpl req2 = new RangerAccessRequestImpl();
+ RangerAccessResult res2 = new RangerAccessResult(0, "svc", null, req);
req2.setRemoteIPAddress("10.1.1.1");
List<String> fwd = new ArrayList<>(Arrays.asList("1.1.1.1"));
req2.setForwardedAddresses(fwd);
- String info = handler.getAdditionalInfo(req2);
+ String info = handler.getAdditionalInfo(req2, res2);
Assertions.assertNotNull(info);
Assertions.assertTrue(info.contains("10.1.1.1"));
}
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index 8453ea111..b27010823 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -2,6 +2,9 @@
!config
!dist/version
!dist/ranger-*-admin.tar.gz
+!dist/ranger-*-ranger-audit-server-service.tar.gz
+!dist/ranger-*-ranger-audit-consumer-solr.tar.gz
+!dist/ranger-*-ranger-audit-consumer-hdfs.tar.gz
!dist/ranger-*-kms.tar.gz
!dist/ranger-*-usersync.tar.gz
!dist/ranger-*-tagsync.tar.gz
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs
new file mode 100644
index 000000000..e143c3247
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-hdfs
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_IMAGE
+ARG RANGER_BASE_VERSION
+FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+
+# Declare ARG for use in this build stage
+ARG RANGER_VERSION
+ARG KERBEROS_ENABLED
+
+# Install required packages including Kerberos client
+RUN apt-get update && apt-get install -y \
+ python3 \
+ python3-pip \
+ curl \
+ vim \
+ net-tools \
+ krb5-user \
+ && rm -rf /var/lib/apt/lists/*
+
+# Volume for keytabs
+VOLUME /etc/keytabs
+
+# Set up working directory
+WORKDIR /opt/ranger-audit-consumer-hdfs
+
+# Copy ranger-audit-consumer-hdfs distribution
+COPY ./dist/ranger-${RANGER_VERSION}-ranger-audit-consumer-hdfs.tar.gz /tmp/
+
+# Copy scripts, Hadoop configs, and Kerberos configuration
+COPY ./scripts/audit-server/service-check-functions.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/audit-server/ranger-audit-consumer-hdfs.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf /etc/krb5.conf
+
+# Extract and setup
+RUN tar xzf /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-hdfs.tar.gz -C /opt/ranger-audit-consumer-hdfs --strip-components=1 && \
+ rm -f /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-hdfs.tar.gz && \
+ mkdir -p /var/log/ranger/ranger-audit-consumer-hdfs && \
+ mkdir -p /home/ranger/scripts && \
+ rm -rf /opt/ranger-audit-consumer-hdfs/logs && \
+ ln -sf /var/log/ranger/ranger-audit-consumer-hdfs /opt/ranger-audit-consumer-hdfs/logs && \
+ chown -R ranger:ranger /opt/ranger-audit-consumer-hdfs/ /var/log/ranger/ ${RANGER_SCRIPTS} && \
+ chmod -R 755 /opt/ranger-audit-consumer-hdfs && \
+ chmod 755 ${RANGER_SCRIPTS}/ranger-audit-consumer-hdfs.sh
+
+# Set Java home
+ENV JAVA_HOME=/opt/java/openjdk
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+# Environment variables for HDFS consumer
+ENV AUDIT_CONSUMER_HOME_DIR=/opt/ranger-audit-consumer-hdfs
+ENV AUDIT_CONSUMER_CONF_DIR=/opt/ranger-audit-consumer-hdfs/conf
+ENV AUDIT_CONSUMER_LOG_DIR=/var/log/ranger/ranger-audit-consumer-hdfs
+ENV RANGER_USER=ranger
+
+# Expose health check port
+EXPOSE 7092
+
+# Health check - test the health endpoint
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+ CMD curl -f -o /dev/null -s -w "%{http_code}" http://localhost:7092/api/health | grep -q "200" || exit 1
+
+# Environment variable for Kerberos support
+ENV KERBEROS_ENABLED=${KERBEROS_ENABLED:-false}
+
+# Switch to ranger user
+USER ranger
+
+# Start the HDFS consumer using the custom startup script
+WORKDIR /opt/ranger-audit-consumer-hdfs
+ENTRYPOINT ["/home/ranger/scripts/ranger-audit-consumer-hdfs.sh"]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr
new file mode 100644
index 000000000..abcb6240e
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-consumer-solr
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_IMAGE
+ARG RANGER_BASE_VERSION
+FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+
+# Declare ARG for use in this build stage
+ARG RANGER_VERSION
+ARG KERBEROS_ENABLED
+
+# Install required packages including Kerberos client
+RUN apt-get update && apt-get install -y \
+ python3 \
+ python3-pip \
+ curl \
+ vim \
+ net-tools \
+ krb5-user \
+ && rm -rf /var/lib/apt/lists/*
+
+# Volume for keytabs
+VOLUME /etc/keytabs
+
+# Set up working directory
+WORKDIR /opt/ranger-audit-consumer-solr
+
+# Copy ranger-audit-consumer-solr distribution
+COPY ./dist/ranger-${RANGER_VERSION}-ranger-audit-consumer-solr.tar.gz /tmp/
+
+# Copy scripts and Kerberos configuration
+COPY ./scripts/audit-server/service-check-functions.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/audit-server/ranger-audit-consumer-solr.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf /etc/krb5.conf
+
+# Extract and setup
+RUN tar xzf /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-solr.tar.gz -C /opt/ranger-audit-consumer-solr --strip-components=1 && \
+ rm -f /tmp/ranger-${RANGER_VERSION}-ranger-audit-consumer-solr.tar.gz && \
+ mkdir -p /var/log/ranger/ranger-audit-consumer-solr && \
+ mkdir -p /home/ranger/scripts && \
+ rm -rf /opt/ranger-audit-consumer-solr/logs && \
+ ln -sf /var/log/ranger/ranger-audit-consumer-solr /opt/ranger-audit-consumer-solr/logs && \
+ chown -R ranger:ranger /opt/ranger-audit-consumer-solr/ /var/log/ranger/ ${RANGER_SCRIPTS} && \
+ chmod -R 755 /opt/ranger-audit-consumer-solr && \
+ chmod 755 ${RANGER_SCRIPTS}/ranger-audit-consumer-solr.sh && \
+ chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh
+
+# Set Java home
+ENV JAVA_HOME=/opt/java/openjdk
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+# Environment variables for Solr consumer
+ENV AUDIT_CONSUMER_HOME_DIR=/opt/ranger-audit-consumer-solr
+ENV AUDIT_CONSUMER_CONF_DIR=/opt/ranger-audit-consumer-solr/conf
+ENV AUDIT_CONSUMER_LOG_DIR=/var/log/ranger/ranger-audit-consumer-solr
+ENV RANGER_USER=ranger
+
+# Expose health check port
+EXPOSE 7091
+
+# Health check - test the health endpoint
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+ CMD curl -f -o /dev/null -s -w "%{http_code}" http://localhost:7091/api/health | grep -q "200" || exit 1
+
+# Environment variable for Kerberos support
+ENV KERBEROS_ENABLED=${KERBEROS_ENABLED:-false}
+
+# Switch to ranger user
+USER ranger
+
+# Start the Solr consumer using the custom startup script
+WORKDIR /opt/ranger-audit-consumer-solr
+ENTRYPOINT ["/home/ranger/scripts/ranger-audit-consumer-solr.sh"]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-audit-server b/dev-support/ranger-docker/Dockerfile.ranger-audit-server
new file mode 100644
index 000000000..e95dc5c7c
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-audit-server
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_IMAGE
+ARG RANGER_BASE_VERSION
+FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+
+# Declare ARG for use in this build stage
+ARG RANGER_VERSION
+ARG KERBEROS_ENABLED
+
+# Install required packages including Kerberos client
+RUN apt-get update && apt-get install -y \
+ python3 \
+ python3-pip \
+ curl \
+ vim \
+ net-tools \
+ krb5-user \
+ && rm -rf /var/lib/apt/lists/*
+
+# Volume for keytabs
+VOLUME /etc/keytabs
+
+# Set up working directory
+WORKDIR /opt/ranger-audit-server
+
+# Copy ranger-audit-server-service distribution
+COPY ./dist/ranger-${RANGER_VERSION}-ranger-audit-server-service.tar.gz /tmp/
+
+# Copy scripts and Kerberos configuration
+COPY ./scripts/audit-server/service-check-functions.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/audit-server/ranger-audit-server.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf /etc/krb5.conf
+
+# Extract and setup
+RUN tar xzf /tmp/ranger-${RANGER_VERSION}-ranger-audit-server-service.tar.gz -C /opt/ranger-audit-server --strip-components=1 && \
+ rm -f /tmp/ranger-${RANGER_VERSION}-ranger-audit-server-service.tar.gz && \
+ mkdir -p /var/log/ranger/ranger-audit-server/audit/spool && \
+ mkdir -p /var/log/ranger/ranger-audit-server/audit/archive && \
+ mkdir -p /home/ranger/scripts && \
+ rm -rf /opt/ranger-audit-server/logs && \
+ ln -sf /var/log/ranger/ranger-audit-server /opt/ranger-audit-server/logs && \
+ chown -R ranger:ranger /opt/ranger-audit-server/ /var/log/ranger/ ${RANGER_SCRIPTS} && \
+ chmod -R 755 /opt/ranger-audit-server && \
+ chmod 755 ${RANGER_SCRIPTS}/ranger-audit-server.sh
+
+# Set Java home
+ENV JAVA_HOME=/opt/java/openjdk
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+# Environment variables for audit server
+ENV AUDIT_SERVER_HOME_DIR=/opt/ranger-audit-server
+ENV AUDIT_SERVER_CONF_DIR=/opt/ranger-audit-server/conf
+ENV AUDIT_SERVER_LOG_DIR=/var/log/ranger/ranger-audit-server
+ENV RANGER_USER=ranger
+
+# Expose ports (HTTP and HTTPS)
+EXPOSE 7081 7182
+
+# Health check - test the REST API endpoint
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+ CMD curl -f -o /dev/null -s -w "%{http_code}" http://localhost:7081/api/audit/health | grep -q -E "200|401" || exit 1
+
+# Environment variable for Kerberos support
+ENV KERBEROS_ENABLED=${KERBEROS_ENABLED:-false}
+
+# Switch to ranger user
+USER ranger
+
+# Start the audit server using the custom startup script
+WORKDIR /opt/ranger-audit-server
+ENTRYPOINT ["/home/ranger/scripts/ranger-audit-server.sh"]
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 39a89440f..bd0528909 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -118,3 +118,8 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.ym
~~~
docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-usersync.yml -f docker-compose.ranger-tagsync.yml -f docker-compose.ranger-kms.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-hive.yml -f docker-compose.ranger-trino.yml -f docker-compose.ranger-knox.yml up -d --no-deps --force-recreate --build <service-1> <service-2>
~~~
+
+#### To bring up audit server, solr and hdfs consumer. Make sure kafka,solr and hdfs containers are running before bring up audit server.
+~~~
+docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-kafka.yml -f docker-compose.ranger-audit-server.yml up -d
+~~~
\ No newline at end of file
diff --git a/dev-support/ranger-docker/docker-compose.ranger-audit-server.yml b/dev-support/ranger-docker/docker-compose.ranger-audit-server.yml
new file mode 100644
index 000000000..1a18d3a21
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-audit-server.yml
@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+ # Core Audit Server Service - Receives audits from plugins and produces to Kafka
+ ranger-audit-server:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-audit-server
+ args:
+ - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE}
+ - RANGER_BASE_VERSION=${RANGER_BASE_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
+ - KERBEROS_ENABLED=${KERBEROS_ENABLED}
+ image: ranger-audit-server:latest
+ container_name: ranger-audit-server
+ hostname: ranger-audit-server.rangernw
+ stdin_open: true
+ tty: true
+ depends_on:
+ ranger-kdc:
+ condition: service_started
+ ranger-kafka:
+ condition: service_started
+ ports:
+ - "7081:7081"
+ - "7182:7182"
+ environment:
+ JAVA_HOME: /opt/java/openjdk
+ AUDIT_SERVER_HOME_DIR: /opt/ranger-audit-server
+ AUDIT_SERVER_CONF_DIR: /opt/ranger-audit-server/conf
+ AUDIT_SERVER_LOG_DIR: /var/log/ranger/ranger-audit-server
+ AUDIT_SERVER_HEAP: "-Xms512m -Xmx2g"
+ RANGER_VERSION: ${RANGER_VERSION}
+ KERBEROS_ENABLED: ${KERBEROS_ENABLED}
+ KAFKA_BOOTSTRAP_SERVERS: ranger-kafka.rangernw:9092
+ networks:
+ - ranger
+ volumes:
+ - ./dist/keytabs/ranger-audit-server:/etc/keytabs
+ - ranger-audit-server-logs:/opt/ranger-audit-server/logs
+ - ranger-audit-server-spool:/var/log/ranger/ranger-audit-server/audit
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:7081/api/audit/health"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 60s
+
+ # Solr Consumer Service - Consumes from Kafka and indexes to Solr
+ ranger-audit-consumer-solr:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-audit-consumer-solr
+ args:
+ - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE}
+ - RANGER_BASE_VERSION=${RANGER_BASE_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
+ - KERBEROS_ENABLED=${KERBEROS_ENABLED}
+ image: ranger-audit-consumer-solr:latest
+ container_name: ranger-audit-consumer-solr
+ hostname: ranger-audit-consumer-solr.rangernw
+ stdin_open: true
+ tty: true
+ depends_on:
+ ranger-kdc:
+ condition: service_started
+ ranger-kafka:
+ condition: service_started
+ ranger-solr:
+ condition: service_started
+ ranger-audit-server:
+ condition: service_healthy
+ ports:
+ - "7091:7091"
+ environment:
+ JAVA_HOME: /opt/java/openjdk
+ AUDIT_CONSUMER_HOME_DIR: /opt/ranger-audit-consumer-solr
+ AUDIT_CONSUMER_CONF_DIR: /opt/ranger-audit-consumer-solr/conf
+ AUDIT_CONSUMER_LOG_DIR: /var/log/ranger/ranger-audit-consumer-solr
+ AUDIT_CONSUMER_HEAP: "-Xms512m -Xmx2g"
+ RANGER_VERSION: ${RANGER_VERSION}
+ KERBEROS_ENABLED: ${KERBEROS_ENABLED}
+ KAFKA_BOOTSTRAP_SERVERS: ranger-kafka.rangernw:9092
+ SOLR_URL: http://ranger-solr.rangernw:8983/solr
+ networks:
+ - ranger
+ volumes:
+ - ./dist/keytabs/ranger-audit-consumer-solr:/etc/keytabs
+ - ranger-audit-consumer-solr-logs:/opt/ranger-audit-consumer-solr/logs
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:7091/api/health"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 60s
+
+ # HDFS Consumer Service - Consumes from Kafka and writes to HDFS
+ ranger-audit-consumer-hdfs:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-audit-consumer-hdfs
+ args:
+ - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE}
+ - RANGER_BASE_VERSION=${RANGER_BASE_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
+ - KERBEROS_ENABLED=${KERBEROS_ENABLED}
+ image: ranger-audit-consumer-hdfs:latest
+ container_name: ranger-audit-consumer-hdfs
+ hostname: ranger-audit-consumer-hdfs.rangernw
+ stdin_open: true
+ tty: true
+ depends_on:
+ ranger-kdc:
+ condition: service_started
+ ranger-kafka:
+ condition: service_started
+ ranger-hadoop:
+ condition: service_healthy
+ ranger-audit-server:
+ condition: service_healthy
+ ports:
+ - "7092:7092"
+ environment:
+ JAVA_HOME: /opt/java/openjdk
+ AUDIT_CONSUMER_HOME_DIR: /opt/ranger-audit-consumer-hdfs
+ AUDIT_CONSUMER_CONF_DIR: /opt/ranger-audit-consumer-hdfs/conf
+ AUDIT_CONSUMER_LOG_DIR: /var/log/ranger/ranger-audit-consumer-hdfs
+ AUDIT_CONSUMER_HEAP: "-Xms512m -Xmx2g"
+ RANGER_VERSION: ${RANGER_VERSION}
+ KERBEROS_ENABLED: ${KERBEROS_ENABLED}
+ KAFKA_BOOTSTRAP_SERVERS: ranger-kafka.rangernw:9092
+ HDFS_NAMENODE: ranger-hadoop.rangernw:9000
+ networks:
+ - ranger
+ volumes:
+ - ./dist/keytabs/ranger-audit-consumer-hdfs:/etc/keytabs
+ - ranger-audit-consumer-hdfs-logs:/opt/ranger-audit-consumer-hdfs/logs
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:7092/api/health"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 60s
+
+volumes:
+ ranger-audit-server-logs:
+ ranger-audit-server-spool:
+ ranger-audit-consumer-solr-logs:
+ ranger-audit-consumer-hdfs-logs:
+
+networks:
+ ranger:
+ name: rangernw
+ external: true
diff --git a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
index 80e386a50..b2713dead 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
@@ -25,7 +25,7 @@ services:
networks:
- ranger
ports:
- - "6667:6667"
+ - "9092:9092"
depends_on:
ranger:
condition: service_started
diff --git a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py
index 5153d5c47..e3dd0aa21 100644
--- a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py
+++ b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py
@@ -27,6 +27,10 @@ def service_not_exists(service):
'default-policy.1.resource.path.is-recursive': 'true',
'default-policy.1.policyItem.1.users': 'hive',
'default-policy.1.policyItem.1.accessTypes': 'read,write,execute',
+ 'default-policy.2.name': 'ranger-audit-path',
+ 'default-policy.2.resource.path': '/ranger/audit/*',
+ 'default-policy.2.policyItem.1.users': 'rangerauditserver',
+ 'default-policy.2.policyItem.1.accessTypes': 'read,write,execute',
'ranger.plugin.hdfs.policy.refresh.synchronous':'true'}})
hive = RangerService({'name': 'dev_hive', 'type': 'hive',
@@ -53,7 +57,19 @@ def service_not_exists(service):
'default-policy.2.resource.consumergroup': 'ranger_entities_consumer',
'default-policy.2.policyItem.1.users': 'rangertagsync',
'default-policy.2.policyItem.1.accessTypes': 'consume,describe',
- 'ranger.plugin.audit.filters': "[{'accessResult': 'DENIED', 'isAudited': true},{'resources':{'topic':{'values':['ATLAS_ENTITIES']}},'users':['rangertagsync'],'actions':['create','consume','describe'],'isAudited':false},{'resources':{'consumergroup':{'values':['ranger_entities_consumer']}},'users':['rangertagsync'],'actions':['consume'],'isAudited':false}]",
+ 'default-policy.3.name': 'topic: ranger_audit',
+ 'default-policy.3.resource.topic': 'ranger_audit*,__consumer_offsets',
+ 'default-policy.3.policyItem.1.users': 'rangerauditserver',
+ 'default-policy.3.policyItem.1.accessTypes': 'create,configure,consume,publish,describe,create,delete,alter,describe_configs,alter_configs',
+ 'default-policy.4.name': 'consumergroup: ranger_audit_consumer_groups',
+ 'default-policy.4.resource.consumergroup': 'ranger_audit*',
+ 'default-policy.4.policyItem.1.users': 'rangerauditserver',
+ 'default-policy.4.policyItem.1.accessTypes': 'consume,describe,delete',
+ 'default-policy.5.name': 'cluster: audit server cluster permissions',
+ 'default-policy.5.resource.cluster': '*,dummy',
+ 'default-policy.5.policyItem.1.users': 'rangerauditserver',
+ 'default-policy.5.policyItem.1.accessTypes': 'configure,describe,alter,create,idempotent_write,describe_configs,alter_configs',
+ 'ranger.plugin.audit.filters': "[{'accessResult': 'DENIED', 'isAudited': true},{'resources':{'topic':{'values':['ATLAS_ENTITIES']}},'users':['rangertagsync'],'actions':['create','consume','describe'],'isAudited':false},{'resources':{'consumergroup':{'values':['ranger_entities_consumer']}},'users':['rangertagsync'],'actions':['consume'],'isAudited':false},{'users':['rangerauditserver'],'isAudited':false}]",
'userstore.download.auth.users': 'kafka',
'ranger.plugin.kafka.policy.refresh.synchronous':'true'}})
@@ -120,6 +136,11 @@ def service_not_exists(service):
solr = RangerService({'name': 'dev_solr', 'type': 'solr',
'configs': {'username': 'solr', 'password': 'rangerR0cks!',
'solr.url': 'http://ranger-solr.rangernw:8983',
+ 'setup.additional.default.policies': 'true',
+ 'default-policy.1.name': 'collection: ranger_audits',
+ 'default-policy.1.resource.collection': 'ranger_audits',
+ 'default-policy.1.policyItem.1.users': 'rangerauditserver',
+ 'default-policy.1.policyItem.1.accessTypes': 'query,update',
'policy.download.auth.users': 'solr',
'tag.download.auth.users': 'solr',
'userstore.download.auth.users': 'solr',
diff --git a/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh
new file mode 100755
index 000000000..7ea7cfc15
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-hdfs.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-/opt/ranger-audit-consumer-hdfs}"
+AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-/opt/ranger-audit-consumer-hdfs/conf}"
+AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-/var/log/ranger/ranger-audit-consumer-hdfs}"
+
+# Create log directory if it doesn't exist
+mkdir -p ${AUDIT_CONSUMER_LOG_DIR}
+chown -R ranger:ranger /var/log/ranger 2>/dev/null || true
+
+echo "=========================================="
+echo "Starting Ranger Audit Consumer - HDFS"
+echo "=========================================="
+echo "AUDIT_CONSUMER_HOME_DIR: ${AUDIT_CONSUMER_HOME_DIR}"
+echo "AUDIT_CONSUMER_CONF_DIR: ${AUDIT_CONSUMER_CONF_DIR}"
+echo "AUDIT_CONSUMER_LOG_DIR: ${AUDIT_CONSUMER_LOG_DIR}"
+echo "=========================================="
+
+# Source service check functions
+source /home/ranger/scripts/service-check-functions.sh
+
+# Wait for Kafka to be available
+KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-ranger-kafka.rangernw:9092}"
+check_tcp_port "Kafka" "${KAFKA_BOOTSTRAP_SERVERS}" 60
+
+# Wait for HDFS to be available
+HDFS_NAMENODE="${HDFS_NAMENODE:-ranger-hadoop.rangernw:9000}"
+check_tcp_port "HDFS NameNode" "${HDFS_NAMENODE}" 60
+
+# Start the HDFS consumer
+echo "[INFO] Starting Ranger Audit Consumer - HDFS..."
+cd ${AUDIT_CONSUMER_HOME_DIR}
+
+# Export environment variables for Java
+export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
+export PATH=$JAVA_HOME/bin:$PATH
+export AUDIT_CONSUMER_LOG_DIR=${AUDIT_CONSUMER_LOG_DIR}
+
+# Set heap size
+AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}"
+export AUDIT_CONSUMER_HEAP
+
+# Set JVM options including logback configuration
+if [ -z "$AUDIT_CONSUMER_OPTS" ]; then
+ AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.dir=${AUDIT_CONSUMER_LOG_DIR}"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.file=ranger-audit-consumer-hdfs.log"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8"
+fi
+
+# Point to krb5.conf and Hadoop configs for Kerberos
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ export AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf"
+
+ # Wait for keytabs if Kerberos is enabled
+ if [ -f /home/ranger/scripts/wait_for_keytab.sh ]; then
+ echo "[INFO] Waiting for Kerberos keytabs..."
+ bash /home/ranger/scripts/wait_for_keytab.sh HTTP.keytab
+ bash /home/ranger/scripts/wait_for_keytab.sh rangerauditserver.keytab
+ echo "[INFO] ✓ Keytabs are available"
+ fi
+fi
+
+export AUDIT_CONSUMER_OPTS
+
+echo "[INFO] JAVA_HOME: ${JAVA_HOME}"
+echo "[INFO] AUDIT_CONSUMER_HEAP: ${AUDIT_CONSUMER_HEAP}"
+echo "[INFO] AUDIT_CONSUMER_OPTS: ${AUDIT_CONSUMER_OPTS}"
+
+# Build classpath from WAR file
+WEBAPP_ROOT="${AUDIT_CONSUMER_HOME_DIR}/webapp"
+WAR_FILE="${WEBAPP_ROOT}/ranger-audit-consumer-hdfs.war"
+WEBAPP_DIR="${WEBAPP_ROOT}/ranger-audit-consumer-hdfs"
+
+# Extract WAR if not already extracted
+if [ -f "${WAR_FILE}" ] && [ ! -d "${WEBAPP_DIR}" ]; then
+ echo "[INFO] Extracting WAR file..."
+ mkdir -p "${WEBAPP_DIR}"
+ cd "${WEBAPP_DIR}"
+ jar xf "${WAR_FILE}"
+ cd -
+fi
+
+# Build classpath
+RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes"
+for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+done
+
+# Add libext directory if it exists
+if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then
+ for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do
+ if [ -f "${jar}" ]; then
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+ fi
+ done
+fi
+
+export RANGER_CLASSPATH
+
+echo "[INFO] Starting Ranger Audit Consumer - HDFS..."
+echo "[INFO] Webapp dir: ${WEBAPP_DIR}"
+java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \
+ -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-hdfs-site.xml \
+ -Dhadoop.config.dir=${AUDIT_CONSUMER_CONF_DIR} \
+ -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \
+ -cp "${RANGER_CLASSPATH}" \
+ org.apache.ranger.audit.consumer.HdfsConsumerApplication \
+ >> ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>&1 &
+
+PID=$!
+echo $PID > ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.pid
+
+echo "[INFO] Ranger Audit Consumer - HDFS started with PID: $PID"
+
+# Keep the container running by tailing logs
+tail -f ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>/dev/null
diff --git a/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh
new file mode 100755
index 000000000..cb59efe09
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-consumer-solr.sh
@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-/opt/ranger-audit-consumer-solr}"
+AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-/opt/ranger-audit-consumer-solr/conf}"
+AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-/var/log/ranger/ranger-audit-consumer-solr}"
+
+# Create log directory if it doesn't exist
+mkdir -p ${AUDIT_CONSUMER_LOG_DIR}
+chown -R ranger:ranger /var/log/ranger 2>/dev/null || true
+
+echo "=========================================="
+echo "Starting Ranger Audit Consumer - Solr"
+echo "=========================================="
+echo "AUDIT_CONSUMER_HOME_DIR: ${AUDIT_CONSUMER_HOME_DIR}"
+echo "AUDIT_CONSUMER_CONF_DIR: ${AUDIT_CONSUMER_CONF_DIR}"
+echo "AUDIT_CONSUMER_LOG_DIR: ${AUDIT_CONSUMER_LOG_DIR}"
+echo "=========================================="
+
+# Source service check functions
+source /home/ranger/scripts/service-check-functions.sh
+
+# Wait for Kafka to be available
+KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-ranger-kafka.rangernw:9092}"
+check_tcp_port "Kafka" "${KAFKA_BOOTSTRAP_SERVERS}" 60
+
+# Wait for Solr to be available
+SOLR_URL="${SOLR_URL:-http://ranger-solr.rangernw:8983/solr}"
+check_http_service "Solr" "${SOLR_URL}/" 60
+
+# Start the Solr consumer
+echo "[INFO] Starting Ranger Audit Consumer - Solr..."
+cd ${AUDIT_CONSUMER_HOME_DIR}
+
+# Export environment variables for Java
+export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
+export PATH=$JAVA_HOME/bin:$PATH
+export AUDIT_CONSUMER_LOG_DIR=${AUDIT_CONSUMER_LOG_DIR}
+
+# Set heap size
+AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}"
+export AUDIT_CONSUMER_HEAP
+
+# Set JVM options including logback configuration
+if [ -z "$AUDIT_CONSUMER_OPTS" ]; then
+ AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.dir=${AUDIT_CONSUMER_LOG_DIR}"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.file=ranger-audit-consumer-solr.log"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8"
+fi
+
+# Point to krb5.conf for Kerberos
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ export AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf"
+
+ # Wait for keytabs if Kerberos is enabled
+ if [ -f /home/ranger/scripts/wait_for_keytab.sh ]; then
+ echo "[INFO] Waiting for Kerberos keytabs..."
+ bash /home/ranger/scripts/wait_for_keytab.sh HTTP.keytab
+ bash /home/ranger/scripts/wait_for_keytab.sh rangerauditserver.keytab
+ echo "[INFO] ✓ Keytabs are available"
+ fi
+fi
+
+export AUDIT_CONSUMER_OPTS
+
+echo "[INFO] JAVA_HOME: ${JAVA_HOME}"
+echo "[INFO] AUDIT_CONSUMER_HEAP: ${AUDIT_CONSUMER_HEAP}"
+echo "[INFO] AUDIT_CONSUMER_OPTS: ${AUDIT_CONSUMER_OPTS}"
+
+# Build classpath from WAR file
+WEBAPP_ROOT="${AUDIT_CONSUMER_HOME_DIR}/webapp"
+WAR_FILE="${WEBAPP_ROOT}/ranger-audit-consumer-solr.war"
+WEBAPP_DIR="${WEBAPP_ROOT}/ranger-audit-consumer-solr"
+
+# Extract WAR if not already extracted
+if [ -f "${WAR_FILE}" ] && [ ! -d "${WEBAPP_DIR}" ]; then
+ echo "[INFO] Extracting WAR file..."
+ mkdir -p "${WEBAPP_DIR}"
+ cd "${WEBAPP_DIR}"
+ jar xf "${WAR_FILE}"
+ cd -
+fi
+
+# Build classpath
+RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes"
+for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+done
+
+# Add libext directory if it exists
+if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then
+ for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do
+ if [ -f "${jar}" ]; then
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+ fi
+ done
+fi
+
+export RANGER_CLASSPATH
+
+echo "[INFO] Starting Ranger Audit Consumer - Solr..."
+echo "[INFO] Webapp dir: ${WEBAPP_DIR}"
+java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \
+ -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-solr-site.xml \
+ -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \
+ -cp "${RANGER_CLASSPATH}" \
+ org.apache.ranger.audit.consumer.SolrConsumerApplication \
+ >> ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>&1 &
+
+PID=$!
+echo $PID > ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.pid
+
+echo "[INFO] Ranger Audit Consumer - Solr started with PID: $PID"
+
+# Keep the container running by tailing logs
+tail -f ${AUDIT_CONSUMER_LOG_DIR}/catalina.out 2>/dev/null
diff --git a/dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh
new file mode 100755
index 000000000..827e5d4ec
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/audit-server/ranger-audit-server.sh
@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+AUDIT_SERVER_HOME_DIR="${AUDIT_SERVER_HOME_DIR:-/opt/ranger-audit-server}"
+AUDIT_SERVER_CONF_DIR="${AUDIT_SERVER_CONF_DIR:-/opt/ranger-audit-server/conf}"
+AUDIT_SERVER_LOG_DIR="${AUDIT_SERVER_LOG_DIR:-/var/log/ranger/ranger-audit-server}"
+
+# Create log directory if it doesn't exist
+mkdir -p ${AUDIT_SERVER_LOG_DIR}
+chown -R ranger:ranger /var/log/ranger 2>/dev/null || true
+
+echo "=========================================="
+echo "Starting Ranger Audit Server Service..."
+echo "=========================================="
+echo "AUDIT_SERVER_HOME_DIR: ${AUDIT_SERVER_HOME_DIR}"
+echo "AUDIT_SERVER_CONF_DIR: ${AUDIT_SERVER_CONF_DIR}"
+echo "AUDIT_SERVER_LOG_DIR: ${AUDIT_SERVER_LOG_DIR}"
+echo "=========================================="
+
+# Source service check functions
+source /home/ranger/scripts/service-check-functions.sh
+
+# Quick check for Kafka availability
+# The audit server has a built-in recovery/spool mechanism for when Kafka is unavailable
+KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-ranger-kafka.rangernw:9092}"
+if check_tcp_port "Kafka" "${KAFKA_BOOTSTRAP_SERVERS}" 30; then
+ echo "[INFO] Kafka is available at startup"
+else
+ echo "[INFO] Kafka not immediately available - audit server will use recovery/spool mechanism"
+fi
+
+# Start the audit server
+echo "[INFO] Starting Ranger Audit Server Service..."
+cd ${AUDIT_SERVER_HOME_DIR}
+
+# Export environment variables for Java and audit server
+export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
+export PATH=$JAVA_HOME/bin:$PATH
+export AUDIT_SERVER_LOG_DIR=${AUDIT_SERVER_LOG_DIR}
+
+# Set heap size
+AUDIT_SERVER_HEAP="${AUDIT_SERVER_HEAP:--Xms512m -Xmx2g}"
+export AUDIT_SERVER_HEAP
+
+# Set JVM options including logback configuration
+if [ -z "$AUDIT_SERVER_OPTS" ]; then
+ AUDIT_SERVER_OPTS="-Dlogback.configurationFile=${AUDIT_SERVER_CONF_DIR}/logback.xml"
+ AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Daudit.server.log.dir=${AUDIT_SERVER_LOG_DIR}"
+ AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Daudit.server.log.file=ranger-audit-server.log"
+ AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Djava.net.preferIPv4Stack=true -server"
+ AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200"
+ AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:InitiatingHeapOccupancyPercent=35"
+ AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8"
+fi
+
+# Point to krb5.conf for Kerberos
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ export AUDIT_SERVER_OPTS="${AUDIT_SERVER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf"
+
+ # Wait for keytabs if Kerberos is enabled
+ if [ -f /home/ranger/scripts/wait_for_keytab.sh ]; then
+ echo "[INFO] Waiting for Kerberos keytabs..."
+ bash /home/ranger/scripts/wait_for_keytab.sh HTTP.keytab
+ bash /home/ranger/scripts/wait_for_keytab.sh rangerauditserver.keytab
+ echo "[INFO] ✓ Keytabs are available"
+ fi
+fi
+
+export AUDIT_SERVER_OPTS
+
+echo "[INFO] JAVA_HOME: ${JAVA_HOME}"
+echo "[INFO] AUDIT_SERVER_HEAP: ${AUDIT_SERVER_HEAP}"
+echo "[INFO] AUDIT_SERVER_OPTS: ${AUDIT_SERVER_OPTS}"
+
+# Build classpath from WAR file
+WEBAPP_ROOT="${AUDIT_SERVER_HOME_DIR}/webapp"
+WAR_FILE="${WEBAPP_ROOT}/ranger-audit-server-service.war"
+WEBAPP_DIR="${WEBAPP_ROOT}/ranger-audit-server-service"
+
+# Extract WAR if not already extracted
+if [ -f "${WAR_FILE}" ] && [ ! -d "${WEBAPP_DIR}" ]; then
+ echo "[INFO] Extracting WAR file..."
+ mkdir -p "${WEBAPP_DIR}"
+ cd "${WEBAPP_DIR}"
+ jar xf "${WAR_FILE}"
+ cd -
+fi
+
+# Build classpath
+RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes"
+for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+done
+
+# Add libext directory if it exists
+if [ -d "${AUDIT_SERVER_HOME_DIR}/libext" ]; then
+ for jar in "${AUDIT_SERVER_HOME_DIR}"/libext/*.jar; do
+ if [ -f "${jar}" ]; then
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+ fi
+ done
+fi
+
+export RANGER_CLASSPATH
+
+echo "[INFO] Starting Ranger Audit Server Service..."
+echo "[INFO] Webapp dir: ${WEBAPP_DIR}"
+java ${AUDIT_SERVER_HEAP} ${AUDIT_SERVER_OPTS} \
+ -Daudit.config=${AUDIT_SERVER_CONF_DIR}/ranger-audit-server-site.xml \
+ -cp "${RANGER_CLASSPATH}" \
+ org.apache.ranger.audit.server.AuditServerApplication \
+ >> ${AUDIT_SERVER_LOG_DIR}/catalina.out 2>&1 &
+
+PID=$!
+echo $PID > ${AUDIT_SERVER_LOG_DIR}/ranger-audit-server.pid
+
+echo "[INFO] Ranger Audit Server started with PID: $PID"
+
+# Keep the container running by tailing logs
+tail -f ${AUDIT_SERVER_LOG_DIR}/catalina.out 2>/dev/null
diff --git a/dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh b/dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh
new file mode 100755
index 000000000..6a106d164
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/audit-server/service-check-functions.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Reusable service availability check functions
+#
+
+# Check if a TCP port is available
+# Usage: check_tcp_port "ServiceName" "host:port" [max_wait_seconds]
+check_tcp_port() {
+ local SERVICE_NAME=$1
+ local HOST_PORT=$2
+ local MAX_WAIT=${3:-60}
+
+ # Parse host and port
+ local HOST="${HOST_PORT%%:*}"
+ local PORT="${HOST_PORT##*:}"
+
+ echo "[INFO] Waiting for ${SERVICE_NAME}..."
+ local WAIT_COUNT=0
+
+ while ! timeout 1 bash -c "echo > /dev/tcp/${HOST}/${PORT}" 2>/dev/null; do
+ WAIT_COUNT=$((WAIT_COUNT+1))
+ if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
+ echo "[WARN] ${SERVICE_NAME} not available after ${MAX_WAIT} seconds, continuing anyway..."
+ return 1
+ fi
+ echo "[INFO] Waiting for ${SERVICE_NAME}... ($WAIT_COUNT/$MAX_WAIT)"
+ sleep 2
+ done
+
+ echo "[INFO] ✓ ${SERVICE_NAME} is available at ${HOST}:${PORT}"
+ return 0
+}
+
+# Check if an HTTP service is available
+# Usage: check_http_service "ServiceName" "http://host:port/path" [max_wait_seconds]
+check_http_service() {
+ local SERVICE_NAME=$1
+ local URL=$2
+ local MAX_WAIT=${3:-60}
+
+ echo "[INFO] Waiting for ${SERVICE_NAME}..."
+ local WAIT_COUNT=0
+
+ while ! curl -f -s -m 2 "${URL}" > /dev/null 2>&1; do
+ WAIT_COUNT=$((WAIT_COUNT+1))
+ if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
+ echo "[WARN] ${SERVICE_NAME} not available after ${MAX_WAIT} seconds, continuing anyway..."
+ return 1
+ fi
+ echo "[INFO] Waiting for ${SERVICE_NAME}... ($WAIT_COUNT/$MAX_WAIT)"
+ sleep 2
+ done
+
+ echo "[INFO] ✓ ${SERVICE_NAME} is available at ${URL}"
+ return 0
+}
diff --git a/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml b/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml
index ab1cc9aee..58525a1bc 100644
--- a/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml
+++ b/dev-support/ranger-docker/scripts/hadoop/hdfs-site.xml
@@ -60,4 +60,16 @@
<name>dfs.web.authentication.kerberos.keytab</name>
<value>/etc/keytabs/HTTP.keytab</value>
</property>
+ <property>
+ <name>dfs.secondary.namenode.kerberos.principal</name>
+ <value>nn/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+ </property>
+ <property>
+ <name>dfs.secondary.namenode.keytab.file</name>
+ <value>/etc/keytabs/nn.keytab</value>
+ </property>
+ <property>
+ <name>dfs.secondary.namenode.kerberos.internal.spnego.principal</name>
+ <value>HTTP/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+ </property>
</configuration>
diff --git a/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties b/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties
index fb876ee27..eae056a9d 100644
--- a/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/hadoop/ranger-hdfs-plugin-install.properties
@@ -20,7 +20,7 @@ COMPONENT_INSTALL_DIR_NAME=/opt/hadoop
CUSTOM_USER=hdfs
CUSTOM_GROUP=hadoop
-XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.IS_ENABLED=false
XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
@@ -39,7 +39,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
-XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.ENABLE=false
XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
XAAUDIT.SOLR.USER=NONE
XAAUDIT.SOLR.PASSWORD=NONE
@@ -64,7 +64,7 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
-XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.ENABLE=false
XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/hdfs/spool
@@ -90,3 +90,7 @@ SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
SSL_KEYSTORE_PASSWORD=myKeyFilePassword
SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
SSL_TRUSTSTORE_PASSWORD=changeit
+
+XAAUDIT.AUDITSERVER.ENABLE=true
+XAAUDIT.AUDITSERVER.URL=http://ranger-audit-server.rangernw:7081
+XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/http/spool
diff --git a/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties b/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties
index 0dae26f43..f7308683b 100644
--- a/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties
+++ b/dev-support/ranger-docker/scripts/hive/ranger-hive-plugin-install.properties
@@ -21,7 +21,7 @@ UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
CUSTOM_USER=hive
CUSTOM_GROUP=hadoop
-XAAUDIT.SOLR.IS_ENABLED=true
+XAAUDIT.SOLR.IS_ENABLED=false
XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
@@ -40,7 +40,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
-XAAUDIT.SOLR.ENABLE=true
+XAAUDIT.SOLR.ENABLE=false
XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
XAAUDIT.SOLR.USER=NONE
XAAUDIT.SOLR.PASSWORD=NONE
@@ -65,7 +65,7 @@ XAAUDIT.ELASTICSEARCH.INDEX=NONE
XAAUDIT.ELASTICSEARCH.PORT=NONE
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
-XAAUDIT.HDFS.ENABLE=true
+XAAUDIT.HDFS.ENABLE=false
XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool
@@ -91,3 +91,7 @@ SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks
SSL_KEYSTORE_PASSWORD=myKeyFilePassword
SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks
SSL_TRUSTSTORE_PASSWORD=changeit
+
+XAAUDIT.AUDITSERVER.ENABLE=true
+XAAUDIT.AUDITSERVER.URL=http://ranger-audit-server.rangernw:7081
+XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool
diff --git a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh
index f41f7322a..067042ada 100755
--- a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh
+++ b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh
@@ -39,25 +39,44 @@ cd ${RANGER_HOME}/ranger-kafka-plugin
sed -i 's/localhost:2181/ranger-zk.rangernw:2181/' ${KAFKA_HOME}/config/server.properties
-cat <<EOF >> ${KAFKA_HOME}/config/server.properties
-# Enable SASL/GSSAPI mechanism
-sasl.enabled.mechanisms=GSSAPI
-sasl.mechanism.inter.broker.protocol=GSSAPI
-security.inter.broker.protocol=SASL_PLAINTEXT
-
-# Listener configuration
-listeners=SASL_PLAINTEXT://:9092
-advertised.listeners=SASL_PLAINTEXT://ranger-kafka.rangernw:9092
-
-# JAAS configuration for Kerberos
-listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \
-useKeyTab=true \
-storeKey=true \
-keyTab="/etc/keytabs/kafka.keytab" \
-principal="kafka/ranger-kafka.rangernw@EXAMPLE.COM";
-
-# Kerberos service name
-sasl.kerberos.service.name=kafka
-
-authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ echo "Configuring Kafka with Kerberos (SASL_PLAINTEXT)"
+ cat <<EOF >> ${KAFKA_HOME}/config/server.properties
+
+ # Enable SASL/GSSAPI mechanism
+ sasl.enabled.mechanisms=GSSAPI
+ sasl.mechanism.inter.broker.protocol=GSSAPI
+ security.inter.broker.protocol=SASL_PLAINTEXT
+
+ # Listener configuration
+ listeners=SASL_PLAINTEXT://:9092
+ advertised.listeners=SASL_PLAINTEXT://ranger-kafka.rangernw:9092
+
+ # JAAS configuration for Kerberos
+ listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \
+ useKeyTab=true \
+ storeKey=true \
+ keyTab="/etc/keytabs/kafka.keytab" \
+ principal="kafka/ranger-kafka.rangernw@EXAMPLE.COM";
+
+ # Kerberos service name
+ sasl.kerberos.service.name=kafka
+
+ # Ranger authorization
+ authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer
+
+ # Super users bypass Ranger authorization for admin operations
+ super.users=User:kafka
+EOF
+else
+ echo "Configuring Kafka with PLAINTEXT (no Kerberos)"
+ cat <<EOF >> ${KAFKA_HOME}/config/server.properties
+ # Listener configuration (PLAINTEXT - no authentication)
+ listeners=PLAINTEXT://:9092
+ advertised.listeners=PLAINTEXT://ranger-kafka.rangernw:9092
+
+ # Ranger authorization
+ # disabling Ranger Plugin for now.
+ # authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer
EOF
+fi
diff --git a/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh b/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh
index cd5e93eff..1e22101c3 100755
--- a/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh
+++ b/dev-support/ranger-docker/scripts/kafka/ranger-kafka.sh
@@ -42,4 +42,13 @@ then
fi
fi
-su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config KAFKA_OPTS='-Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/kafka/config/kafka-server-jaas.conf' ./bin/kafka-server-start.sh config/server.properties" kafka
+# Configure KAFKA_OPTS based on KERBEROS_ENABLED
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ echo "Starting Kafka with Kerberos authentication (SASL_PLAINTEXT)"
+ KAFKA_OPTS='-Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/kafka/config/kafka-server-jaas.conf'
+else
+ echo "Starting Kafka with PLAINTEXT (no Kerberos authentication)"
+ KAFKA_OPTS=''
+fi
+
+su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config KAFKA_OPTS='${KAFKA_OPTS}' ./bin/kafka-server-start.sh config/server.properties" kafka
diff --git a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh
index 7ee36e23f..08ca178ec 100644
--- a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh
+++ b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh
@@ -74,6 +74,19 @@ function create_keytabs() {
create_principal_and_keytab rangeradmin ranger
create_principal_and_keytab rangerlookup ranger
+ create_principal_and_keytab HTTP ranger-audit
+ create_principal_and_keytab rangerauditserver ranger-audit
+
+ # Ranger Audit Server Microservices
+ create_principal_and_keytab HTTP ranger-audit-server
+ create_principal_and_keytab rangerauditserver ranger-audit-server
+
+ create_principal_and_keytab HTTP ranger-audit-consumer-solr
+ create_principal_and_keytab rangerauditserver ranger-audit-consumer-solr
+
+ create_principal_and_keytab HTTP ranger-audit-consumer-hdfs
+ create_principal_and_keytab rangerauditserver ranger-audit-consumer-hdfs
+
create_principal_and_keytab rangertagsync ranger-tagsync
create_principal_and_keytab rangerusersync ranger-usersync
@@ -132,7 +145,7 @@ if [ ! -f $DB_DIR/principal ]; then
echo "Database initialized"
create_keytabs
- create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino ranger-opensearch
+ create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-audit-server ranger-audit-consumer-solr ranger-audit-consumer-hdfs ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino ranger-opensearch
else
echo "KDC DB already exists; skipping create"
fi
diff --git a/distro/pom.xml b/distro/pom.xml
index 709cf9f83..cb8277021 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -66,6 +66,26 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-consumer-hdfs</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-consumer-solr</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger-audit-dest-cloudwatch</artifactId>
@@ -102,6 +122,13 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-server-service</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger-authn</artifactId>
@@ -472,6 +499,9 @@
<descriptor>src/main/assembly/plugin-presto.xml</descriptor>
<descriptor>src/main/assembly/plugin-trino.xml</descriptor>
<descriptor>src/main/assembly/sample-client.xml</descriptor>
+ <descriptor>src/main/assembly/ranger-audit-server-service.xml</descriptor>
+ <descriptor>src/main/assembly/ranger-audit-consumer-solr.xml</descriptor>
+ <descriptor>src/main/assembly/ranger-audit-consumer-hdfs.xml</descriptor>
</descriptors>
</configuration>
</execution>
diff --git a/distro/src/main/assembly/hdfs-agent.xml b/distro/src/main/assembly/hdfs-agent.xml
index bfbe915c7..56b3b14e9 100644
--- a/distro/src/main/assembly/hdfs-agent.xml
+++ b/distro/src/main/assembly/hdfs-agent.xml
@@ -72,6 +72,7 @@
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
<include>org.apache.ranger:ranger-audit-core</include>
+ <include>org.apache.ranger:ranger-audit-dest-auditserver</include>
<include>org.apache.ranger:ranger-audit-dest-hdfs</include>
<include>org.apache.ranger:ranger-audit-dest-solr</include>
<include>org.apache.ranger:ranger-authz-api</include>
diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml
index 7a72129c5..d20d3f7cf 100644
--- a/distro/src/main/assembly/hive-agent.xml
+++ b/distro/src/main/assembly/hive-agent.xml
@@ -42,6 +42,7 @@
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
<include>org.apache.ranger:ranger-audit-core</include>
+ <include>org.apache.ranger:ranger-audit-dest-auditserver</include>
<include>org.apache.ranger:ranger-audit-dest-hdfs</include>
<include>org.apache.ranger:ranger-audit-dest-solr</include>
<include>org.apache.ranger:ranger-authz-api</include>
diff --git a/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml b/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml
new file mode 100644
index 000000000..0130321a6
--- /dev/null
+++ b/distro/src/main/assembly/ranger-audit-consumer-hdfs.xml
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>ranger-audit-consumer-hdfs</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <baseDirectory>${project.parent.name}-${project.version}-ranger-audit-consumer-hdfs</baseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <!-- Scripts directory -->
+ <fileSet>
+ <outputDirectory>bin</outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/scripts</directory>
+ <includes>
+ <include>**/*.sh</include>
+ <include>**/*.py</include>
+ </includes>
+ <fileMode>755</fileMode>
+ </fileSet>
+
+ <!-- Configuration files -->
+ <fileSet>
+ <outputDirectory>conf</outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf</directory>
+ <includes>
+ <include>ranger-audit-consumer-hdfs-site.xml</include>
+ <include>core-site.xml</include>
+ <include>hdfs-site.xml</include>
+ <include>logback.xml</include>
+ </includes>
+ <fileMode>644</fileMode>
+ </fileSet>
+
+ <!-- Version file -->
+ <fileSet>
+ <outputDirectory></outputDirectory>
+ <directory>${project.build.directory}</directory>
+ <includes>
+ <include>version</include>
+ </includes>
+ <fileMode>444</fileMode>
+ </fileSet>
+
+ <!-- Create empty libext directory -->
+ <fileSet>
+ <outputDirectory>libext</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
+
+ <!-- Create logs directory -->
+ <fileSet>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
+
+ <!-- NOTICE file -->
+ <fileSet>
+ <outputDirectory></outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/</directory>
+ <includes>
+ <include>NOTICE.txt</include>
+ </includes>
+ <fileMode>644</fileMode>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <!-- WAR file -->
+ <file>
+ <outputDirectory>webapp</outputDirectory>
+ <source>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-hdfs/target/ranger-audit-consumer-hdfs-${project.version}.war</source>
+ <destName>ranger-audit-consumer-hdfs.war</destName>
+ </file>
+ </files>
+</assembly>
diff --git a/distro/src/main/assembly/ranger-audit-consumer-solr.xml b/distro/src/main/assembly/ranger-audit-consumer-solr.xml
new file mode 100644
index 000000000..edd06938a
--- /dev/null
+++ b/distro/src/main/assembly/ranger-audit-consumer-solr.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>ranger-audit-consumer-solr</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <baseDirectory>${project.parent.name}-${project.version}-ranger-audit-consumer-solr</baseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <!-- Scripts directory -->
+ <fileSet>
+ <outputDirectory>bin</outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/scripts</directory>
+ <includes>
+ <include>**/*.sh</include>
+ <include>**/*.py</include>
+ </includes>
+ <fileMode>755</fileMode>
+ </fileSet>
+
+ <!-- Configuration files -->
+ <fileSet>
+ <outputDirectory>conf</outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/src/main/resources/conf</directory>
+ <includes>
+ <include>ranger-audit-consumer-solr-site.xml</include>
+ <include>logback.xml</include>
+ </includes>
+ <fileMode>644</fileMode>
+ </fileSet>
+
+ <!-- Version file -->
+ <fileSet>
+ <outputDirectory></outputDirectory>
+ <directory>${project.build.directory}</directory>
+ <includes>
+ <include>version</include>
+ </includes>
+ <fileMode>444</fileMode>
+ </fileSet>
+
+ <!-- Create empty libext directory -->
+ <fileSet>
+ <outputDirectory>libext</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
+
+ <!-- Create logs directory -->
+ <fileSet>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
+
+ <!-- NOTICE file -->
+ <fileSet>
+ <outputDirectory></outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/</directory>
+ <includes>
+ <include>NOTICE.txt</include>
+ </includes>
+ <fileMode>644</fileMode>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <!-- WAR file -->
+ <file>
+ <outputDirectory>webapp</outputDirectory>
+ <source>${project.parent.basedir}/ranger-audit-server/ranger-audit-consumer-solr/target/ranger-audit-consumer-solr-${project.version}.war</source>
+ <destName>ranger-audit-consumer-solr.war</destName>
+ </file>
+ </files>
+</assembly>
diff --git a/distro/src/main/assembly/ranger-audit-server-service.xml b/distro/src/main/assembly/ranger-audit-server-service.xml
new file mode 100644
index 000000000..53540f40f
--- /dev/null
+++ b/distro/src/main/assembly/ranger-audit-server-service.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>ranger-audit-server-service</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <baseDirectory>${project.parent.name}-${project.version}-ranger-audit-server-service</baseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <!-- Scripts directory -->
+ <fileSet>
+ <outputDirectory>bin</outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/scripts</directory>
+ <includes>
+ <include>**/*.sh</include>
+ <include>**/*.py</include>
+ </includes>
+ <fileMode>755</fileMode>
+ </fileSet>
+
+ <!-- Configuration files -->
+ <fileSet>
+ <outputDirectory>conf</outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/src/main/resources/conf</directory>
+ <includes>
+ <include>ranger-audit-server-site.xml</include>
+ <include>logback.xml</include>
+ </includes>
+ <fileMode>644</fileMode>
+ </fileSet>
+
+ <!-- Version file -->
+ <fileSet>
+ <outputDirectory></outputDirectory>
+ <directory>${project.build.directory}</directory>
+ <includes>
+ <include>version</include>
+ </includes>
+ <fileMode>444</fileMode>
+ </fileSet>
+
+ <!-- Create empty libext directory -->
+ <fileSet>
+ <outputDirectory>libext</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
+
+ <!-- Create logs directory -->
+ <fileSet>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
+
+ <!-- NOTICE file -->
+ <fileSet>
+ <outputDirectory></outputDirectory>
+ <directory>${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/</directory>
+ <includes>
+ <include>NOTICE.txt</include>
+ </includes>
+ <fileMode>644</fileMode>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <!-- WAR file -->
+ <file>
+ <outputDirectory>webapp</outputDirectory>
+ <source>${project.parent.basedir}/ranger-audit-server/ranger-audit-server-service/target/ranger-audit-server-service-${project.version}.war</source>
+ <destName>ranger-audit-server-service.war</destName>
+ </file>
+ </files>
+</assembly>
diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
index 49244c66b..3be456cd4 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
+++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
@@ -88,3 +88,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA
xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists
xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists
xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists
+
+#Audit Service Destination
+xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists
+xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml
index e16ea590d..2b8d8556c 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit.xml
+++ b/hdfs-agent/conf/ranger-hdfs-audit.xml
@@ -211,6 +211,21 @@
<property>
<name>xasecure.audit.solr.solr_url</name>
<value>http://localhost:6083/solr/ranger_audits</value>
- </property>
-
+ </property>
+
+ <!-- Audit Server destination -->
+ <property>
+ <name>xasecure.audit.destination.auditserver</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.auditserver.url</name>
+ <value>http://localhost:7081</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.auditserver.batch.filespool.dir</name>
+ <value>/var/log/hive/audit/http/spool</value>
+ </property>
</configuration>
diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml
index 910f837ec..1affa6c4c 100644
--- a/hdfs-agent/pom.xml
+++ b/hdfs-agent/pom.xml
@@ -114,6 +114,11 @@
<artifactId>httpcore</artifactId>
<version>${httpcomponents.httpcore.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-auditserver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger-audit-dest-hdfs</artifactId>
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index fedecc1d2..20deb540c 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -116,6 +116,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+#Audit Server Provider
+XAAUDIT.AUDITSERVER.ENABLE=false
+XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081
+XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool
# End of V3 properties
#
diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java
index 4f7e74cd3..5ce5b5631 100644
--- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java
+++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuditHandler.java
@@ -94,7 +94,7 @@ public void processResult(RangerAccessResult result) {
setRequestData();
auditEvent.setAction(getAccessType(request.getAccessType()));
- auditEvent.setAdditionalInfo(getAdditionalInfo(request));
+ auditEvent.setAdditionalInfo(getAdditionalInfo(request, result));
Set<String> tags = getTags(request);
@@ -108,8 +108,8 @@ public void processResult(RangerAccessResult result) {
}
@Override
- public String getAdditionalInfo(RangerAccessRequest request) {
- String additionalInfo = super.getAdditionalInfo(request);
+ public String getAdditionalInfo(RangerAccessRequest request, RangerAccessResult result) {
+ String additionalInfo = super.getAdditionalInfo(request, result);
Map<String, String> addInfoMap = JsonUtils.jsonToMapStringString(additionalInfo);
if (addInfoMap == null || addInfoMap.isEmpty()) {
diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg
index 651372d3e..c81a52746 100644
--- a/hive-agent/conf/ranger-hive-audit-changes.cfg
+++ b/hive-agent/conf/ranger-hive-audit-changes.cfg
@@ -89,3 +89,8 @@ xasecure.audit.log4j.async.max.queue.size %XAAUDIT.LOG4J.ASYNC.MA
xasecure.audit.log4j.async.max.flush.interval.ms %XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS% mod create-if-not-exists
xasecure.audit.destination.log4j %XAAUDIT.LOG4J.DESTINATION.LOG4J% mod create-if-not-exists
xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER% mod create-if-not-exists
+
+#Audit Service Destination
+xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists
+xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml
index d42e58935..5693a1be1 100644
--- a/hive-agent/conf/ranger-hive-audit.xml
+++ b/hive-agent/conf/ranger-hive-audit.xml
@@ -211,6 +211,21 @@
<property>
<name>xasecure.audit.solr.solr_url</name>
<value>http://localhost:6083/solr/ranger_audits</value>
- </property>
-
+ </property>
+
+ <!-- Audit Server HTTP destination -->
+ <property>
+ <name>xasecure.audit.destination.auditserver</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.auditserver.url</name>
+ <value>http://localhost:7081</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.auditserver.batch.filespool.dir</name>
+ <value>/var/log/hive/audit/http/spool</value>
+ </property>
</configuration>
diff --git a/hive-agent/pom.xml b/hive-agent/pom.xml
index 825f0f1e5..8a800e1e0 100644
--- a/hive-agent/pom.xml
+++ b/hive-agent/pom.xml
@@ -182,6 +182,11 @@
<artifactId>httpcore</artifactId>
<version>${httpcomponents.httpcore.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-auditserver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger-audit-dest-hdfs</artifactId>
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 4d8976b6d..8ac29281d 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -119,6 +119,10 @@ XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+#Audit Server Provider
+XAAUDIT.AUDITSERVER.ENABLE=false
+XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081
+XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool
# End of V3 properties
diff --git a/pom.xml b/pom.xml
index e703f964a..1136edebb 100755
--- a/pom.xml
+++ b/pom.xml
@@ -99,6 +99,8 @@
<!-- GCP HSM -->
<google.cloud.kms>2.3.0</google.cloud.kms>
+ <google.guava.version>33.4.8-jre</google.guava.version>
+ <google.re2j.version>1.2</google.re2j.version>
<googlecode.log4jdbc.version>1.2</googlecode.log4jdbc.version>
<graalvm.version>22.3.0</graalvm.version>
<gson.version>2.9.0</gson.version>
@@ -165,6 +167,7 @@
<logback.version>1.3.14</logback.version>
<lucene.version>8.11.3</lucene.version>
<maven-plugin-sortpom.version>3.0.1</maven-plugin-sortpom.version>
+ <maven-war-plugin.version>3.4.0</maven-war-plugin.version>
<maven.exec.plugin.version>1.6.0</maven.exec.plugin.version>
<maven.pmd.plugin.version>3.14.0</maven.pmd.plugin.version>
<maven.surefire.plugin.version>3.0.0-M6</maven.surefire.plugin.version>
@@ -824,6 +827,7 @@
<module>plugin-trino</module>
<module>plugin-yarn</module>
<module>ranger-atlas-plugin-shim</module>
+ <module>ranger-audit-server</module>
<module>ranger-authn</module>
<module>ranger-common-ha</module>
<module>ranger-elasticsearch-plugin-shim</module>
@@ -1152,6 +1156,7 @@
<module>plugin-trino</module>
<module>plugin-yarn</module>
<module>ranger-atlas-plugin-shim</module>
+ <module>ranger-audit-server</module>
<module>ranger-authn</module>
<module>ranger-common-ha</module>
<module>ranger-elasticsearch-plugin-shim</module>
@@ -1236,6 +1241,7 @@
<module>plugin-trino</module>
<module>plugin-yarn</module>
<module>ranger-atlas-plugin-shim</module>
+ <module>ranger-audit-server</module>
<module>ranger-authn</module>
<module>ranger-common-ha</module>
<module>ranger-elasticsearch-plugin-shim</module>
@@ -1302,6 +1308,27 @@
</execution>
</executions>
</plugin>
+ <!--
+ Plugin sortpom purpose is to avoid merge conflicts and maintain consistent ordering
+ run command for new pom.xml: mvn com.github.ekryd.sortpom:sortpom-maven-plugin:sort
+ -->
+ <plugin>
+ <groupId>com.github.ekryd.sortpom</groupId>
+ <artifactId>sortpom-maven-plugin</artifactId>
+ <version>4.0.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>sort</goal>
+ </goals>
+ <configuration>
+ <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
+ <sortDependencies>groupId,artifactId</sortDependencies>
+ <keepBlankLines>true</keepBlankLines>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</profile>
diff --git a/ranger-audit-server/pom.xml b/ranger-audit-server/pom.xml
new file mode 100644
index 000000000..1652dc11a
--- /dev/null
+++ b/ranger-audit-server/pom.xml
@@ -0,0 +1,460 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-server</artifactId>
+ <packaging>pom</packaging>
+ <name>Ranger Audit Server Parent</name>
+ <description>Parent project for Ranger Audit Server microservices - Producer and Consumer services</description>
+
+ <!-- Child Modules -->
+ <modules>
+ <module>ranger-audit-common</module>
+ <module>ranger-audit-consumer-hdfs</module>
+ <module>ranger-audit-consumer-solr</module>
+ <module>ranger-audit-server-service</module>
+ </modules>
+
+ <properties>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <version>${guice.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ <version>${jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <version>${jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-spring</artifactId>
+ <version>${jersey-spring.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>${jaxb-impl.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons.codec.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax.servlet.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop.thirdparty</groupId>
+ <artifactId>hadoop-shaded-guava</artifactId>
+ <version>${hadoop-shaded-guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ <version>${httpcomponents.httpasyncclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-nio</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-grouping</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-highlighter</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-join</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-misc</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queries</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-sandbox</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial-extras</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial3d</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-suggest</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ <version>${orc.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-shims</artifactId>
+ <version>${orc.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-hdfs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-solr</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-authn</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-cred</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-annotations-api</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-el</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-jasper</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-websocket</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-tx</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-config</artifactId>
+ <version>${springframework.security.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-web</artifactId>
+ <version>${springframework.security.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.1</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ <version>${maven-war-plugin.version}</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <version>${maven.pmd.plugin.version}</version>
+ <configuration>
+ <rulesets>
+ <ruleset>${project.parent.parent.basedir}/dev-support/ranger-pmd-ruleset.xml</ruleset>
+ </rulesets>
+ <sourceEncoding>UTF-8</sourceEncoding>
+ <failOnViolation>true</failOnViolation>
+ <linkXRef>false</linkXRef>
+ <includeTests>true</includeTests>
+ <verbose>true</verbose>
+ <excludeRoots>
+ <excludeRoot>${basedir}/src/main/generated</excludeRoot>
+ </excludeRoots>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <phase>verify</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/ranger-audit-server/ranger-audit-common/pom.xml b/ranger-audit-server/ranger-audit-common/pom.xml
new file mode 100644
index 000000000..9c6c6de54
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/pom.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-server</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-common</artifactId>
+ <packaging>jar</packaging>
+ <name>Ranger Audit Common</name>
+ <description>Shared utilities and base classes for Ranger Audit Server services</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-annotations-api</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java
new file mode 100644
index 000000000..3d9753648
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer.kafka;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+
+import java.util.Properties;
+
+/**
+ * Interface for Ranger Kafka consumers that consume audit events from Kafka
+ * and forward them to various destinations like Solr, HDFS, etc.
+ */
+public interface AuditConsumer extends Runnable {
+ /**
+ * Initialize the consumer with the given properties and property prefix.
+ * This method should set up the destination handler and configure the Kafka consumer.
+ *
+ * @param props Configuration properties
+ * @param propPrefix Property prefix for consumer-specific configuration
+ * @throws Exception if initialization fails
+ */
+ void init(Properties props, String propPrefix) throws Exception;
+
+ /**
+ * Start consuming messages from Kafka topic and process them.
+ * This method should implement the main consumption loop.
+ * Implementation should handle subscription to topics and polling for messages.
+ */
+ @Override
+ void run();
+
+ /**
+ * Get the underlying Kafka consumer instance.
+ *
+ * @return KafkaConsumer instance used by this consumer
+ */
+ KafkaConsumer<String, String> getKafkaConsumer();
+
+ /**
+ * Process a single audit message received from Kafka.
+ * Implementation should handle the specific logic for forwarding
+ * the message to the appropriate destination (Solr, HDFS, etc.).
+ *
+ * @param audit The audit message in JSON format
+ * @throws Exception if message processing fails
+ */
+ void processMessage(String audit) throws Exception;
+
+ /**
+ * Get the topic name this consumer is subscribed to.
+ *
+ * @return Kafka topic name
+ */
+ String getTopicName();
+
+ /**
+ * Shutdown the consumer gracefully.
+ * This method should clean up resources and close connections.
+ */
+ void shutdown();
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java
new file mode 100644
index 000000000..0964da050
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerBase.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.audit.consumer.kafka;
+
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.utils.AuditMessageQueueUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+public class AuditConsumerBase {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditConsumerBase.class);
+
+ public Properties consumerProps = new Properties();
+ public KafkaConsumer<String, String> consumer;
+ public String topicName;
+ public String consumerGroupId;
+
+ public AuditConsumerBase(Properties props, String propPrefix, String consumerGroupId) throws Exception {
+ AuditMessageQueueUtils auditMessageQueueUtils = new AuditMessageQueueUtils(props);
+
+ this.consumerGroupId = getConsumerGroupId(props, propPrefix, consumerGroupId);
+
+ consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS));
+ consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, this.consumerGroupId);
+ consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringDeserializer"));
+ consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringDeserializer"));
+
+ String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL);
+ consumerProps.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol);
+
+ consumerProps.put(AuditServerConstants.PROP_SASL_MECHANISM, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM));
+ consumerProps.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME);
+
+ if (securityProtocol.toUpperCase().contains(AuditServerConstants.PROP_SECURITY_PROTOCOL_VALUE)) {
+ consumerProps.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, auditMessageQueueUtils.getJAASConfig(props, propPrefix));
+ }
+
+ consumerProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_MAX_POLL_RECORDS, AuditServerConstants.DEFAULT_MAX_POLL_RECORDS));
+ consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+
+ // Configure re-balancing parameters for subscribe mode
+ // These ensure stable consumer group behavior during horizontal scaling
+ int sessionTimeoutMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_SESSION_TIMEOUT_MS, AuditServerConstants.DEFAULT_SESSION_TIMEOUT_MS);
+ int maxPollIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_MAX_POLL_INTERVAL_MS, AuditServerConstants.DEFAULT_MAX_POLL_INTERVAL_MS);
+ int heartbeatIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_HEARTBEAT_INTERVAL_MS, AuditServerConstants.DEFAULT_HEARTBEAT_INTERVAL_MS);
+
+ consumerProps.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeoutMs);
+ consumerProps.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs);
+ consumerProps.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, heartbeatIntervalMs);
+
+ // Configure partition assignment strategy
+ String partitionAssignmentStrategy = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_PARTITION_ASSIGNMENT_STRATEGY, AuditServerConstants.DEFAULT_PARTITION_ASSIGNMENT_STRATEGY);
+ consumerProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partitionAssignmentStrategy);
+
+ LOG.info("Consumer '{}' configured for subscription-based partition assignment with re-balancing support", this.consumerGroupId);
+ LOG.info("Re-balancing config - session.timeout.ms: {}, max.poll.interval.ms: {}, heartbeat.interval.ms: {}", sessionTimeoutMs, maxPollIntervalMs, heartbeatIntervalMs);
+ LOG.info("Partition assignment strategy: {}", partitionAssignmentStrategy);
+
+ consumer = new KafkaConsumer<>(consumerProps);
+
+ topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC);
+ }
+
+ private String getConsumerGroupId(Properties props, String propPrefix, String defaultConsumerGroupId) {
+ String configuredGroupId = MiscUtil.getStringProperty(props, propPrefix + ".consumer.group.id");
+ if (configuredGroupId != null && !configuredGroupId.trim().isEmpty()) {
+ return configuredGroupId.trim();
+ }
+ return defaultConsumerGroupId;
+ }
+
+ public KafkaConsumer<String, String> getConsumer() {
+ return consumer;
+ }
+
+ public String getConsumerGroupId() {
+ return consumerGroupId;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java
new file mode 100644
index 000000000..e98675612
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer.kafka;
+
+import java.util.Properties;
+
+/**
+ * Factory interface for creating audit consumer instances.
+ */
+@FunctionalInterface
+public interface AuditConsumerFactory {
+ /**
+ * Create a consumer instance with the given configuration.
+ *
+ * @param props Configuration properties
+ * @param propPrefix Property prefix for consumer configuration
+ * @return Initialized consumer instance ready to run
+ * @throws Exception if consumer creation or initialization fails
+ */
+ AuditConsumer createConsumer(Properties props, String propPrefix) throws Exception;
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java
new file mode 100644
index 000000000..1b002d629
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRebalanceListener.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer.kafka;
+
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Reusable ConsumerRebalanceListener for Kafka consumer group re-balancing.
+ *
+ * This listener handles graceful partition re-balancing by:
+ * - Committing pending offsets before partitions are revoked
+ * - Updating partition assignments when partitions are assigned
+ * - Logging re-balancing events with customizable log prefixes
+ *
+ * Used by both AuditSolrConsumer, AuditHDFSConsumer and any new consumers added to the audit serve, ensuring
+ * no message duplication during scaling operations.
+ */
+public class AuditConsumerRebalanceListener implements ConsumerRebalanceListener {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditConsumerRebalanceListener.class);
+
+ private final String workerId;
+ private final String destinationType;
+ private final String topicName;
+ private final String offsetCommitStrategy;
+ private final String consumerGroupId;
+ private final KafkaConsumer<String, String> workerConsumer;
+ private final Map<TopicPartition, OffsetAndMetadata> pendingOffsets;
+ private final AtomicInteger messagesProcessedSinceLastCommit;
+ private final AtomicLong lastCommitTime;
+ private final List<Integer> assignedPartitions;
+
+ public AuditConsumerRebalanceListener(
+ String workerId,
+ String destinationType,
+ String topicName,
+ String offsetCommitStrategy,
+ String consumerGroupId,
+ KafkaConsumer<String, String> workerConsumer,
+ Map<TopicPartition, OffsetAndMetadata> pendingOffsets,
+ AtomicInteger messagesProcessedSinceLastCommit,
+ AtomicLong lastCommitTime,
+ List<Integer> assignedPartitions) {
+ this.workerId = workerId;
+ this.destinationType = destinationType;
+ this.topicName = topicName;
+ this.offsetCommitStrategy = offsetCommitStrategy;
+ this.consumerGroupId = consumerGroupId;
+ this.workerConsumer = workerConsumer;
+ this.pendingOffsets = pendingOffsets;
+ this.messagesProcessedSinceLastCommit = messagesProcessedSinceLastCommit;
+ this.lastCommitTime = lastCommitTime;
+ this.assignedPartitions = assignedPartitions;
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
+ LOG.info("[{}-REBALANCE] Worker '{}': Partitions REVOKED: {} (count: {})", destinationType, workerId, partitions, partitions.size());
+
+ // Commit pending offsets before partitions are revoked
+ if (!pendingOffsets.isEmpty()) {
+ try {
+ workerConsumer.commitSync(pendingOffsets);
+ LOG.info("[{}-REBALANCE] Worker '{}': Successfully committed {} pending offsets before rebalance",
+ destinationType, workerId, pendingOffsets.size());
+ pendingOffsets.clear();
+ } catch (Exception e) {
+ LOG.error("[{}-REBALANCE] Worker '{}': Failed to commit offsets during rebalance",
+ destinationType, workerId, e);
+ }
+ }
+
+ // Reset counters
+ messagesProcessedSinceLastCommit.set(0);
+ lastCommitTime.set(System.currentTimeMillis());
+ }
+
+ @Override
+ public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
+ LOG.info("[{}-REBALANCE] Worker '{}': Partitions ASSIGNED: {} (count: {})", destinationType, workerId, partitions, partitions.size());
+
+ // Update assigned partitions list for tracking
+ assignedPartitions.clear();
+ for (TopicPartition tp : partitions) {
+ assignedPartitions.add(tp.partition());
+ }
+
+ // Log assignment details
+ LOG.info("[{}-CONSUMER-ASSIGNED] Worker '{}' | Topic: '{}' | Partitions: {} | Offset-Strategy: {} | Consumer-Group: {}",
+ destinationType, workerId, topicName, assignedPartitions, offsetCommitStrategy, consumerGroupId);
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java
new file mode 100644
index 000000000..d1cf56074
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditConsumerRegistry.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer.kafka;
+
+import org.apache.ranger.audit.provider.AuditProviderFactory;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Registry for managing audit consumer factories and instances.
+ * Supports dynamic consumer registration and creation based on configuration.
+ */
+public class AuditConsumerRegistry {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditConsumerRegistry.class);
+
+ private static final AuditConsumerRegistry auditConsumerRegistry = new AuditConsumerRegistry();
+ private final Map<String, AuditConsumerFactory> consumerFactories = new ConcurrentHashMap<>();
+ private final Map<String, AuditConsumer> activeConsumers = new ConcurrentHashMap<>();
+
+ private AuditConsumerRegistry() {
+ LOG.debug("AuditConsumerRegistry instance created");
+ }
+
+ public static AuditConsumerRegistry getInstance() {
+ return auditConsumerRegistry;
+ }
+
+ /**
+ * Register a consumer factory for a specific destination type.
+ * This method can be called early in the application lifecycle (static initializers)
+ * to register consumers before the AuditMessageQueue is initialized.
+ *
+ * @param destinationType The destination type identifier (e.g., "solr", "hdfs", "elasticsearch")
+ * @param factory The factory that creates consumer instances
+ */
+ public void registerFactory(String destinationType, AuditConsumerFactory factory) {
+ if (destinationType == null || destinationType.trim().isEmpty()) {
+ LOG.warn("Attempted to register factory with null or empty destination type");
+ return;
+ }
+
+ if (factory == null) {
+ LOG.warn("Attempted to register null factory for destination type: {}", destinationType);
+ return;
+ }
+
+ AuditConsumerFactory existing = consumerFactories.put(destinationType, factory);
+ if (existing != null) {
+ LOG.warn("Replaced existing factory for destination type: {}", destinationType);
+ } else {
+ LOG.info("Registered consumer factory for destination type: {}", destinationType);
+ }
+ }
+
+ /**
+ * Create consumer instances for all enabled destinations based on configuration.
+ *
+ * @param props Configuration properties
+ * @param propPrefix Property prefix for Kafka configuration
+ * @return List of created consumer instances
+ */
+ public List<AuditConsumer> createConsumers(Properties props, String propPrefix) {
+ LOG.info("==> AuditConsumerRegistry.createConsumers()");
+
+ List<AuditConsumer> consumers = new ArrayList<>();
+
+ for (Map.Entry<String, AuditConsumerFactory> entry : consumerFactories.entrySet()) {
+ String destinationType = entry.getKey();
+ AuditConsumerFactory factory = entry.getValue();
+ String destPropPrefix = AuditProviderFactory.AUDIT_DEST_BASE + "." + destinationType;
+ boolean isEnabled = MiscUtil.getBooleanProperty(props, destPropPrefix, false);
+
+ if (isEnabled) {
+ try {
+ LOG.info("Creating consumer for enabled destination: {}", destinationType);
+ AuditConsumer consumer = factory.createConsumer(props, propPrefix);
+
+ if (consumer != null) {
+ consumers.add(consumer);
+ activeConsumers.put(destinationType, consumer);
+ LOG.info("Successfully created consumer for destination: {}", destinationType);
+ } else {
+ LOG.warn("Factory returned null consumer for destination: {}", destinationType);
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to create consumer for destination: {}", destinationType, e);
+ }
+ } else {
+ LOG.debug("Destination '{}' is disabled (property: {} = false)", destinationType, destPropPrefix);
+ }
+ }
+
+ LOG.info("<== AuditConsumerRegistry.createConsumers(): Created {} consumers out of {} registered factories", consumers.size(), consumerFactories.size());
+
+ return consumers;
+ }
+
+ public Collection<AuditConsumer> getActiveConsumers() {
+ return activeConsumers.values();
+ }
+
+ public Collection<String> getRegisteredDestinationTypes() {
+ return consumerFactories.keySet();
+ }
+
+ /**
+ * Clear all active consumer references.
+ * Called during shutdown after consumers have been stopped.
+ */
+ public void clearActiveConsumers() {
+ LOG.debug("Clearing {} active consumer references", activeConsumers.size());
+ activeConsumers.clear();
+ }
+
+ public int getFactoryCount() {
+ return consumerFactories.size();
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java
new file mode 100644
index 000000000..68b8eda6c
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.server;
+
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * Base configuration class for Ranger Audit Server services.
+ * Can be extended by specific services to load their custom configuration files.
+ */
+public class AuditConfig extends RangerConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditConfig.class);
+ private static volatile AuditConfig sInstance;
+
+ protected AuditConfig() {
+ super();
+ }
+
+ /**
+ * Get the singleton instance of AuditConfig.
+ * Subclasses should override this method to return their specific instance.
+ */
+ public static AuditConfig getInstance() {
+ AuditConfig ret = AuditConfig.sInstance;
+
+ if (ret == null) {
+ synchronized (AuditConfig.class) {
+ ret = AuditConfig.sInstance;
+
+ if (ret == null) {
+ ret = new AuditConfig();
+ AuditConfig.sInstance = ret;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ public Properties getProperties() {
+ return this.getProps();
+ }
+
+ /**
+ * Add a resource file to the configuration.
+ * Subclasses can override to load their specific config files.
+ *
+ * @param resourcePath Path to the resource file (e.g., "conf/ranger-audit-server-site.xml")
+ * @param required Whether this resource is required
+ * @return true if resource was loaded successfully or is optional, false otherwise
+ */
+ protected boolean addAuditResource(String resourcePath, boolean required) {
+ LOG.debug("==> addAuditResource(path={}, required={})", resourcePath, required);
+
+ boolean success = addResourceIfReadable(resourcePath);
+
+ if (success) {
+ LOG.info("Successfully loaded configuration: {}", resourcePath);
+ } else if (required) {
+ LOG.error("Failed to load required configuration: {}", resourcePath);
+ } else {
+ LOG.warn("Failed to load optional configuration: {}", resourcePath);
+ }
+
+ LOG.debug("<== addAuditResource(path={}, required={}), result={}", resourcePath, required, success);
+
+ return success || !required;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java
new file mode 100644
index 000000000..9b3141bc7
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.server;
+
+public class AuditServerConstants {
+ private AuditServerConstants() {}
+
+ public static final String AUDIT_SERVER_APPNAME = "ranger-audit";
+ public static final String AUDIT_SERVER_PROP_PREFIX = "ranger.audit.service.";
+ public static final String JAAS_KRB5_MODULE = "com.sun.security.auth.module.Krb5LoginModule required";
+ public static final String JAAS_USE_KEYTAB = "useKeyTab=true";
+ public static final String JAAS_KEYTAB = "keyTab=\"";
+ public static final String JAAS_STOKE_KEY = "storeKey=true";
+ public static final String JAAS_SERVICE_NAME = "serviceName=kafka";
+ public static final String JAAS_USER_TICKET_CACHE = "useTicketCache=false";
+ public static final String JAAS_PRINCIPAL = "principal=\"";
+ public static final String PROP_BOOTSTRAP_SERVERS = "bootstrap.servers";
+ public static final String PROP_TOPIC_NAME = "topic.name";
+ public static final String PROP_SECURITY_PROTOCOL = "security.protocol";
+ public static final String PROP_SASL_MECHANISM = "sasl.mechanism";
+ public static final String PROP_SASL_JAAS_CONFIG = "sasl.jaas.config";
+ public static final String PROP_SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name";
+ public static final String PROP_REQ_TIMEOUT_MS = "request.timeout.ms";
+ public static final String PROP_CONN_MAX_IDEAL_MS = "connections.max.idle.ms";
+ public static final String PROP_SOLR_DEST_PREFIX = "solr";
+ public static final String PROP_HDFS_DEST_PREFIX = "hdfs";
+ public static final String PROP_CONSUMER_THREAD_COUNT = "consumer.thread.count";
+ public static final String PROP_CONSUMER_OFFSET_COMMIT_STRATEGY = "consumer.offset.commit.strategy";
+ public static final String PROP_CONSUMER_OFFSET_COMMIT_INTERVAL = "consumer.offset.commit.interval.ms";
+ public static final String PROP_CONSUMER_MAX_POLL_RECORDS = "consumer.max.poll.records";
+ public static final String PROP_CONSUMER_SESSION_TIMEOUT_MS = "consumer.session.timeout.ms";
+ public static final String PROP_CONSUMER_MAX_POLL_INTERVAL_MS = "consumer.max.poll.interval.ms";
+ public static final String PROP_CONSUMER_HEARTBEAT_INTERVAL_MS = "consumer.heartbeat.interval.ms";
+ public static final String PROP_CONSUMER_PARTITION_ASSIGNMENT_STRATEGY = "consumer.partition.assignment.strategy";
+ public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "kerberos.principal";
+ public static final String PROP_AUDIT_SERVICE_KEYTAB = "kerberos.keytab";
+ public static final String PROP_KAFKA_PROP_PREFIX = "xasecure.audit.destination.kafka";
+ public static final String PROP_KAFKA_STARTUP_MAX_RETRIES = "kafka.startup.max.retries";
+ public static final String PROP_KAFKA_STARTUP_RETRY_DELAY_MS = "kafka.startup.retry.delay.ms";
+
+ // ranger_audits topic configuration
+ public static final String PROP_TOPIC_PARTITIONS = "topic.partitions";
+ public static final String PROP_REPLICATION_FACTOR = "replication.factor";
+ public static final short DEFAULT_REPLICATION_FACTOR = 3;
+
+ // Hadoop security configuration for UGI
+ public static final String PROP_HADOOP_AUTHENTICATION_TYPE = "hadoop.security.authentication";
+ public static final String PROP_HADOOP_AUTH_TYPE_KERBEROS = "kerberos";
+ public static final String PROP_HADOOP_KERBEROS_NAME_RULES = "hadoop.security.auth_to_local";
+
+ // Kafka Topic and defaults
+ public static final String DEFAULT_TOPIC = "ranger_audits";
+ public static final String DEFAULT_SASL_MECHANISM = "PLAIN";
+ public static final String DEFAULT_SECURITY_PROTOCOL = "PLAINTEXT";
+ public static final String DEFAULT_SERVICE_NAME = "kafka";
+ public static final String DEFAULT_RANGER_AUDIT_HDFS_CONSUMER_GROUP = "ranger_audit_hdfs_consumer_group";
+ public static final String DEFAULT_RANGER_AUDIT_SOLR_CONSUMER_GROUP = "ranger_audit_solr_consumer_group";
+ public static final String PROP_SECURITY_PROTOCOL_VALUE = "SASL";
+
+ // Offset commit strategies
+ public static final String PROP_OFFSET_COMMIT_STRATEGY_MANUAL = "manual";
+ public static final String PROP_OFFSET_COMMIT_STRATEGY_BATCH = "batch";
+ public static final String DEFAULT_OFFSET_COMMIT_STRATEGY = PROP_OFFSET_COMMIT_STRATEGY_BATCH;
+ public static final long DEFAULT_OFFSET_COMMIT_INTERVAL_MS = 30000; // 30 seconds
+ public static final int DEFAULT_MAX_POLL_RECORDS = 500; // Kafka default batch size
+
+ // Kafka consumer rebalancing timeouts (for subscribe mode)
+ public static final int DEFAULT_SESSION_TIMEOUT_MS = 60000; // 60 seconds - failure detection
+ public static final int DEFAULT_MAX_POLL_INTERVAL_MS = 300000; // 5 minutes - max processing time
+ public static final int DEFAULT_HEARTBEAT_INTERVAL_MS = 10000; // 10 seconds - heartbeat frequency
+
+ // Kafka consumer partition assignment strategy
+ public static final String DEFAULT_PARTITION_ASSIGNMENT_STRATEGY = "org.apache.kafka.clients.consumer.RangeAssignor";
+
+ // Destination
+ public static final String DESTINATION_HDFS = "HDFS";
+ public static final String DESTINATION_SOLR = "SOLR";
+
+ // Consumer Registry Configuration
+ public static final String PROP_CONSUMER_CLASSES = "consumer.classes";
+ public static final String DEFAULT_CONSUMER_CLASSES = "org.apache.ranger.audit.consumer.kafka.AuditSolrConsumer, org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer";
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java
new file mode 100644
index 000000000..1826e5759
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java
@@ -0,0 +1,780 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.server;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.connector.Connector;
+import org.apache.catalina.core.StandardContext;
+import org.apache.catalina.startup.Tomcat;
+import org.apache.catalina.valves.AccessLogValve;
+import org.apache.catalina.webresources.StandardRoot;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class EmbeddedServer {
+ private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class);
+
+ private static final String ACCESS_LOG_PREFIX = "accesslog.prefix";
+ private static final String ACCESS_LOG_DATE_FORMAT = "accesslog.dateformat";
+ private static final String ACCESS_LOG_PATTERN = "accesslog.pattern";
+ private static final String ACCESS_LOG_ROTATE_MAX_DAYS = "accesslog.rotate.max.days";
+ private static final String DEFAULT_LOG_DIR = "/tmp";
+ private static final int DEFAULT_HTTP_PORT = 6081;
+ private static final int DEFAULT_HTTPS_PORT = -1;
+ private static final int DEFAULT_SHUTDOWN_PORT = 6185;
+ private static final String DEFAULT_SHUTDOWN_COMMAND = "SHUTDOWN";
+ private static final String DEFAULT_WEBAPPS_ROOT_FOLDER = "webapps";
+ private static final String DEFAULT_ENABLED_PROTOCOLS = "TLSv1.2";
+ public static final String DEFAULT_NAME_RULE = "DEFAULT";
+ private static final String RANGER_KEYSTORE_FILE_TYPE_DEFAULT = "jks";
+ private static final String RANGER_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks";
+ private static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2";
+ private static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm();
+ private static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm();
+ public static final String KEYSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType();
+ public static final String TRUSTSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType();
+
+ private final Configuration configuration;
+ private final String appName;
+ private final String configPrefix;
+ private volatile Tomcat server;
+ private volatile Context webappContext;
+
+ public EmbeddedServer(Configuration configuration, String appName, String configPrefix) {
+ LOG.info("==> EmbeddedServer(appName={}, configPrefix={})", appName, configPrefix);
+
+ this.configuration = new Configuration(configuration);
+ this.appName = appName;
+ this.configPrefix = configPrefix;
+
+ LOG.info("<== EmbeddedServer(appName={}, configPrefix={}", appName, configPrefix);
+ }
+
+ public static void main(String[] args) {
+ Configuration config = AuditConfig.getInstance();
+ String appName = AuditServerConstants.AUDIT_SERVER_APPNAME;
+ String configPrefix = AuditServerConstants.AUDIT_SERVER_PROP_PREFIX;
+
+ new EmbeddedServer(config, appName, configPrefix).start();
+ }
+
+ @SuppressWarnings("deprecation")
+ public void start() {
+ LOG.info("==> EmbeddedServer.start(appName={})", appName);
+
+ SSLContext sslContext = getSSLContext();
+
+ if (sslContext != null) {
+ SSLContext.setDefault(sslContext);
+ }
+
+ this.server = new Tomcat();
+
+ // Add shutdown hook for graceful cleanup
+ addShutdownHook();
+
+ String logDir = getConfig("log.dir", DEFAULT_LOG_DIR);
+ String hostName = getConfig("host");
+ int serverPort = getIntConfig("http.port", DEFAULT_HTTP_PORT);
+ int sslPort = getIntConfig("https.port", DEFAULT_HTTPS_PORT);
+ int shutdownPort = getIntConfig("shutdown.port", DEFAULT_SHUTDOWN_PORT);
+ String shutdownCommand = getConfig("shutdown.command", DEFAULT_SHUTDOWN_COMMAND);
+ boolean isHttpsEnabled = getBooleanConfig("https.attrib.ssl.enabled", false);
+ boolean ajpEnabled = getBooleanConfig("ajp.enabled", false);
+
+ server.setHostname(hostName);
+ server.setPort(serverPort);
+ server.getServer().setPort(shutdownPort);
+ server.getServer().setShutdown(shutdownCommand);
+
+ if (ajpEnabled) {
+ Connector ajpConnector = new Connector("org.apache.coyote.ajp.AjpNioProtocol");
+
+ ajpConnector.setPort(serverPort);
+ ajpConnector.setProperty("protocol", "AJP/1.3");
+
+ server.getService().addConnector(ajpConnector);
+
+ // Making this as a default connector
+ server.setConnector(ajpConnector);
+
+ LOG.info("Created AJP Connector");
+ } else if (isHttpsEnabled && sslPort > 0) {
+ String clientAuth = getConfig("https.attrib.clientAuth", "false");
+ String providerPath = getConfig("credential.provider.path");
+ String keyAlias = getConfig("https.attrib.keystore.credential.alias", "keyStoreCredentialAlias");
+ String keystorePass = null;
+ String enabledProtocols = getConfig("https.attrib.ssl.enabled.protocols", DEFAULT_ENABLED_PROTOCOLS);
+ String ciphers = getConfig("tomcat.ciphers");
+
+ if (StringUtils.equalsIgnoreCase(clientAuth, "false")) {
+ clientAuth = getConfig("https.attrib.client.auth", "want");
+ }
+
+ if (providerPath != null && keyAlias != null) {
+ keystorePass = getDecryptedString(providerPath.trim(), keyAlias.trim(), getConfig("keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT));
+
+ if (StringUtils.isBlank(keystorePass) || StringUtils.equalsIgnoreCase(keystorePass.trim(), "none")) {
+ keystorePass = getConfig("https.attrib.keystore.pass");
+ }
+ }
+
+ Connector ssl = new Connector();
+ String sslKeystoreKeyAlias = getConfig("https.attrib.keystore.keyalias", "");
+ ssl.setPort(sslPort);
+ ssl.setSecure(true);
+ ssl.setScheme("https");
+ ssl.setAttribute("SSLEnabled", "true");
+ ssl.setAttribute("sslProtocol", getConfig("https.attrib.ssl.protocol", "TLSv1.2"));
+ ssl.setAttribute("clientAuth", clientAuth);
+ if (StringUtils.isNotBlank(sslKeystoreKeyAlias)) {
+ ssl.setAttribute("keyAlias", sslKeystoreKeyAlias);
+ }
+ ssl.setAttribute("keystorePass", keystorePass);
+ ssl.setAttribute("keystoreFile", getKeystoreFile());
+ ssl.setAttribute("sslEnabledProtocols", enabledProtocols);
+ ssl.setAttribute("keystoreType", getConfig("audit.keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT));
+ ssl.setAttribute("truststoreType", getConfig("audit.truststore.file.type", TRUSTSTORE_FILE_TYPE_DEFAULT));
+
+ if (StringUtils.isNotBlank(ciphers)) {
+ ssl.setAttribute("ciphers", ciphers);
+ }
+
+ server.getService().addConnector(ssl);
+
+ //
+ // Making this as a default connector
+ //
+ server.setConnector(ssl);
+ }
+
+ updateHttpConnectorAttribConfig(server);
+
+ File logDirectory = new File(logDir);
+
+ if (!logDirectory.exists()) {
+ boolean created = logDirectory.mkdirs();
+ if (!created) {
+ LOG.error("Failed to create log directory: {}", logDir);
+ throw new RuntimeException("Failed to create log directory: " + logDir);
+ }
+ }
+
+ String logPattern = getConfig(ACCESS_LOG_PATTERN, "%h %l %u %t \"%r\" %s %b");
+
+ AccessLogValve valve = new AccessLogValve();
+
+ valve.setRotatable(true);
+ valve.setAsyncSupported(true);
+ valve.setBuffered(false);
+ valve.setEnabled(true);
+ valve.setPrefix(getConfig(ACCESS_LOG_PREFIX, "access_log-" + hostName + "-"));
+ valve.setFileDateFormat(getConfig(ACCESS_LOG_DATE_FORMAT, "yyyy-MM-dd.HH"));
+ valve.setDirectory(logDirectory.getAbsolutePath());
+ valve.setSuffix(".log");
+ valve.setPattern(logPattern);
+ valve.setMaxDays(getIntConfig(ACCESS_LOG_ROTATE_MAX_DAYS, 15));
+
+ server.getHost().getPipeline().addValve(valve);
+
+ try {
+ String webappDir = getConfig("webapp.dir");
+
+ if (StringUtils.isBlank(webappDir)) {
+ LOG.error("Tomcat Server failed to start: {}.webapp.dir is not set", configPrefix);
+
+ System.exit(1);
+ }
+
+ String webContextName = getConfig("contextName", "");
+
+ if (StringUtils.isBlank(webContextName)) {
+ webContextName = "";
+ } else if (!webContextName.startsWith("/")) {
+ LOG.info("Context Name [{}] is being loaded as [ /{}]", webContextName, webContextName);
+
+ webContextName = "/" + webContextName;
+ }
+
+ File wad = new File(webappDir);
+
+ if (wad.isDirectory()) {
+ LOG.info("Webapp dir={}, webAppName={}", webappDir, webContextName);
+ } else if (wad.isFile()) {
+ File webAppDir = new File(DEFAULT_WEBAPPS_ROOT_FOLDER);
+
+ if (!webAppDir.exists()) {
+ boolean created = webAppDir.mkdirs();
+ if (!created) {
+ LOG.error("Failed to create webapp directory: {}", DEFAULT_WEBAPPS_ROOT_FOLDER);
+ throw new RuntimeException("Failed to create webapp directory: " + DEFAULT_WEBAPPS_ROOT_FOLDER);
+ }
+ }
+
+ LOG.info("Webapp file={}, webAppName={}", webappDir, webContextName);
+ }
+
+ LOG.info("Adding webapp [{}] = path [{}] .....", webContextName, webappDir);
+
+ this.webappContext = server.addWebapp(webContextName, new File(webappDir).getAbsolutePath());
+ if (webappContext instanceof StandardContext) {
+ boolean allowLinking = getBooleanConfig("allow.linking", true);
+ StandardContext standardContext = (StandardContext) webappContext;
+ String workDirPath = getConfig("tomcat.work.dir", "");
+ if (!workDirPath.isEmpty() && new File(workDirPath).exists()) {
+ standardContext.setWorkDir(workDirPath);
+ } else {
+ LOG.debug("Skipping to set tomcat server work directory {} as it is blank or directory does not exist.", workDirPath);
+ }
+ WebResourceRoot resRoot = new StandardRoot(standardContext);
+ webappContext.setResources(resRoot);
+ webappContext.getResources().setAllowLinking(allowLinking);
+ LOG.debug("Tomcat Configuration - allowLinking=[{}]", allowLinking);
+ } else {
+ LOG.error("Tomcat Context [{}] is either NULL OR it's NOT instanceof StandardContext", webappContext);
+ }
+
+ webappContext.init();
+
+ LOG.info("Finished init of webapp [{}] = path [{}].", webContextName, webappDir);
+ } catch (LifecycleException lce) {
+ LOG.error("Tomcat Server failed to start webapp", lce);
+
+ lce.printStackTrace();
+ }
+
+ String authenticationType = getConfig(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE);
+
+ if (StringUtils.equalsIgnoreCase(authenticationType, AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS)) {
+ String keyTab = getConfig(AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB);
+ String principal = null;
+ String nameRules = getConfig(AuditServerConstants.PROP_HADOOP_KERBEROS_NAME_RULES, DEFAULT_NAME_RULE);
+
+ try {
+ principal = SecureClientLogin.getPrincipal(getConfig(AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL), hostName);
+ } catch (IOException excp) {
+ LOG.warn("Failed to get principal" + excp);
+ }
+
+ if (SecureClientLogin.isKerberosCredentialExists(principal, keyTab)) {
+ try {
+ AuditServerLogFormatter.builder("Kerberos Login Attempt")
+ .add("Principal", principal)
+ .add("Keytab", keyTab)
+ .add("Name rules", nameRules)
+ .logInfo(LOG);
+
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE, AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS);
+ hadoopConf.set(AuditServerConstants.PROP_HADOOP_KERBEROS_NAME_RULES, nameRules);
+ UserGroupInformation.setConfiguration(hadoopConf);
+ UserGroupInformation.loginUserFromKeytab(principal, keyTab);
+ UserGroupInformation currentUGI = UserGroupInformation.getLoginUser();
+
+ AuditServerLogFormatter.builder("Kerberos Login Successful")
+ .add("UGI Username", currentUGI.getUserName())
+ .add("UGI Real User", currentUGI.getUserName())
+ .add("Authentication Method", currentUGI.getAuthenticationMethod().toString())
+ .add("Has Kerberos Credentials", currentUGI.hasKerberosCredentials())
+ .logInfo(LOG);
+
+ MiscUtil.setUGILoginUser(currentUGI, null);
+
+ LOG.info("Starting Server using kerberos credential");
+ startServer(server);
+ } catch (Exception excp) {
+ LOG.error("Tomcat Server failed to start", excp);
+ }
+ } else {
+ LOG.warn("Kerberos principal={} not found in keytab={}. Starting server in non-kerberos mode", principal, keyTab);
+
+ startServer(server);
+ }
+ } else {
+ LOG.info("Starting server in non-kerberos mode");
+
+ startServer(server);
+ }
+
+ LOG.info("<== EmbeddedServer.start(appName={})", appName);
+ }
+
+ public String getDecryptedString(String credentialProviderPath, String alias, String storeType) {
+ String ret = null;
+
+ if (StringUtils.isNotBlank(credentialProviderPath) && StringUtils.isNotBlank(alias)) {
+ try {
+ Configuration conf = new Configuration();
+ String prefixJceks = JavaKeyStoreProvider.SCHEME_NAME + "://file";
+ String prefixLocalJceks = "localjceks://file";
+ String prefixBcfks = BouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file";
+ String prefixLocalBcfks = LocalBouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file";
+
+ String lowerPath = credentialProviderPath.toLowerCase();
+ if (lowerPath.startsWith(prefixJceks.toLowerCase()) || lowerPath.startsWith(prefixLocalJceks.toLowerCase()) || lowerPath.startsWith(prefixBcfks.toLowerCase()) || lowerPath.startsWith(prefixLocalBcfks.toLowerCase())) {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath);
+ } else {
+ if (credentialProviderPath.startsWith("/")) {
+ if ("bcfks".equalsIgnoreCase(storeType)) {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath);
+ } else {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + credentialProviderPath);
+ }
+ } else {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + "/" + credentialProviderPath);
+ }
+ }
+
+ List<CredentialProvider> providers = CredentialProviderFactory.getProviders(conf);
+
+ LOG.debug("CredentialProviderPath={} alias={} storeType={}", credentialProviderPath, alias, storeType);
+ LOG.debug("List of CredentialProvider = {}", providers.toString());
+
+ for (CredentialProvider provider : providers) {
+ List<String> aliasesList = provider.getAliases();
+
+ if (CollectionUtils.isNotEmpty(aliasesList) && aliasesList.contains(alias)) {
+ CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias);
+ char[] pass = credEntry.getCredential();
+
+ if (pass != null && pass.length > 0) {
+ ret = String.valueOf(pass);
+
+ break;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("CredentialReader failed while decrypting provided string. Reason: {}", ex.toString());
+
+ ret = null;
+ }
+ }
+
+ LOG.debug("getDecryptedString() : ret = {}", ret);
+
+ return ret;
+ }
+
+ protected long getLongConfig(String key, long defaultValue) {
+ long ret = defaultValue;
+ String retStr = getConfig(key);
+
+ try {
+ if (retStr != null) {
+ ret = Long.parseLong(retStr);
+ }
+ } catch (Exception err) {
+ LOG.warn(retStr + " can't be parsed to long. Reason: {}", err.toString());
+ }
+
+ return ret;
+ }
+
+ private void startServer(final Tomcat server) {
+ LOG.info("==> EmbeddedServer.startServer(appName={})", appName);
+
+ try {
+ server.start();
+
+ server.getServer().await();
+
+ shutdownServer();
+ } catch (LifecycleException e) {
+ LOG.error("Tomcat Server failed to start", e);
+
+ e.printStackTrace();
+ } catch (Exception e) {
+ LOG.error("Tomcat Server failed to start", e);
+
+ e.printStackTrace();
+ }
+
+ LOG.info("<== EmbeddedServer.startServer(appName={})", appName);
+ }
+
+ private void addShutdownHook() {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ LOG.info("==> EmbeddedServer shutdown hook triggered");
+ try {
+ gracefulShutdown();
+ } catch (Exception e) {
+ LOG.error("Error during graceful shutdown", e);
+ }
+ LOG.info("<== EmbeddedServer shutdown hook completed");
+ }, "EmbeddedServer-ShutdownHook"));
+ }
+
+ private void gracefulShutdown() {
+ LOG.info("==> EmbeddedServer.gracefulShutdown()");
+
+ // Instead of trying to manually shutdown Spring context, let's rely on Tomcat's
+ // normal webapp lifecycle which will trigger ContextLoaderListener.contextDestroyed()
+ // and call @PreDestroy methods automatically
+
+ if (server != null) {
+ try {
+ LOG.info("Initiating Tomcat server stop to trigger webapp lifecycle shutdown");
+ server.stop();
+ LOG.info("Tomcat server stop completed - Spring @PreDestroy methods should have been called");
+ } catch (Exception e) {
+ LOG.error("Error stopping Tomcat server during graceful shutdown", e);
+
+ // Fallback: try to stop the webapp context directly
+ if (webappContext != null) {
+ try {
+ LOG.info("Fallback: stopping webapp context directly");
+ webappContext.stop();
+ LOG.info("Webapp context stopped");
+ } catch (Exception contextE) {
+ LOG.error("Error stopping webapp context", contextE);
+ }
+ }
+ }
+ }
+
+ // Give some time for Spring components to shutdown through normal lifecycle
+ try {
+ Thread.sleep(3000); // Increased to 3 seconds to allow for proper cleanup
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for Spring shutdown", e);
+ Thread.currentThread().interrupt();
+ }
+
+ LOG.info("<== EmbeddedServer.gracefulShutdown()");
+ }
+
+ private void shutdownServer() {
+ LOG.info("==> EmbeddedServer.shutdownServer(appName={})", appName);
+
+ int timeWaitForShutdownInSeconds = getIntConfig("waitTimeForForceShutdownInSeconds", 0);
+
+ if (timeWaitForShutdownInSeconds > 0) {
+ long endTime = System.currentTimeMillis() + (timeWaitForShutdownInSeconds * 1000L);
+
+ LOG.info("Will wait for all threads to shutdown gracefully. Final shutdown Time: {}", new Date(endTime));
+
+ while (System.currentTimeMillis() < endTime) {
+ int activeCount = Thread.activeCount();
+
+ if (activeCount == 0) {
+ LOG.info("Number of active threads = {}", activeCount);
+
+ break;
+ } else {
+ LOG.info("Number of active threads = {}. Waiting for all threads to shutdown ...", activeCount);
+
+ try {
+ Thread.sleep(5000L);
+ } catch (InterruptedException e) {
+ LOG.warn("shutdownServer process is interrupted with exception", e);
+
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.info("Shutting down the Server.");
+
+ LOG.info("<== EmbeddedServer.shutdownServer(appName={})", appName);
+
+ System.exit(0);
+ }
+
+ @SuppressWarnings("deprecation")
+ private void updateHttpConnectorAttribConfig(Tomcat server) {
+ server.getConnector().setAllowTrace(getBooleanConfig("http.connector.attrib.allowTrace", false));
+ server.getConnector().setAsyncTimeout(getLongConfig("http.connector.attrib.asyncTimeout", 10000));
+ server.getConnector().setEnableLookups(getBooleanConfig("http.connector.attrib.enableLookups", false));
+ server.getConnector().setMaxParameterCount(getIntConfig("http.connector.attrib.maxParameterCount", 10000));
+ server.getConnector().setMaxPostSize(getIntConfig("http.connector.attrib.maxPostSize", 2097152));
+ server.getConnector().setMaxSavePostSize(getIntConfig("http.connector.attrib.maxSavePostSize", 4096));
+ server.getConnector().setParseBodyMethods(getConfig("http.connector.attrib.methods", "POST"));
+ server.getConnector().setURIEncoding(getConfig("http.connector.attrib.URIEncoding", "UTF-8"));
+ server.getConnector().setProperty("acceptCount", getConfig("http.connector.attrib.acceptCount", "1024"));
+ server.getConnector().setXpoweredBy(false);
+ server.getConnector().setAttribute("server", getConfig("servername", "Audit Server"));
+
+ int maxThreads = getIntConfig("http.connector.attrib.maxThreads", 2000);
+ int maxKeepAliveRequests = getIntConfig("http.connector.attrib.maxKeepAliveRequests", 1000);
+ int minSpareThreads = getIntConfig("http.connector.attrib.minSpareThreads", (int) (maxThreads * 0.8)); // (default 80% of maxThreads)
+ boolean prestartminSpareThreads = getBooleanConfig("http.connector.attrib.prestartminSpareThreads", true);
+
+ server.getConnector().setAttribute("maxThreads", maxThreads);
+ server.getConnector().setAttribute("maxKeepAliveRequests", maxKeepAliveRequests);
+ server.getConnector().setAttribute("minSpareThreads", minSpareThreads);
+ server.getConnector().setAttribute("prestartminSpareThreads", prestartminSpareThreads);
+ server.getConnector().setProperty("sendReasonPhrase", getConfig(configPrefix + "http.connector.property.sendReasonPhrase", "true"));
+
+ for (Map.Entry<String, String> entry : configuration) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (key != null && key.startsWith(configPrefix + "http.connector.property.")) {
+ String property = key.replace(configPrefix + "http.connector.property.", "");
+
+ server.getConnector().setProperty(property, value);
+
+ LOG.info("{}:{}", property, server.getConnector().getProperty(property));
+ }
+ }
+ }
+
+ private SSLContext getSSLContext() {
+ SSLContext ret = null;
+ KeyManager[] kmList = getKeyManagers();
+ TrustManager[] tmList = getTrustManagers();
+
+ if (tmList != null) {
+ try {
+ ret = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
+
+ ret.init(kmList, tmList, new SecureRandom());
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is not available in the environment", e);
+ } catch (KeyManagementException e) {
+ LOG.error("Unable to initials the SSLContext", e);
+ }
+ }
+
+ return ret;
+ }
+
+ private KeyManager[] getKeyManagers() {
+ KeyManager[] ret = null;
+ String keyStoreFile = getConfig("keystore.file");
+ String keyStoreAlias = getConfig("keystore.alias");
+ String credentialProviderPath = getConfig("credential.provider.path");
+ String keyStoreFilepwd = getCredential(credentialProviderPath, keyStoreAlias);
+
+ if (StringUtils.isNotBlank(keyStoreFile) && StringUtils.isNotBlank(keyStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(keyStoreFile);
+
+ if (in != null) {
+ KeyStore keyStore = KeyStore.getInstance(RANGER_KEYSTORE_FILE_TYPE_DEFAULT);
+
+ keyStore.load(in, keyStoreFilepwd.toCharArray());
+
+ KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
+
+ keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
+
+ ret = keyManagerFactory.getKeyManagers();
+ } else {
+ LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL Keystore Files", e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL Keystore Files", e);
+ } catch (UnrecoverableKeyException e) {
+ LOG.error("Unable to recover the key from keystore", e);
+ } finally {
+ close(in, keyStoreFile);
+ }
+ }
+
+ return ret;
+ }
+
+ private TrustManager[] getTrustManagers() {
+ TrustManager[] ret = null;
+ String truststoreFile = getConfig("truststore.file");
+ String truststoreAlias = getConfig("truststore.alias");
+ String credentialProviderPath = getConfig("credential.provider.path");
+ String trustStoreFilepwd = getCredential(credentialProviderPath, truststoreAlias);
+
+ if (StringUtils.isNotBlank(truststoreFile) && StringUtils.isNotBlank(trustStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(truststoreFile);
+
+ if (in != null) {
+ KeyStore trustStore = KeyStore.getInstance(RANGER_TRUSTSTORE_FILE_TYPE_DEFAULT);
+
+ trustStore.load(in, trustStoreFilepwd.toCharArray());
+
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
+
+ trustManagerFactory.init(trustStore);
+
+ ret = trustManagerFactory.getTrustManagers();
+ } else {
+ LOG.error("Unable to obtain truststore from file [{}]", truststoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL TrustStore File:{}", truststoreFile, e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL TrustStore Files:{}", truststoreFile, e);
+ } finally {
+ close(in, truststoreFile);
+ }
+ }
+
+ return ret;
+ }
+
+ private String getKeystoreFile() {
+ String ret = getConfig("https.attrib.keystore.file");
+
+ if (StringUtils.isBlank(ret)) {
+ // new property not configured, lets use the old property
+ ret = getConfig("https.attrib.keystore.file");
+ }
+
+ return ret;
+ }
+
+ private String getCredential(String url, String alias) {
+ return RangerCredentialProvider.getInstance().getCredentialString(url, alias);
+ }
+
+ private String getConfig(String key) {
+ String propertyWithPrefix = configPrefix + key;
+ String value = configuration.get(propertyWithPrefix);
+ if (value == null) { // config-with-prefix not found; look at System properties
+ value = System.getProperty(propertyWithPrefix);
+ if (value == null) { // config-with-prefix not found in System properties; look for config-without-prefix
+ value = configuration.get(key);
+ if (value == null) { // config-without-prefix not found; look at System properties
+ value = System.getProperty(key);
+ }
+ }
+ }
+
+ return value;
+ }
+
+ private String getConfig(String key, String defaultValue) {
+ String ret = getConfig(key);
+
+ if (ret == null) {
+ ret = defaultValue;
+ }
+
+ return ret;
+ }
+
+ private int getIntConfig(String key, int defaultValue) {
+ int ret = defaultValue;
+ String strValue = getConfig(key);
+
+ try {
+ if (strValue != null) {
+ ret = Integer.parseInt(strValue);
+ }
+ } catch (Exception err) {
+ LOG.warn(strValue + " can't be parsed to int. Reason: {}", err.toString());
+ }
+
+ return ret;
+ }
+
+ private boolean getBooleanConfig(String key, boolean defaultValue) {
+ boolean ret = defaultValue;
+ String strValue = getConfig(key);
+
+ if (StringUtils.isNotBlank(strValue)) {
+ ret = Boolean.valueOf(strValue);
+ }
+
+ return ret;
+ }
+
+ private InputStream getFileInputStream(String fileName) throws IOException {
+ InputStream ret = null;
+
+ if (StringUtils.isNotEmpty(fileName)) {
+ File f = new File(fileName);
+
+ if (f.exists()) {
+ ret = new FileInputStream(f);
+ } else {
+ ret = ClassLoader.getSystemResourceAsStream(fileName);
+ }
+ }
+
+ return ret;
+ }
+
+ private void close(InputStream str, String filename) {
+ if (str != null) {
+ try {
+ str.close();
+ } catch (IOException excp) {
+ LOG.error("Error while closing file: [{}]", filename, excp);
+ }
+ }
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java
new file mode 100644
index 000000000..de8a5a724
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.CreatePartitionsResult;
+import org.apache.kafka.clients.admin.DescribeTopicsResult;
+import org.apache.kafka.clients.admin.NewPartitions;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.ranger.audit.provider.AuditProviderFactory;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+public class AuditMessageQueueUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditMessageQueueUtils.class);
+
+ AuditServerUtils auditServerUtils = new AuditServerUtils();
+ boolean isTopicReady;
+ boolean isSolrConsumerEnabled;
+ boolean isHDFSConsumerEnabled;
+
+ public AuditMessageQueueUtils(Properties props) {
+ isHDFSConsumerEnabled = MiscUtil.getBooleanProperty(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_HDFS_DEST_PREFIX, false);
+ isSolrConsumerEnabled = MiscUtil.getBooleanProperty(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_SOLR_DEST_PREFIX, false);
+ }
+
+ public String createAuditsTopicIfNotExists(Properties props, String propPrefix) {
+ LOG.info("==> AuditMessageQueueUtils:createAuditsTopicIfNotExists()");
+
+ String ret = null;
+ String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC);
+ String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS);
+ String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL);
+ String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM);
+ int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000);
+ int partitions = getPartitions(props, propPrefix);
+ short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR);
+ int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000);
+
+ // Retry configuration for Kafka connection during startup
+ int maxRetries = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_STARTUP_MAX_RETRIES, 10);
+ int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_STARTUP_RETRY_DELAY_MS, 3000);
+ int currentAttempt = 0;
+
+ Map<String, Object> kafkaProp = new HashMap<>();
+ kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ kafkaProp.put(AuditServerConstants.PROP_SASL_MECHANISM, saslMechanism);
+ kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol);
+
+ if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) {
+ kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix));
+ }
+
+ kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS);
+ kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS);
+
+ while (currentAttempt <= maxRetries && ret == null) {
+ currentAttempt++;
+
+ try (AdminClient admin = AdminClient.create(kafkaProp)) {
+ if (currentAttempt > 1) {
+ LOG.info("Attempting to connect to Kafka (attempt {}/{})", currentAttempt, maxRetries + 1);
+ }
+
+ Set<String> names = admin.listTopics().names().get();
+ if (!names.contains(topicName)) {
+ NewTopic topic = new NewTopic(topicName, partitions, replicationFactor);
+ admin.createTopics(Collections.singletonList(topic)).all().get();
+ ret = topic.name();
+ LOG.info("Creating topic '{}' with {} partitions and replication factor {}", topicName, partitions, replicationFactor);
+ // Wait for metadata to propagate across the cluster
+ boolean isTopicReady = auditServerUtils.waitUntilTopicReady(admin, topicName, Duration.ofSeconds(60));
+ if (isTopicReady) {
+ try {
+ DescribeTopicsResult result = admin.describeTopics(Collections.singletonList(topicName));
+ TopicDescription topicDescription = result.values().get(topicName).get();
+ ret = topicDescription.name();
+
+ int partitionCount = topicDescription.partitions().size();
+ setTopicReady(isTopicReady);
+
+ LOG.info("Topic: {} successfully created with {} partitions", ret, partitionCount);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to fetch metadata for topic:" + topicName, e);
+ }
+ }
+ } else {
+ /***
+ * Topic already existing. Check and update number of partitions for audit server. This is for upgrade
+ * from existing audit mechanism to audit server
+ */
+ ret = updateExistingTopicPartitions(admin, topicName, partitions, replicationFactor);
+ }
+ } catch (Exception ex) {
+ if (currentAttempt <= maxRetries) {
+ LOG.warn("AuditMessageQueueUtils:createAuditsTopicIfNotExists(): Failed to connect to Kafka on attempt {}/{}. Retrying in {} ms. Error: {}",
+ currentAttempt, maxRetries + 1, retryDelayMs, ex.getMessage());
+ try {
+ Thread.sleep(retryDelayMs);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.error("Interrupted while waiting to retry Kafka connection", ie);
+ break;
+ }
+ } else {
+ LOG.error("AuditMessageQueueUtils:createAuditsTopicIfNotExists(): Error creating topic after {} attempts", currentAttempt, ex);
+ }
+ }
+ }
+
+ LOG.info("<== AuditMessageQueueUtils:createAuditsTopicIfNotExists() ret: {}", ret);
+
+ return ret;
+ }
+
+ public boolean isSolrConsumerEnabled() {
+ return isSolrConsumerEnabled;
+ }
+
+ public boolean isHDFSConsumerEnabled() {
+ return isHDFSConsumerEnabled;
+ }
+
+ public boolean isTopicReady() {
+ return isTopicReady;
+ }
+
+ public void setTopicReady(boolean topicReady) {
+ isTopicReady = topicReady;
+ }
+
+ public String getJAASConfig(Properties props, String propPrefix) {
+ // Use ranger service principal and keytab for Kafka authentication
+ // This ensures consistent identity across all Ranger services and destination writes
+ String hostName = props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + "host");
+ String principal = props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL);
+ String keytab = props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB);
+
+ AuditServerLogFormatter.builder("Kerberos Configuration")
+ .add("Principal (raw)", principal)
+ .add("Hostname", hostName)
+ .add("Keytab path", keytab)
+ .logInfo(LOG);
+
+ // Validate keytab file exists and is readable
+ if (keytab != null) {
+ java.io.File keytabFile = new java.io.File(keytab);
+ if (!keytabFile.exists()) {
+ LOG.error("ERROR: Keytab file does not exist: {}", keytab);
+ throw new IllegalStateException("Keytab file not found: " + keytab);
+ }
+ if (!keytabFile.canRead()) {
+ LOG.error("ERROR: Keytab file is not readable: {}", keytab);
+ throw new IllegalStateException("Keytab file not readable: " + keytab);
+ }
+
+ AuditServerLogFormatter.builder("Keytab File Validation")
+ .add("Exists", keytabFile.exists())
+ .add("Readable", keytabFile.canRead())
+ .add("Size (bytes)", keytabFile.length())
+ .logInfo(LOG);
+ }
+
+ try {
+ principal = SecureClientLogin.getPrincipal(props.getProperty(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL), hostName);
+ LOG.info("Principal (resolved): {}", principal);
+ } catch (Exception e) {
+ principal = null;
+ LOG.error("ERROR: Failed to resolve principal from _HOST pattern!", e);
+ }
+
+ if (keytab == null || principal == null) {
+ AuditServerLogFormatter.builder("Please configure the following properties in ranger-audit-server-site.xml:")
+ .add(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL, "ranger/_HOST@YOUR-REALM")
+ .add(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB, "/path/to/ranger.keytab")
+ .logError(LOG);
+ throw new IllegalStateException("Ranger service principal and keytab must be configured for Kafka authentication. ");
+ }
+
+ String jaasConfig = new StringBuilder()
+ .append(AuditServerConstants.JAAS_KRB5_MODULE).append(" ")
+ .append(AuditServerConstants.JAAS_USE_KEYTAB).append(" ")
+ .append(AuditServerConstants.JAAS_KEYTAB).append(keytab).append("\"").append(" ")
+ .append(AuditServerConstants.JAAS_STOKE_KEY).append(" ")
+ .append(AuditServerConstants.JAAS_USER_TICKET_CACHE).append(" ")
+ .append(AuditServerConstants.JAAS_SERVICE_NAME).append(" ")
+ .append(AuditServerConstants.JAAS_PRINCIPAL).append(principal).append("\";")
+ .toString();
+
+ AuditServerLogFormatter.builder("JAAS Configuration Generated")
+ .add("Principal", principal)
+ .add("Keytab", keytab)
+ .add("Full JAAS Config", jaasConfig)
+ .logInfo(LOG);
+
+ return jaasConfig;
+ }
+
+ public String updateExistingTopicPartitions(AdminClient admin, String topicName, int partitions, short replicationFactor) {
+ LOG.info("==> AuditMessageQueueUtils:updateExistingTopicPartitions() topic: {}, desired partitions: {}", topicName, partitions);
+
+ String ret = null;
+ int maxRetries = 3;
+ int retryDelayMs = 1000; // Start with 1 second
+
+ try {
+ // Describe the existing topic to get current partition count
+ DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singletonList(topicName));
+ TopicDescription topicDescription = describeTopicsResult.values().get(topicName).get();
+ int currentPartitions = topicDescription.partitions().size();
+
+ ret = topicDescription.name();
+ LOG.info("Topic '{}' already exists with {} partitions", ret, currentPartitions);
+
+ // Check if we need to increase partitions
+ if (partitions > currentPartitions) {
+ LOG.info("Upgrading topic '{}' from {} to {} partitions for audit server mechanism", topicName, currentPartitions, partitions);
+
+ boolean updateSuccess = false;
+ Exception lastException = null;
+
+ // Retry logic while updating partitions
+ for (int attempt = 1; attempt <= maxRetries; attempt++) {
+ try {
+ LOG.info("Partition update attempt {}/{} for topic '{}'", attempt, maxRetries, topicName);
+
+ // Create partition increase request
+ Map<String, NewPartitions> newPartitionsMap = new HashMap<>();
+ newPartitionsMap.put(topicName, NewPartitions.increaseTo(partitions));
+
+ // Execute partition increase
+ CreatePartitionsResult createPartitionsResult = admin.createPartitions(newPartitionsMap);
+ createPartitionsResult.all().get(); // Wait for operation to complete
+
+ LOG.info("Successfully initiated partition increase for topic '{}' on attempt {}", topicName, attempt);
+
+ // Wait for metadata to propagate across the cluster
+ boolean isTopicReady = auditServerUtils.waitUntilTopicReady(admin, topicName, Duration.ofSeconds(60));
+
+ if (isTopicReady) {
+ // Verify the partition count after update
+ DescribeTopicsResult verifyResult = admin.describeTopics(Collections.singletonList(topicName));
+ TopicDescription updatedDescription = verifyResult.values().get(topicName).get();
+ int updatedPartitions = updatedDescription.partitions().size();
+
+ if (updatedPartitions >= partitions) {
+ setTopicReady(true);
+ LOG.info("Topic '{}' successfully upgraded from {} to {} partitions on attempt {}", topicName, currentPartitions, updatedPartitions, attempt);
+ updateSuccess = true;
+ break;
+ } else {
+ LOG.warn("Topic '{}' partition update completed but verification shows only {} partitions (expected {})", topicName, updatedPartitions, partitions);
+ throw new IllegalStateException("Partition count verification failed");
+ }
+ } else {
+ LOG.warn("Topic '{}' partition update completed but topic not ready within timeout on attempt {}", topicName, attempt);
+ throw new IllegalStateException("Topic not ready after partition update");
+ }
+ } catch (Exception e) {
+ lastException = e;
+ LOG.warn("Partition update attempt {}/{} failed for topic '{}': {}", attempt, maxRetries, topicName, e.getMessage());
+ if (attempt < maxRetries) {
+ int currentDelay = retryDelayMs * (1 << (attempt - 1));
+ LOG.info("Retrying partition update in {} ms...", currentDelay);
+ Thread.sleep(currentDelay);
+ }
+ }
+ }
+ if (!updateSuccess) {
+ String errorMsg = String.format(
+ "FATAL: Failed to update partitions for topic '%s' after %d attempts. " +
+ "Required: %d partitions, Current: %d partitions. " +
+ "Cannot proceed without sufficient partitions.", topicName, maxRetries, partitions, currentPartitions);
+ LOG.error(errorMsg, lastException);
+ throw new RuntimeException(errorMsg, lastException);
+ }
+ } else if (partitions < currentPartitions) {
+ LOG.warn("Topic '{}' has {} partitions, which is more than the configured {} partitions. " +
+ "Kafka does not support reducing partition count. Using existing partition count.", topicName, currentPartitions, partitions);
+ setTopicReady(true);
+ } else {
+ LOG.info("Topic '{}' already has the correct number of partitions: {}", topicName, currentPartitions);
+ setTopicReady(true);
+ }
+ } catch (Exception e) {
+ String errorMsg = String.format("FATAL: Error updating partitions for topic '%s'", topicName);
+ LOG.error(errorMsg, e);
+ throw new RuntimeException(errorMsg, e);
+ }
+
+ LOG.info("<== AuditMessageQueueUtils:updateExistingTopicPartitions() ret: {}", ret);
+
+ return ret;
+ }
+
+ /**
+ * Get the number of partitions for the Kafka topic
+ *
+ * Configurable via xasecure.audit.destination.kafka.topic.partitions property.
+ * Default: 30 partitions for balanced distribution with Kafka's default partitioner.
+ *
+ * With default partitioner (murmur2 hash), messages with same key (appId) always
+ * go to the same partition, providing ordering per appId while distributing load
+ * evenly across all partitions.
+ *
+ * @return Number of partitions for the topic
+ */
+ private int getPartitions(Properties prop, String propPrefix) {
+ int defaultPartitions = 30;
+ int partitions = MiscUtil.getIntProperty(prop,
+ propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS,
+ defaultPartitions);
+
+ LOG.info("Kafka topic partition count: {} (configured: {})",
+ partitions, prop.getProperty(propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, "default"));
+
+ return partitions;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java
new file mode 100644
index 000000000..1fca5554b
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.slf4j.Logger;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Utility for logging in the ranger-audit-server
+ */
+public class AuditServerLogFormatter {
+ private AuditServerLogFormatter() {}
+
+ /**
+ * Log a map of logDetails at INFO level with a title
+ * @param logger The logger to use
+ * @param title The title/header for this log section
+ * @param logDetails Map of key-value pairs to log
+ */
+ public static void logInfo(Logger logger, String title, Map<String, Object> logDetails) {
+ if (logDetails != null && !logDetails.isEmpty()) {
+ logger.info("{}:", title);
+ logDetails.forEach((key, value) -> logger.info(" {} = [{}]", key, value));
+ }
+ }
+
+ /**
+ * Log a map of logDetails at DEBUG level with a title
+ * @param logger The logger to use
+ * @param title The title/header for this log section
+ * @param logDetails Map of key-value pairs to log
+ */
+ public static void logDebug(Logger logger, String title, Map<String, Object> logDetails) {
+ if (logDetails != null && !logDetails.isEmpty()) {
+ logger.debug("{}:", title);
+ logDetails.forEach((key, value) -> logger.debug(" {} = [{}]", key, value));
+ }
+ }
+
+ /**
+ * Log a map of logDetails at DEBUG level with a title
+ * @param logger The logger to use
+ * @param title The title/header for this log section
+ * @param logDetails Map of key-value pairs to log
+ */
+ public static void logError(Logger logger, String title, Map<String, Object> logDetails) {
+ if (logDetails != null && !logDetails.isEmpty()) {
+ logger.error("{}:", title);
+ logDetails.forEach((key, value) -> logger.error(" {} = [{}]", key, value));
+ }
+ }
+
+ /**
+ * Create a builder for constructing LogDetails maps to log
+ * @param title The title for this log section
+ * @return A new LogBuilder
+ */
+ public static LogBuilder builder(String title) {
+ return new LogBuilder(title);
+ }
+
+ /**
+ * Builder class for constructing structured log messages
+ */
+ public static class LogBuilder {
+ private final String title;
+ private final Map<String, Object> logDetails = new LinkedHashMap<>();
+
+ private LogBuilder(String title) {
+ this.title = title;
+ }
+
+ /**
+ * Add a log key value pair
+ * @param key
+ * @param value
+ * @return This builder for chaining
+ */
+ public LogBuilder add(String key, Object value) {
+ logDetails.put(key, value);
+ return this;
+ }
+
+ /**
+ * Add a log only if the value is not null
+ * @param key
+ * @param value
+ * @return This builder for chaining
+ */
+ public LogBuilder addIfNotNull(String key, Object value) {
+ if (value != null) {
+ logDetails.put(key, value);
+ }
+ return this;
+ }
+
+ /**
+ * Log the accumulated LogDetails at INFO level
+ * @param logger The logger to use
+ */
+ public void logInfo(Logger logger) {
+ AuditServerLogFormatter.logInfo(logger, title, logDetails);
+ }
+
+ /**
+ * Log the accumulated LogDetails at DEBUG level
+ * @param logger The logger to use
+ */
+ public void logDebug(Logger logger) {
+ AuditServerLogFormatter.logDebug(logger, title, logDetails);
+ }
+
+ /**
+ * Log the accumulated LogDetails at DEBUG level
+ * @param logger The logger to use
+ */
+ public void logError(Logger logger) {
+ AuditServerLogFormatter.logError(logger, title, logDetails);
+ }
+
+ /**
+ * Get the LogDetails map
+ * @return The LogDetails map
+ */
+ public Map<String, Object> getLogDetails() {
+ return new LinkedHashMap<>(logDetails);
+ }
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java
new file mode 100644
index 000000000..1536134a3
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.DescribeTopicsResult;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.AuditHandler;
+import org.apache.ranger.audit.provider.AuditProviderFactory;
+import org.apache.ranger.authorization.utils.JsonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ranger.audit.provider.MiscUtil.TOKEN_APP_TYPE;
+import static org.apache.ranger.audit.provider.MiscUtil.TOKEN_END;
+import static org.apache.ranger.audit.provider.MiscUtil.TOKEN_START;
+import static org.apache.ranger.audit.provider.MiscUtil.toArray;
+
+public class AuditServerUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditServerUtils.class);
+
+ private Properties auditConfig;
+ private ConcurrentHashMap<String, Map<String, AuditProviderFactory>> auditProviderMap = new ConcurrentHashMap<>();
+
+ public Properties getAuditConfig() {
+ if (auditConfig == null) {
+ return null;
+ }
+ Properties copy = new Properties();
+ copy.putAll(auditConfig);
+ return copy;
+ }
+
+ public void setAuditConfig(Properties auditConfig) {
+ if (auditConfig != null) {
+ this.auditConfig = new Properties();
+ this.auditConfig.putAll(auditConfig);
+ } else {
+ this.auditConfig = null;
+ }
+ }
+
+ public AuditHandler getAuditProvider(AuthzAuditEvent auditEvent) throws Exception {
+ AuditHandler ret = null;
+
+ AuditProviderFactory auditProviderFactory = getAuditProvideFactory(auditEvent);
+ if (auditProviderFactory != null) {
+ ret = auditProviderFactory.getAuditProvider();
+ }
+
+ return ret;
+ }
+
+ private AuditProviderFactory getAuditProvideFactory(AuthzAuditEvent auditEvent) throws Exception {
+ LOG.debug("==> AuditServerUtils.getAuditProviderFactory()");
+
+ AuditProviderFactory ret = null;
+ String hostName = auditEvent.getAgentHostname();
+ String appId = auditEvent.getAgentId();
+ String serviceType = getServiceType(auditEvent);
+
+ Map<String, AuditProviderFactory> auditProviderFactoryMap = auditProviderMap.get(hostName);
+ if (MapUtils.isNotEmpty(auditProviderFactoryMap)) {
+ ret = auditProviderFactoryMap.get(appId);
+ }
+
+ if (ret == null) {
+ ret = createAndCacheAuditProvider(hostName, serviceType, appId);
+ if (MapUtils.isEmpty(auditProviderFactoryMap)) {
+ auditProviderFactoryMap = new HashMap<>();
+ }
+ auditProviderFactoryMap.put(appId, ret);
+ auditProviderMap.put(hostName, auditProviderFactoryMap);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ logAuditProviderCreated();
+ LOG.debug("<== AuditServerUtils.getAuditProviderFactory(): {}", ret);
+ }
+
+ return ret;
+ }
+
+ private AuditProviderFactory createAndCacheAuditProvider(String hostName, String serviceType, String appId) throws Exception {
+ LOG.debug("==> AuditServerUtils.createAndCacheAuditProvider(hostname: {}, serviceType: {}, appId: {})", hostName, serviceType, appId);
+
+ AuditProviderFactory ret = initAuditProvider(serviceType, appId);
+ if (!ret.isInitDone()) {
+ ret = initAuditProvider(serviceType, appId);
+ }
+ if (!ret.isInitDone()) {
+ String msg = String.format("AuditHandler for appId={%s}, hostname={%s} not initialized correctly. Please check audit configuration...", appId, hostName);
+ LOG.error(msg);
+ throw new Exception(msg);
+ }
+
+ LOG.debug("<== AuditServerUtils.createAndCacheAuditProvider(hostname: {}, serviceType: {}, appId: {})", hostName, serviceType, appId);
+ return ret;
+ }
+
+ private AuditProviderFactory initAuditProvider(String serviceType, String appId) {
+ Properties properties = getConfigurationForAuditService(serviceType);
+ AuditProviderFactory ret = AuditProviderFactory.getInstance();
+ ret.init(properties, appId);
+ return ret;
+ }
+
+ private Properties getConfigurationForAuditService(String serviceType) {
+ Properties ret = getAuditConfig();
+ setCloudStorageLocationProperty(ret, serviceType);
+ setSpoolFolderForDestination(ret, serviceType);
+ return ret;
+ }
+
+ private String getServiceType(AuthzAuditEvent auditEvent) {
+ String ret = null;
+ String additionalInfo = auditEvent.getAdditionalInfo();
+ Map<String, String> addInfoMap = JsonUtils.jsonToMapStringString(additionalInfo);
+ if (MapUtils.isNotEmpty(addInfoMap)) {
+ ret = addInfoMap.get("serviceType");
+ }
+ return ret;
+ }
+
+ private void setCloudStorageLocationProperty(Properties prop, String serviceType) {
+ String hdfsDir = prop.getProperty("xasecure.audit.destination.hdfs.dir");
+ if (StringUtils.isNotBlank(hdfsDir)) {
+ StringBuilder sb = new StringBuilder(hdfsDir).append("/").append(serviceType);
+ setProperty(prop, "xasecure.audit.destination.hdfs.dir", sb.toString());
+ }
+ }
+
+ private void setSpoolFolderForDestination(Properties ret, String serviceType) {
+ ArrayList<String> enabledDestinations = getAuditDestinationList(ret);
+ for (String dest : enabledDestinations) {
+ String spoolDirProp = new StringBuilder("xasecure.audit.destination.").append(dest).append(".batch.filespool.dir").toString();
+ String spoolDir = ret.getProperty(spoolDirProp);
+ if (StringUtils.isNotBlank(spoolDir)) {
+ spoolDir = replaceToken(spoolDir, serviceType);
+ setProperty(ret, spoolDirProp, spoolDir);
+ }
+ }
+ }
+
+ private void setProperty(Properties properties, String key, String val) {
+ properties.setProperty(key, val);
+ }
+
+ private ArrayList<String> getAuditDestinationList(Properties props) {
+ ArrayList<String> ret = new ArrayList<>();
+
+ for (Object propNameObj : props.keySet()) {
+ String propName = propNameObj.toString();
+ if (!propName.startsWith(AuditProviderFactory.AUDIT_DEST_BASE)) {
+ continue;
+ }
+ String destName = propName.substring(AuditProviderFactory.AUDIT_DEST_BASE.length() + 1);
+ List<String> splits = toArray(destName, ".");
+ if (splits.size() > 1) {
+ continue;
+ }
+ String value = props.getProperty(propName);
+ if ("enable".equalsIgnoreCase(value) || "enabled".equalsIgnoreCase(value) || "true".equalsIgnoreCase(value)) {
+ ret.add(destName);
+ LOG.info("Audit destination {} is set to {}", propName, value);
+ }
+ }
+ return ret;
+ }
+
+ private String replaceToken(String str, String appType) {
+ String ret = str;
+ for (int startPos = 0; startPos < str.length(); ) {
+ int tagStartPos = str.indexOf(TOKEN_START, startPos);
+
+ if (tagStartPos == -1) {
+ break;
+ }
+
+ int tagEndPos = str.indexOf(TOKEN_END, tagStartPos + TOKEN_START.length());
+
+ if (tagEndPos == -1) {
+ break;
+ }
+
+ String tag = str.substring(tagStartPos, tagEndPos + TOKEN_END.length());
+ String token = tag.substring(TOKEN_START.length(), tag.lastIndexOf(TOKEN_END));
+ String val = "";
+ if (token != null) {
+ if (token.equals(TOKEN_APP_TYPE)) {
+ val = appType;
+ }
+ }
+ ret = str.substring(0, tagStartPos) + val + str.substring(tagEndPos + TOKEN_END.length());
+ startPos = tagStartPos + val.length();
+ }
+
+ return ret;
+ }
+
+ private void logAuditProviderCreated() {
+ if (MapUtils.isNotEmpty(auditProviderMap)) {
+ for (Map.Entry<String, Map<String, AuditProviderFactory>> entry : auditProviderMap.entrySet()) {
+ String hostname = entry.getKey();
+ Map<String, AuditProviderFactory> providerFactoryMap = entry.getValue();
+ if (MapUtils.isNotEmpty(providerFactoryMap)) {
+ for (Map.Entry<String, AuditProviderFactory> ap : providerFactoryMap.entrySet()) {
+ String serviceAppId = ap.getKey();
+ AuditProviderFactory auditProviderFactoryVal = ap.getValue();
+ String apVal = auditProviderFactoryVal.getAuditProvider().getName();
+ LOG.debug("AuditProvider created for HostName: {} ServiceAppId: {} AuditProvider: {}", hostname, serviceAppId, apVal);
+ }
+ }
+ }
+ }
+ }
+
+ public boolean waitUntilTopicReady(Admin admin, String topic, Duration totalWait) throws Exception {
+ long endTime = System.nanoTime() + totalWait.toNanos();
+ long baseSleepMs = 100L;
+ long maxSleepMs = 2000L;
+
+ while (System.nanoTime() < endTime) {
+ try {
+ DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singleton(topic));
+ TopicDescription topicDescription = describeTopicsResult.values().get(topic).get();
+ boolean allHaveLeader = topicDescription.partitions().stream().allMatch(partitionInfo -> partitionInfo.leader() != null);
+ boolean allHaveISR = topicDescription.partitions().stream().allMatch(partitionInfo -> !partitionInfo.isr().isEmpty());
+ if (allHaveLeader && allHaveISR) {
+ return true;
+ }
+ } catch (Exception e) {
+ // If topic hasn't propagated yet, you'll see UnknownTopicOrPartitionException
+ // continue to wait for topic availability
+ if (!(rootCause(e) instanceof UnknownTopicOrPartitionException)) {
+ throw e;
+ }
+ }
+
+ // Sleep until the created topic is available for metadata fetch
+ baseSleepMs = Math.min(maxSleepMs, baseSleepMs * 2);
+ long sleep = baseSleepMs + ThreadLocalRandom.current().nextLong(0, baseSleepMs / 2 + 1);
+ Thread.sleep(sleep);
+ }
+ return false;
+ }
+
+ private static Throwable rootCause(Throwable t) {
+ Throwable throwable = t;
+ while (throwable.getCause() != null) {
+ throwable = throwable.getCause();
+ }
+ return throwable;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml
new file mode 100644
index 000000000..5e80254cc
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/pom.xml
@@ -0,0 +1,345 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-server</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-consumer-hdfs</artifactId>
+ <packaging>war</packaging>
+ <name>Ranger Audit Consumer HDFS</name>
+ <description>Kafka consumer service for writing audits to HDFS/S3/Azure</description>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+
+ <!-- Logging -->
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ <version>${jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-spring</artifactId>
+ <version>${jersey-spring.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax.servlet.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop.thirdparty</groupId>
+ <artifactId>hadoop-shaded-guava</artifactId>
+ <version>${hadoop-shaded-guava.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Kafka Consumer -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ <version>${orc.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-shims</artifactId>
+ <version>${orc.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-hdfs</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-annotations-api</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-el</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-jasper</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <filtering>true</filtering>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh
new file mode 100755
index 000000000..a86f7a67d
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/start-consumer-hdfs.sh
@@ -0,0 +1,164 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ========================================
+# Ranger Audit Consumer HDFS Start Script
+# ========================================
+# This script starts the HDFS consumer service
+# The service consumes audit events from Kafka and writes them to HDFS/S3/Azure
+
+set -e
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SERVICE_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Default directories - can be overridden by environment variables
+AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-${SERVICE_DIR}/target}"
+AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-${SERVICE_DIR}/src/main/resources/conf}"
+AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}"
+
+# Create log directory if it doesn't exist
+mkdir -p "${AUDIT_CONSUMER_LOG_DIR}"
+
+echo "=========================================="
+echo "Starting Ranger Audit Consumer - HDFS"
+echo "=========================================="
+echo "Service Directory: ${SERVICE_DIR}"
+echo "Home Directory: ${AUDIT_CONSUMER_HOME_DIR}"
+echo "Config Directory: ${AUDIT_CONSUMER_CONF_DIR}"
+echo "Log Directory: ${AUDIT_CONSUMER_LOG_DIR}"
+echo "=========================================="
+
+# Check if Java is available
+if [ -z "$JAVA_HOME" ]; then
+ JAVA_CMD=$(which java 2>/dev/null || true)
+ if [ -z "$JAVA_CMD" ]; then
+ echo "[ERROR] JAVA_HOME is not set and java is not in PATH"
+ exit 1
+ fi
+ JAVA_HOME=$(dirname $(dirname $(readlink -f "$JAVA_CMD")))
+ echo "[INFO] JAVA_HOME not set, detected: ${JAVA_HOME}"
+fi
+
+export JAVA_HOME
+export PATH=$JAVA_HOME/bin:$PATH
+
+echo "[INFO] Java version:"
+java -version
+
+# Set heap size (default: 512MB to 2GB)
+AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}"
+
+# Set JVM options
+if [ -z "$AUDIT_CONSUMER_OPTS" ]; then
+ AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.dir=${AUDIT_CONSUMER_LOG_DIR}"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.hdfs.log.file=ranger-audit-consumer-hdfs.log"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8"
+fi
+
+# Add Kerberos configuration if needed
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf"
+ echo "[INFO] Kerberos is enabled"
+fi
+
+export AUDIT_CONSUMER_OPTS
+export AUDIT_CONSUMER_LOG_DIR
+
+echo "[INFO] JAVA_HOME: ${JAVA_HOME}"
+echo "[INFO] HEAP: ${AUDIT_CONSUMER_HEAP}"
+echo "[INFO] JVM_OPTS: ${AUDIT_CONSUMER_OPTS}"
+
+# Find the WAR file
+WAR_FILE=$(find "${AUDIT_CONSUMER_HOME_DIR}" -name "ranger-audit-consumer-hdfs*.war" | head -1)
+
+if [ -z "$WAR_FILE" ] || [ ! -f "$WAR_FILE" ]; then
+ echo "[ERROR] WAR file not found in ${AUDIT_CONSUMER_HOME_DIR}"
+ echo "[ERROR] Please build the project first using: mvn clean package"
+ exit 1
+fi
+
+echo "[INFO] Using WAR file: ${WAR_FILE}"
+
+# Extract WAR if not already extracted
+WEBAPP_DIR="${AUDIT_CONSUMER_HOME_DIR}/webapp/ranger-audit-consumer-hdfs"
+if [ ! -d "${WEBAPP_DIR}" ]; then
+ echo "[INFO] Extracting WAR file..."
+ mkdir -p "${WEBAPP_DIR}"
+ cd "${WEBAPP_DIR}"
+ jar xf "${WAR_FILE}"
+ cd - > /dev/null
+fi
+
+# Build classpath
+RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes"
+for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+done
+
+# Add libext directory if it exists
+if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then
+ for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do
+ if [ -f "${jar}" ]; then
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+ fi
+ done
+fi
+
+export RANGER_CLASSPATH
+
+# Check if already running
+PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.pid"
+if [ -f "${PID_FILE}" ]; then
+ OLD_PID=$(cat "${PID_FILE}")
+ if kill -0 "$OLD_PID" 2>/dev/null; then
+ echo "[WARNING] Ranger Audit Consumer - HDFS is already running (PID: ${OLD_PID})"
+ echo "[INFO] Use stop-consumer-hdfs.sh to stop it first"
+ exit 1
+ else
+ echo "[INFO] Removing stale PID file"
+ rm -f "${PID_FILE}"
+ fi
+fi
+
+# Start the service
+echo "[INFO] Starting Ranger Audit Consumer - HDFS..."
+nohup java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \
+ -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-hdfs-site.xml \
+ -Dhadoop.config.dir=${AUDIT_CONSUMER_CONF_DIR} \
+ -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \
+ -cp "${RANGER_CLASSPATH}" \
+ org.apache.ranger.audit.consumer.HdfsConsumerApplication \
+ >> "${AUDIT_CONSUMER_LOG_DIR}/catalina.out" 2>&1 &
+
+PID=$!
+echo $PID > "${PID_FILE}"
+
+echo "[INFO] ✓ Ranger Audit Consumer - HDFS started successfully"
+echo "[INFO] PID: ${PID}"
+echo "[INFO] Log file: ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.log"
+echo "[INFO] Catalina out: ${AUDIT_CONSUMER_LOG_DIR}/catalina.out"
+echo "[INFO] Health check: http://localhost:7092/api/health"
+echo ""
+echo "To monitor logs: tail -f ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.log"
+echo "To stop service: ${SCRIPT_DIR}/stop-consumer-hdfs.sh"
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh
new file mode 100755
index 000000000..7e98af9b1
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/scripts/stop-consumer-hdfs.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ========================================
+# Ranger Audit Consumer HDFS Stop Script
+# ========================================
+
+set -e
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SERVICE_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Default log directory
+AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}"
+PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-hdfs.pid"
+
+echo "=========================================="
+echo "Stopping Ranger Audit Consumer - HDFS"
+echo "=========================================="
+
+if [ ! -f "${PID_FILE}" ]; then
+ echo "[WARNING] PID file not found: ${PID_FILE}"
+ echo "[INFO] Service may not be running"
+ exit 0
+fi
+
+PID=$(cat "${PID_FILE}")
+
+if ! kill -0 "$PID" 2>/dev/null; then
+ echo "[WARNING] Process ${PID} is not running"
+ echo "[INFO] Removing stale PID file"
+ rm -f "${PID_FILE}"
+ exit 0
+fi
+
+echo "[INFO] Stopping process ${PID}..."
+kill "$PID"
+
+# Wait for process to stop (max 30 seconds)
+TIMEOUT=30
+COUNT=0
+while kill -0 "$PID" 2>/dev/null && [ $COUNT -lt $TIMEOUT ]; do
+ sleep 1
+ COUNT=$((COUNT + 1))
+ echo -n "."
+done
+echo ""
+
+if kill -0 "$PID" 2>/dev/null; then
+ echo "[WARNING] Process did not stop gracefully, forcing shutdown..."
+ kill -9 "$PID"
+ sleep 2
+fi
+
+rm -f "${PID_FILE}"
+echo "[INFO] ✓ Ranger Audit Consumer - HDFS stopped successfully"
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java
new file mode 100644
index 000000000..51f391ab7
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerApplication.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ranger.audit.server.EmbeddedServer;
+import org.apache.ranger.audit.server.HdfsConsumerConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Main application class for Audit Consumer HDFS Service.
+ * This service consumes audit events from Kafka and writes them to HDFS.
+ * It uses embedded Tomcat server for lifecycle management and health check endpoints.
+ */
+public class HdfsConsumerApplication {
+ private static final Logger LOG = LoggerFactory.getLogger(HdfsConsumerApplication.class);
+ private static final String APP_NAME = "ranger-audit-consumer-hdfs";
+ private static final String CONFIG_PREFIX = "ranger.audit.consumer.hdfs.service.";
+
+ private HdfsConsumerApplication() {
+ }
+
+ public static void main(String[] args) {
+ LOG.info("==========================================================================");
+ LOG.info("==> Starting Ranger Audit Consumer HDFS Service");
+ LOG.info("==========================================================================");
+
+ try {
+ // Load configuration (includes core-site.xml, hdfs-site.xml, and ranger-audit-consumer-hdfs-site.xml)
+ Configuration config = HdfsConsumerConfig.getInstance();
+
+ EmbeddedServer server = new EmbeddedServer(config, APP_NAME, CONFIG_PREFIX);
+ server.start();
+
+ LOG.info("<== Ranger Audit Consumer HDFS Service Started Successfully");
+ } catch (Exception e) {
+ LOG.error("<== Failed to start Ranger Audit Consumer HDFS Service", e);
+ System.exit(1);
+ }
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java
new file mode 100644
index 000000000..643de22c7
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/HdfsConsumerManager.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer;
+
+import org.apache.ranger.audit.consumer.kafka.AuditConsumer;
+import org.apache.ranger.audit.consumer.kafka.AuditConsumerRegistry;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.server.HdfsConsumerConfig;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Spring component that manages the lifecycle of HDFS consumer threads.
+ * This manager:
+ * - Initializes the consumer registry
+ * - Creates HDFS consumer instances
+ * - Starts consumer threads
+ * - Handles graceful shutdown
+ */
+@Component
+public class HdfsConsumerManager {
+ private static final Logger LOG = LoggerFactory.getLogger(HdfsConsumerManager.class);
+
+ private final AuditConsumerRegistry consumerRegistry = AuditConsumerRegistry.getInstance();
+ private final List<AuditConsumer> consumers = new ArrayList<>();
+ private final List<Thread> consumerThreads = new ArrayList<>();
+
+ @PostConstruct
+ public void init() {
+ LOG.info("==> HdfsConsumerManager.init()");
+
+ try {
+ HdfsConsumerConfig config = HdfsConsumerConfig.getInstance();
+ Properties props = config.getProperties();
+
+ if (props == null) {
+ LOG.error("Configuration properties are null");
+ throw new RuntimeException("Failed to load configuration");
+ }
+
+ // Initialize and register HDFS Consumer
+ initializeConsumerClasses(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX);
+
+ // Create consumers from registry
+ List<AuditConsumer> createdConsumers = consumerRegistry.createConsumers(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX);
+ consumers.addAll(createdConsumers);
+
+ if (consumers.isEmpty()) {
+ LOG.warn("No consumers were created! Verify that xasecure.audit.destination.hdfs=true");
+ } else {
+ LOG.info("Created {} HDFS consumer(s)", consumers.size());
+
+ // Start consumer threads
+ startConsumers();
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to initialize HdfsConsumerManager", e);
+ throw new RuntimeException("Failed to initialize HdfsConsumerManager", e);
+ }
+
+ LOG.info("<== HdfsConsumerManager.init() - {} consumer thread(s) started", consumerThreads.size());
+ }
+
+ private void initializeConsumerClasses(Properties props, String propPrefix) {
+ LOG.info("==> HdfsConsumerManager.initializeConsumerClasses()");
+
+ String clsStr = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_CLASSES, "org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer");
+
+ String[] hdfsConsumerClasses = clsStr.split(",");
+
+ LOG.info("Initializing {} consumer class(es)", hdfsConsumerClasses.length);
+
+ for (String hdfsConsumerClassName : hdfsConsumerClasses) {
+ hdfsConsumerClassName = hdfsConsumerClassName.trim();
+
+ if (hdfsConsumerClassName.isEmpty()) {
+ continue;
+ }
+
+ try {
+ Class<?> consumerClass = Class.forName(hdfsConsumerClassName);
+ LOG.info("Successfully initialized consumer class: {}", consumerClass.getName());
+ } catch (ClassNotFoundException e) {
+ LOG.error("Consumer class not found: {}. Ensure the class is on the classpath.", hdfsConsumerClassName, e);
+ } catch (Exception e) {
+ LOG.error("Error initializing consumer class: {}", hdfsConsumerClassName, e);
+ }
+ }
+
+ LOG.info("Registered consumer factories: {}", consumerRegistry.getRegisteredDestinationTypes());
+ LOG.info("<== HdfsConsumerManager.initializeConsumerClasses()");
+ }
+
+ /**
+ * Start all consumer threads
+ */
+ private void startConsumers() {
+ LOG.info("==> HdfsConsumerManager.startConsumers()");
+
+ logConsumerStartup();
+
+ for (AuditConsumer consumer : consumers) {
+ try {
+ String consumerName = consumer.getClass().getSimpleName();
+ Thread consumerThread = new Thread(consumer, consumerName);
+ consumerThread.setDaemon(true);
+ consumerThread.start();
+ consumerThreads.add(consumerThread);
+
+ LOG.info("Started {} thread [Thread-ID: {}, Thread-Name: '{}']",
+ consumerName, consumerThread.getId(), consumerThread.getName());
+ } catch (Exception e) {
+ LOG.error("Error starting consumer: {}", consumer.getClass().getSimpleName(), e);
+ }
+ }
+
+ LOG.info("<== HdfsConsumerManager.startConsumers() - {} thread(s) started", consumerThreads.size());
+ }
+
+ private void logConsumerStartup() {
+ LOG.info("################## HDFS CONSUMER SERVICE STARTUP ######################");
+
+ if (consumers.isEmpty()) {
+ LOG.warn("WARNING: No HDFS consumers are enabled!");
+ LOG.warn("Verify: xasecure.audit.destination.hdfs=true in configuration");
+ } else {
+ AuditServerLogFormatter.LogBuilder builder = AuditServerLogFormatter.builder("HDFS Consumer Status");
+
+ for (AuditConsumer consumer : consumers) {
+ String consumerType = consumer.getClass().getSimpleName();
+ builder.add(consumerType, "ENABLED");
+ builder.add("Topic", consumer.getTopicName());
+ }
+
+ builder.logInfo(LOG);
+ LOG.info("Starting {} HDFS consumer thread(s)...", consumers.size());
+ }
+ LOG.info("########################################################################");
+ }
+
+ @PreDestroy
+ public void shutdown() {
+ LOG.info("==> HdfsConsumerManager.shutdown()");
+
+ // Shutdown all consumers
+ for (AuditConsumer consumer : consumers) {
+ try {
+ LOG.info("Shutting down consumer: {}", consumer.getClass().getSimpleName());
+ consumer.shutdown();
+ LOG.info("Consumer shutdown completed: {}", consumer.getClass().getSimpleName());
+ } catch (Exception e) {
+ LOG.error("Error shutting down consumer: {}", consumer.getClass().getSimpleName(), e);
+ }
+ }
+
+ // Wait for threads to terminate
+ for (Thread thread : consumerThreads) {
+ if (thread.isAlive()) {
+ try {
+ LOG.info("Waiting for thread to terminate: {}", thread.getName());
+ thread.join(10000); // Wait up to 10 seconds
+ if (thread.isAlive()) {
+ LOG.warn("Thread did not terminate within 10 seconds: {}", thread.getName());
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for thread to terminate: {}", thread.getName(), e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ consumers.clear();
+ consumerThreads.clear();
+ consumerRegistry.clearActiveConsumers();
+
+ LOG.info("<== HdfsConsumerManager.shutdown() - All HDFS consumers stopped");
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java
new file mode 100644
index 000000000..a41d8e87b
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditHDFSConsumer.java
@@ -0,0 +1,497 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.audit.consumer.kafka;
+
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.ranger.audit.provider.AuditProviderFactory;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.server.HdfsConsumerConfig;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class AuditHDFSConsumer extends AuditConsumerBase implements AuditConsumer {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditHDFSConsumer.class);
+ private static final String RANGER_AUDIT_HDFS_CONSUMER_GROUP = AuditServerConstants.DEFAULT_RANGER_AUDIT_HDFS_CONSUMER_GROUP;
+
+ // Register AuditHDFSConsumer factory in the audit consumer registry
+ static {
+ try {
+ AuditConsumerRegistry.getInstance().registerFactory(AuditServerConstants.PROP_HDFS_DEST_PREFIX, (props, propPrefix) -> new AuditHDFSConsumer(props, propPrefix));
+ LOG.info("Registered HDFS consumer with AuditConsumerRegistry");
+ } catch (Exception e) {
+ LOG.error("Failed to register HDFS consumer factory", e);
+ }
+ }
+
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private ExecutorService consumerThreadPool;
+ private final Map<String, ConsumerWorker> consumerWorkers = new ConcurrentHashMap<>();
+ private int consumerThreadCount = 1;
+ // Offset management configuration (batch or manual only supported)
+ private String offsetCommitStrategy = AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY;
+ private long offsetCommitInterval = AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS;
+ // Message routing handler for HDFS destination
+ private AuditRouterHDFS auditRouterHDFS;
+ private Properties props;
+ private String propPrefix;
+
+ public AuditHDFSConsumer(Properties props, String propPrefix) throws Exception {
+ super(props, propPrefix, RANGER_AUDIT_HDFS_CONSUMER_GROUP);
+ this.props = new Properties();
+ this.props.putAll(props);
+ this.propPrefix = propPrefix;
+ init(props, propPrefix);
+ }
+
+ @Override
+ public void init(Properties props, String propPrefix) throws Exception {
+ LOG.info("==> AuditHDFSConsumer.init(): AuditHDFSConsumer initializing with appId-based threading and offset management");
+
+ consumer = getKafkaConsumer();
+
+ // Initialize Ranger UGI for HDFS operations if Kerberos is enabled
+ initializeRangerUGI();
+
+ // Add Hadoop configuration properties from HdfsConsumerConfig to props
+ addHadoopConfigToProps(props);
+
+ // Initialize consumer configuration
+ initConsumerConfig(props, propPrefix);
+
+ // Initialize destination handler for message processing
+ auditRouterHDFS = new AuditRouterHDFS();
+ auditRouterHDFS.init(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_HDFS_DEST_PREFIX);
+
+ LOG.info("<== AuditHDFSConsumer.init(): AuditHDFSConsumer initialized successfully");
+ }
+
+ private void initializeRangerUGI() throws Exception {
+ LOG.info("==> AuditHDFSConsumer.initializeRangerUGI()");
+
+ try {
+ HdfsConsumerConfig auditConfig = HdfsConsumerConfig.getInstance();
+ String authType = auditConfig.get(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE, "simple");
+
+ if (!AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) {
+ LOG.info("Hadoop authentication is not Kerberos ({}), skipping Ranger UGI initialization", authType);
+ return;
+ }
+
+ String principal = auditConfig.get(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL);
+ String keytab = auditConfig.get(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB);
+ String hostName = auditConfig.get(AuditServerConstants.AUDIT_SERVER_PROP_PREFIX + "host");
+
+ if (principal == null || keytab == null) {
+ LOG.warn("Kerberos is enabled but principal or keytab is null! principal={}, keytab={}", principal, keytab);
+ String msg = String.format("Kerberos is enabled but principal or keytab is null! principal={}, keytab={}", principal, keytab);
+ throw new Exception(msg);
+ }
+
+ if (principal.contains("_HOST")) {
+ try {
+ principal = SecureClientLogin.getPrincipal(principal, hostName);
+ LOG.info("Resolved principal from [{}] using hostname [{}]", principal, hostName);
+ } catch (IOException e) {
+ LOG.error("Failed to resolve principal pattern [{}] with hostname [{}]", principal, hostName, e);
+ throw e;
+ }
+ }
+
+ // Set Hadoop security configuration from core-site.xml
+ org.apache.hadoop.conf.Configuration coreSite = auditConfig.getCoreSiteConfiguration();
+ UserGroupInformation.setConfiguration(coreSite);
+
+ LOG.info("Initializing Ranger UGI for HDFS writes: principal={}, keytab={}", principal, keytab);
+
+ UserGroupInformation rangerUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab);
+
+ MiscUtil.setUGILoginUser(rangerUGI, null);
+
+ LOG.info("<== AuditHDFSConsumer.initializeRangerUGI(): Ranger UGI initialized successfully: user={}, auth={}, hasKerberos={}",
+ rangerUGI.getUserName(), rangerUGI.getAuthenticationMethod(), rangerUGI.hasKerberosCredentials());
+ } catch (IOException e) {
+ LOG.error("Failed to initialize Ranger UGI for HDFS writes", e);
+ throw e;
+ }
+ }
+
+ /**
+ * Add Hadoop configuration properties from core-site.xml and hdfs-site.xml to props.
+ */
+ private void addHadoopConfigToProps(Properties props) {
+ LOG.info("==> AuditHDFSConsumer.addHadoopConfigToProps()");
+
+ try {
+ HdfsConsumerConfig hdfsConfig = HdfsConsumerConfig.getInstance();
+ String configPrefix = "xasecure.audit.destination.hdfs.config.";
+
+ Properties hadoopProps = hdfsConfig.getHadoopPropertiesWithPrefix(configPrefix);
+ props.putAll(hadoopProps);
+
+ LOG.info("<== AuditHDFSConsumer.addHadoopConfigToProps(): Added {} Hadoop configuration properties from HdfsConsumerConfig", hadoopProps.size());
+ } catch (Exception e) {
+ LOG.error("Failed to add Hadoop configuration properties to props", e);
+ }
+ }
+
+ private void initConsumerConfig(Properties props, String propPrefix) {
+ LOG.info("==> AuditHDFSConsumer.initConsumerConfig()");
+
+ // Get consumer thread count
+ this.consumerThreadCount = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_THREAD_COUNT, 1);
+ LOG.info("HDFS consumer thread count: {}", consumerThreadCount);
+
+ // Initialize offset management configuration
+ initializeOffsetManagement(props, propPrefix);
+
+ LOG.info("<== AuditHDFSConsumer.initConsumerConfig()");
+ }
+
+ private void initializeOffsetManagement(Properties props, String propPrefix) {
+ LOG.info("==> AuditHDFSConsumer.initializeOffsetManagement()");
+
+ // Get offset commit strategy
+ this.offsetCommitStrategy = MiscUtil.getStringProperty(props,
+ propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_STRATEGY,
+ AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY);
+
+ // Get offset commit interval (only used for manual strategy)
+ this.offsetCommitInterval = MiscUtil.getLongProperty(props,
+ propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_INTERVAL,
+ AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS);
+
+ AuditServerLogFormatter.builder("HDFS Consumer Offset Management Configuration")
+ .add("Commit Strategy", offsetCommitStrategy)
+ .add("Commit Interval (ms)", offsetCommitInterval + " (used in manual mode only)")
+ .logInfo(LOG);
+
+ LOG.info("<== AuditHDFSConsumer.initializeOffsetManagement()");
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (auditRouterHDFS == null) {
+ init(this.props, this.propPrefix);
+ }
+
+ LOG.info("Starting AuditHDFSConsumer with appId-based thread");
+ startMultithreadedConsumption();
+
+ // Keep main thread alive while consumer threads are running
+ while (running.get()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ LOG.info("HDFS consumer main thread interrupted");
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Error in AuditHDFSConsumer", e);
+ } finally {
+ shutdown();
+ }
+ }
+
+ @Override
+ public KafkaConsumer<String, String> getKafkaConsumer() {
+ return consumer;
+ }
+
+ @Override
+ public void processMessage(String audit) throws Exception {
+ processMessage(audit, null);
+ }
+
+ public void processMessage(String message, String partitionKey) throws Exception {
+ auditRouterHDFS.routeAuditMessage(message, partitionKey);
+ }
+
+ @Override
+ public String getTopicName() {
+ return topicName;
+ }
+
+ public String getConsumerGroupId() {
+ return consumerGroupId;
+ }
+
+ private void startMultithreadedConsumption() {
+ LOG.debug("==> AuditHDFSConsumer.startMultithreadedConsumption()");
+
+ if (running.compareAndSet(false, true)) {
+ startConsumerWorkers();
+ }
+
+ LOG.debug("<== AuditHDFSConsumer.startMultithreadedConsumption()");
+ }
+
+ private void startConsumerWorkers() {
+ LOG.info("==> AuditHDFSConsumer.startConsumerWorkers(): Creating {} consumer workers for horizontal scaling", consumerThreadCount);
+ LOG.info("Each worker will subscribe to topic '{}' and process partitions assigned by Kafka", topicName);
+
+ // Create thread pool sized for consumer workers
+ consumerThreadPool = Executors.newFixedThreadPool(consumerThreadCount);
+ LOG.info("Created thread pool with {} threads for scalable HDFS consumption", consumerThreadCount);
+
+ // Create HDFS consumer workers
+ for (int i = 0; i < consumerThreadCount; i++) {
+ String workerId = "hdfs-worker-" + i;
+ ConsumerWorker worker = new ConsumerWorker(workerId, new ArrayList<>());
+ consumerWorkers.put(workerId, worker);
+ consumerThreadPool.submit(worker);
+
+ LOG.info("Started HDFS consumer worker '{}' - will process ANY appId assigned by Kafka", workerId);
+ }
+
+ LOG.info("<== AuditHDFSConsumer.startConsumerWorkers(): All {} workers started in SUBSCRIBE mode", consumerThreadCount);
+ }
+
+ private class ConsumerWorker implements Runnable {
+ private final String workerId;
+ private final List<Integer> assignedPartitions;
+ private KafkaConsumer<String, String> workerConsumer;
+
+ // Offset management
+ private final Map<TopicPartition, OffsetAndMetadata> pendingOffsets = new HashMap<>();
+ private final AtomicLong lastCommitTime = new AtomicLong(System.currentTimeMillis());
+ private final AtomicInteger messagesProcessedSinceLastCommit = new AtomicInteger(0);
+
+ public ConsumerWorker(String workerId, List<Integer> assignedPartitions) {
+ this.workerId = workerId;
+ this.assignedPartitions = assignedPartitions;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // Create consumer for this worker with offset management configuration
+ Properties workerConsumerProps = new Properties();
+ workerConsumerProps.putAll(consumerProps);
+
+ // Configure offset management based on strategy
+ configureOffsetManagement(workerConsumerProps);
+
+ workerConsumer = new KafkaConsumer<>(workerConsumerProps);
+
+ // Create re-balance listener
+ AuditConsumerRebalanceListener rebalanceListener = new AuditConsumerRebalanceListener(
+ workerId,
+ AuditServerConstants.DESTINATION_HDFS,
+ topicName,
+ offsetCommitStrategy,
+ consumerGroupId,
+ workerConsumer,
+ pendingOffsets,
+ messagesProcessedSinceLastCommit,
+ lastCommitTime,
+ assignedPartitions);
+
+ // Subscribe to topic with re-balance listener
+ workerConsumer.subscribe(Collections.singletonList(topicName), rebalanceListener);
+
+ LOG.info("[HDFS-CONSUMER] Worker '{}' subscribed successfully, waiting for partition assignment from Kafka", workerId);
+ long threadId = Thread.currentThread().getId();
+ String threadName = Thread.currentThread().getName();
+ LOG.info("[HDFS-CONSUMER-STARTUP] Worker '{}' [Thread-ID: {}, Thread-Name: '{}'] started | Topic: '{}' | Consumer-Group: {} | Mode: SUBSCRIBE",
+ workerId, threadId, threadName, topicName, consumerGroupId);
+
+ // Consume messages
+ while (running.get()) {
+ ConsumerRecords<String, String> records = workerConsumer.poll(Duration.ofMillis(100));
+
+ if (!records.isEmpty()) {
+ processRecordBatch(records);
+ // Handle offset committing based on strategy
+ handleOffsetCommitting();
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error in HDFS consumer worker '{}'", workerId, e);
+ } finally {
+ // Final offset commit before shutdown
+ commitPendingOffsets(true);
+
+ if (workerConsumer != null) {
+ try {
+ LOG.info("HDFS Worker '{}': Unsubscribing from topic", workerId);
+ workerConsumer.unsubscribe();
+ } catch (Exception e) {
+ LOG.warn("HDFS Worker '{}': Error during unsubscribe", workerId, e);
+ }
+
+ try {
+ LOG.info("HDFS Worker '{}': Closing consumer", workerId);
+ workerConsumer.close();
+ } catch (Exception e) {
+ LOG.error("Error closing consumer for HDFS worker '{}'", workerId, e);
+ }
+ }
+ LOG.info("HDFS consumer worker '{}' stopped", workerId);
+ }
+ }
+
+ private void configureOffsetManagement(Properties consumerProps) {
+ // Always disable auto commit - only batch or manual strategies supported
+ consumerProps.put("enable.auto.commit", "false");
+ LOG.debug("HDFS worker '{}' configured for manual offset commit with strategy: {}", workerId, offsetCommitStrategy);
+ }
+
+ private void processRecordBatch(ConsumerRecords<String, String> records) {
+ for (ConsumerRecord<String, String> record : records) {
+ try {
+ LOG.debug("HDFS worker '{}' consumed: partition={}, key={}, offset={}",
+ workerId, record.partition(), record.key(), record.offset());
+
+ // Process the message using the destination handler
+ // The partition key (record.key()) contains the appId for HDFS path routing
+ processMessage(record.value(), record.key());
+
+ // Track offset for manual commit strategies
+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ pendingOffsets.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ messagesProcessedSinceLastCommit.incrementAndGet();
+ } catch (Exception e) {
+ LOG.error("Error processing message in HDFS worker '{}': partition={}, key={}, offset={}",
+ workerId, record.partition(), record.key(), record.offset(), e);
+
+ // On error, track offset to prevent reprocessing
+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ pendingOffsets.put(partition, new OffsetAndMetadata(record.offset()));
+ }
+ }
+ }
+
+ private void handleOffsetCommitting() {
+ boolean shouldCommit = false;
+ long currentTime = System.currentTimeMillis();
+
+ if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_BATCH.equals(offsetCommitStrategy)) {
+ // Commit after processing each batch
+ shouldCommit = !pendingOffsets.isEmpty();
+ } else if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_MANUAL.equals(offsetCommitStrategy)) {
+ // Commit based on time interval
+ shouldCommit = (currentTime - lastCommitTime.get()) >= offsetCommitInterval && !pendingOffsets.isEmpty();
+ }
+
+ if (shouldCommit) {
+ commitPendingOffsets(false);
+ }
+ }
+
+ private void commitPendingOffsets(boolean isShutdown) {
+ if (pendingOffsets.isEmpty()) {
+ return;
+ }
+
+ try {
+ workerConsumer.commitSync(pendingOffsets);
+
+ LOG.debug("HDFS worker '{}' committed {} offsets, processed {} messages",
+ workerId, pendingOffsets.size(), messagesProcessedSinceLastCommit.get());
+
+ // Clear committed offsets
+ pendingOffsets.clear();
+ lastCommitTime.set(System.currentTimeMillis());
+ messagesProcessedSinceLastCommit.set(0);
+ } catch (Exception e) {
+ LOG.error("Error committing offsets in HDFS worker '{}': {}", workerId, pendingOffsets, e);
+
+ if (isShutdown) {
+ // During shutdown, retry to avoid loss of any offsets
+ try {
+ Thread.sleep(1000);
+ workerConsumer.commitSync(pendingOffsets);
+ LOG.info("Successfully committed offsets on retry during shutdown for HDFS worker '{}'", workerId);
+ } catch (Exception retryException) {
+ LOG.error("Failed to commit offsets even on retry during shutdown for HDFS worker '{}'", workerId, retryException);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ LOG.info("==> AuditHDFSConsumer.shutdown()");
+
+ // Stop consumer threads
+ running.set(false);
+
+ // Shutdown consumer workers
+ if (consumerThreadPool != null) {
+ consumerThreadPool.shutdownNow();
+ try {
+ if (!consumerThreadPool.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) {
+ LOG.warn("HDFS consumer thread pool did not terminate within 30 seconds");
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for HDFS consumer thread pool to terminate", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ consumerWorkers.clear();
+
+ // Shutdown destination handler
+ if (auditRouterHDFS != null) {
+ try {
+ auditRouterHDFS.shutdown();
+ } catch (Exception e) {
+ LOG.error("Error shutting down HDFS destination handler", e);
+ }
+ }
+
+ // Close main Kafka consumer
+ if (consumer != null) {
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ LOG.error("Error closing main Kafka consumer", e);
+ }
+ }
+
+ LOG.info("<== AuditHDFSConsumer.shutdown() complete");
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java
new file mode 100644
index 000000000..49080aacb
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditRouterHDFS.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer.kafka;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ranger.audit.destination.HDFSAuditDestination;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Router class that routes audit messages to different HDFSAuditDestination threads
+ * based on the app_id.
+ * Each app_id gets its own HDFSAuditDestination instance running in a separate thread.
+ *
+ * This router writes audits to HDFS as the rangerauditserver user. The audit folder in HDFS
+ * should be configured with appropriate permissions to allow rangerauditserver to write audits
+ * either by Ranger policy or by HDFS acl.
+ */
+public class AuditRouterHDFS {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditRouterHDFS.class);
+
+ private final Map<String, HDFSAuditDestination> destinationMap = new ConcurrentHashMap<>();
+ private final Map<String, ExecutorService> executorMap = new ConcurrentHashMap<>();
+ private final ObjectMapper jsonMapper = new ObjectMapper();
+ private Properties props;
+ private String hdfsPropPrefix;
+
+ public AuditRouterHDFS() {
+ }
+
+ public void init(Properties props, String hdfsPropPrefix) {
+ LOG.info("==> AuditRouterHDFS.init()");
+
+ this.props = new Properties();
+ this.props.putAll(props);
+ this.hdfsPropPrefix = hdfsPropPrefix;
+
+ LOG.info("<== AuditRouterHDFS.init()");
+ }
+
+ /**
+ * Routes audit message to appropriate HDFSAuditDestination based on app_id
+ * @param message JSON audit message
+ * @param partitionKey The partition key from Kafka (used as app_id)
+ */
+ public void routeAuditMessage(String message, String partitionKey) {
+ LOG.debug("==> AuditRouterHDFS:routeAuditMessage(): Message => {}, partitionKey => {}", message, partitionKey);
+
+ try {
+ String appId = extractAppId(message, partitionKey);
+ String serviceType = extractServiceType(message);
+ String agentHostname = extractAgentHostname(message);
+
+ LOG.debug("Routing audit message for app_id: {}, serviceType: {}, agentHostname: {}", appId, serviceType, agentHostname);
+
+ // Get or create HDFSAuditDestination for the appId
+ if (appId != null) {
+ HDFSAuditDestination hdfsAuditDestination = getHdfsAuditDestination(appId, serviceType, agentHostname);
+
+ // Submit message to destination's thread for processing
+ final String finalAppId = appId;
+ ExecutorService executor = executorMap.get(appId);
+
+ if (executor != null && !executor.isShutdown()) {
+ executor.submit(() -> {
+ try {
+ hdfsAuditDestination.logJSON(Collections.singletonList(message));
+ LOG.debug("Successfully wrote audit for app_id: {}", finalAppId);
+ } catch (Exception e) {
+ LOG.error("Error processing audit message for app_id: {}", finalAppId, e);
+ }
+ });
+ } else {
+ LOG.warn("Executor is null or shutdown for app_id: {}", appId);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("AuditRouterHDFS:routeAuditMessage(): Error routing audit message: {}", message, e);
+ }
+
+ LOG.debug("<== AuditRouterHDFS:routeAuditMessage()");
+ }
+
+ /**
+ * Extract app_id from audit message. First tries partition key, then falls back to parsing JSON.
+ */
+ private String extractAppId(String message, String partitionKey) {
+ // First try to use partition key as app_id if available
+ if (partitionKey != null && !partitionKey.trim().isEmpty()) {
+ return partitionKey;
+ }
+ // Fall back to extracting from JSON message
+ try {
+ JsonNode rootNode = jsonMapper.readTree(message);
+ JsonNode agentIdNode = rootNode.get("agent");
+ if (agentIdNode != null) {
+ return agentIdNode.asText();
+ }
+ } catch (Exception e) {
+ LOG.debug("AuditRouterHDFS:extractAppId(): Failed to parse JSON message for app_id extraction: {}", e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Extract serviceType from audit message additional_info
+ */
+ private String extractServiceType(String message) {
+ try {
+ JsonNode rootNode = jsonMapper.readTree(message);
+ JsonNode additionalInfoNode = rootNode.get("additional_info");
+ JsonNode additionalInfoJson = jsonMapper.readTree(additionalInfoNode.asText());
+ JsonNode serviceTypeNode = additionalInfoJson.get("serviceType");
+ if (serviceTypeNode != null) {
+ return serviceTypeNode.asText();
+ }
+ } catch (Exception e) {
+ LOG.debug("AuditRouterHDFS:extractServiceType(): Failed to parse JSON message for serviceType extraction: {}", e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Extract agentHostname from audit message
+ */
+ private String extractAgentHostname(String message) {
+ try {
+ JsonNode rootNode = jsonMapper.readTree(message);
+ JsonNode agentHostNode = rootNode.get("agentHost");
+ if (agentHostNode != null) {
+ return agentHostNode.asText();
+ }
+ } catch (Exception e) {
+ LOG.debug("AuditRouterHDFS:extractAgentHostname(): Failed to parse JSON message for agentHostname extraction: {}", e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Get or create HDFSAuditDestination for the given app_id
+ */
+ private synchronized HDFSAuditDestination getHdfsAuditDestination(String appId, String serviceType, String agentHostname) {
+ LOG.debug("==> AuditRouterHDFS:getHdfsAuditDestination() for app_id: {}, serviceType: {} and agentHostname: {}", appId, serviceType, agentHostname);
+ HDFSAuditDestination destination = destinationMap.get(appId);
+ if (destination == null) {
+ try {
+ destination = createHDFSDestination(appId, serviceType, agentHostname);
+ destinationMap.put(appId, destination);
+
+ // Create dedicated executor for this destination
+ ExecutorService executor = Executors.newSingleThreadExecutor(r -> {
+ Thread t = new Thread(r, "HDFSAuditDestination-" + appId);
+ t.setDaemon(true);
+ return t;
+ });
+ executorMap.put(appId, executor);
+ } catch (Exception e) {
+ LOG.error("Failed to create HDFSAuditDestination for app_id: {}", appId, e);
+ throw new RuntimeException("AuditRouterHDFS:getOrCreateDestination() : Failed to create destination for app_id: " + appId, e);
+ }
+ }
+
+ LOG.debug("<== AuditRouterHDFS:getHdfsAuditDestination()..got HDFSAuditDestination for app_id: {}", appId);
+
+ return destination;
+ }
+
+ /**
+ * Create and initialize HDFSAuditDestination with app_id specific configuration
+ */
+ private HDFSAuditDestination createHDFSDestination(String appId, String serviceType, String agentHostname) throws Exception {
+ LOG.debug("==> AuditRouterHDFS:createHDFSDestination(): Creating new HDFSAuditDestination for app_id: {}, serviceType: {}, agentHostname: {}", appId, serviceType, agentHostname);
+
+ HDFSAuditDestination destination = new HDFSAuditDestination();
+
+ // Create app_id specific properties
+ Properties appSpecificProps = new Properties();
+ appSpecificProps.putAll(props);
+
+ // Determine file type and writer implementation
+ String fileType = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".batch.filequeue.filetype", "json");
+ String writerImpl = getWriterImplementation(fileType);
+
+ // Configure directory properties
+ String baseDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".dir", "/ranger/audit/" + serviceType);
+ String subDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".subdir", appId + "/%time:yyyyMMdd%/");
+
+ // Set file extension based on file type
+ String fileExtension = getFileExtension(fileType);
+ // Get unique instance identifier for filename uniqueness across scaled audit services
+ String instanceId = getUniqueInstanceIdentifier();
+
+ // Use agentHostname from audit message if available, otherwise fall back to local hostname token
+ String hostnameValue = (agentHostname != null && !agentHostname.isEmpty()) ? agentHostname : "%hostname%";
+
+ // Build default filename with agent hostname to properly identify the source system
+ String defaultFileName = appId + "_ranger_audit_" + hostnameValue + "_" + instanceId + fileExtension;
+ String fileNameFormat = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".filename.format", defaultFileName);
+
+ // If filename format contains %hostname% and we have agentHostname, replace it
+ if (agentHostname != null && !agentHostname.isEmpty() && fileNameFormat.contains("%hostname%")) {
+ fileNameFormat = fileNameFormat.replace("%hostname%", agentHostname);
+ }
+
+ // Set the enhanced properties
+ appSpecificProps.setProperty(hdfsPropPrefix + ".dir", baseDir);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".subdir", subDir);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".filename.format", fileNameFormat);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".filewriter.impl", writerImpl);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".batch.filequeue.filetype", fileType);
+ // Preserve other properties
+ preserveFileSystemProperties(appSpecificProps, hdfsPropPrefix);
+
+ // Log configuration
+ AuditServerLogFormatter.builder("Initializing HDFSAuditDestination for app_id: " + appId)
+ .add("Base directory", baseDir)
+ .add("Subdirectory pattern", subDir)
+ .add("Filename format", fileNameFormat)
+ .add("Agent hostname", agentHostname)
+ .add("Instance identifier", instanceId)
+ .add("File type", fileType)
+ .add("Writer implementation", writerImpl)
+ .logInfo(LOG);
+
+ destination.init(appSpecificProps, hdfsPropPrefix);
+ destination.start();
+
+ LOG.debug("<== AuditRouterHDFS:createHDFSDestination(): Created new HDFSDestination {}", destination.getName());
+
+ return destination;
+ }
+
+ private String getWriterImplementation(String fileType) {
+ switch (fileType.toLowerCase()) {
+ case "orc":
+ return "org.apache.ranger.audit.utils.RangerORCAuditWriter";
+ case "json":
+ default:
+ return "org.apache.ranger.audit.utils.RangerJSONAuditWriter";
+ }
+ }
+
+ private String getFileExtension(String fileType) {
+ switch (fileType.toLowerCase()) {
+ case "orc":
+ return ".orc";
+ case "json":
+ default:
+ return ".log";
+ }
+ }
+
+ private void preserveFileSystemProperties(Properties appSpecificProps, String baseDestPrefix) {
+ String fileRolloverSec = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.sec");
+ if (fileRolloverSec != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.sec", fileRolloverSec);
+ }
+
+ String rolloverPeriod = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.period");
+ if (rolloverPeriod != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.period", rolloverPeriod);
+ }
+
+ String appendEnabled = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.append.enabled");
+ if (appendEnabled != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.append.enabled", appendEnabled);
+ }
+
+ String periodicRolloverEnabled = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.enable.periodic.rollover");
+ if (periodicRolloverEnabled != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.enable.periodic.rollover", periodicRolloverEnabled);
+ }
+
+ String periodicRolloverCheckTime = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.periodic.rollover.check.sec");
+ if (periodicRolloverCheckTime != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.periodic.rollover.check.sec", periodicRolloverCheckTime);
+ }
+ }
+
+ private String getUniqueInstanceIdentifier() {
+ String jvmInstanceId = MiscUtil.getJvmInstanceId();
+ LOG.info("Using JVM instance ID as unique identifier: {}", jvmInstanceId);
+ return jvmInstanceId;
+ }
+
+ public void shutdown() {
+ LOG.info("==> AuditRouterHDFS.shutdown()");
+
+ // Shutdown all executors
+ for (Map.Entry<String, ExecutorService> entry : executorMap.entrySet()) {
+ String appId = entry.getKey();
+ ExecutorService executor = entry.getValue();
+
+ LOG.info("Shutting down executor for app_id: {}", appId);
+ executor.shutdown();
+
+ try {
+ if (!executor.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) {
+ LOG.warn("Executor for app_id {} did not terminate gracefully, forcing shutdown", appId);
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for executor shutdown for app_id: {}", appId);
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ // Stop all destinations
+ for (Map.Entry<String, HDFSAuditDestination> entry : destinationMap.entrySet()) {
+ String appId = entry.getKey();
+ HDFSAuditDestination destination = entry.getValue();
+
+ LOG.info("Stopping HDFSAuditDestination for app_id: {}", appId);
+ try {
+ destination.stop();
+ } catch (Exception e) {
+ LOG.error("Error stopping HDFSAuditDestination for app_id: {}", appId, e);
+ }
+ }
+
+ destinationMap.clear();
+ executorMap.clear();
+
+ LOG.info("<== AuditRouterHDFS.shutdown()");
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java
new file mode 100644
index 000000000..ba1303582
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.rest;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ranger.audit.consumer.HdfsConsumerManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Health check REST endpoint for HDFS Consumer Service
+ */
+@Path("/health")
+@Component
+@Scope("request")
+public class HealthCheckREST {
+ private static final Logger LOG = LoggerFactory.getLogger(HealthCheckREST.class);
+
+ @Autowired(required = false)
+ HdfsConsumerManager hdfsConsumerManager;
+
+ /**
+ * Health check endpoint
+ */
+ @GET
+ @Produces("application/json")
+ public Response healthCheck() {
+ LOG.debug("==> HealthCheckREST.healthCheck()");
+ Response ret;
+ String jsonString;
+
+ try {
+ // Check if consumer manager is available and healthy
+ if (hdfsConsumerManager != null) {
+ Map<String, Object> resp = new HashMap<>();
+ resp.put("status", "UP");
+ resp.put("service", "audit-consumer-hdfs");
+ resp.put("timestamp", System.currentTimeMillis());
+ jsonString = buildResponse(resp);
+ ret = Response.ok()
+ .entity(jsonString)
+ .build();
+ } else {
+ Map<String, Object> resp = new HashMap<>();
+ resp.put("status", "DOWN");
+ resp.put("service", "audit-consumer-hdfs");
+ resp.put("reason", "HdfsConsumerManager not available");
+ resp.put("timestamp", System.currentTimeMillis());
+ jsonString = buildResponse(resp);
+ ret = Response.status(Response.Status.SERVICE_UNAVAILABLE)
+ .entity(jsonString)
+ .build();
+ }
+ } catch (Exception e) {
+ LOG.error("Health check failed", e);
+ Map<String, Object> resp = new HashMap<>();
+ resp.put("status", "DOWN");
+ resp.put("service", "audit-consumer-hdfs");
+ resp.put("reason", e.getMessage());
+ resp.put("timestamp", System.currentTimeMillis());
+ jsonString = buildResponse(resp);
+ ret = Response.status(Response.Status.SERVICE_UNAVAILABLE)
+ .entity(jsonString)
+ .build();
+ }
+
+ LOG.debug("<== HealthCheckREST.healthCheck(): {}", ret);
+
+ return ret;
+ }
+
+ private String buildResponse(Map<String, Object> respMap) {
+ String ret;
+
+ try {
+ ObjectMapper objectMapper = new ObjectMapper();
+ ret = objectMapper.writeValueAsString(respMap);
+ } catch (Exception e) {
+ ret = "Error: " + e.getMessage();
+ }
+
+ return ret;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java
new file mode 100644
index 000000000..6c38a3ac9
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/java/org/apache/ranger/audit/server/HdfsConsumerConfig.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.server;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Loads HDFS consumer-specific configuration files including Hadoop configs.
+ */
+public class HdfsConsumerConfig extends AuditConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(HdfsConsumerConfig.class);
+ private static final String CONFIG_FILE_PATH = "conf/ranger-audit-consumer-hdfs-site.xml";
+ private static final String CORE_SITE_FILE_PATH = "conf/core-site.xml";
+ private static final String HDFS_SITE_FILE_PATH = "conf/hdfs-site.xml";
+ private static volatile HdfsConsumerConfig sInstance;
+
+ private HdfsConsumerConfig() {
+ super();
+ addHdfsConsumerResources();
+ }
+
+ public static HdfsConsumerConfig getInstance() {
+ HdfsConsumerConfig ret = HdfsConsumerConfig.sInstance;
+
+ if (ret == null) {
+ synchronized (HdfsConsumerConfig.class) {
+ ret = HdfsConsumerConfig.sInstance;
+
+ if (ret == null) {
+ ret = new HdfsConsumerConfig();
+ HdfsConsumerConfig.sInstance = ret;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ private boolean addHdfsConsumerResources() {
+ LOG.debug("==> HdfsConsumerConfig.addHdfsConsumerResources()");
+
+ boolean ret = true;
+
+ if (!addAuditResource(CORE_SITE_FILE_PATH, false)) {
+ LOG.warn("Could not load required configuration: {}", CORE_SITE_FILE_PATH);
+ ret = false;
+ }
+
+ if (!addAuditResource(HDFS_SITE_FILE_PATH, true)) {
+ LOG.error("Could not load required configuration: {}", HDFS_SITE_FILE_PATH);
+ ret = false;
+ }
+
+ if (!addAuditResource(CONFIG_FILE_PATH, true)) {
+ LOG.error("Could not load required configuration: {}", CONFIG_FILE_PATH);
+ ret = false;
+ }
+
+ LOG.debug("<== HdfsConsumerConfig.addHdfsConsumerResources(), result={}", ret);
+
+ return ret;
+ }
+
+ /**
+ * Get Hadoop configuration properties (from core-site.xml and hdfs-site.xml) with a specific prefix.
+ * @param prefix The prefix to add to each property name ("xasecure.audit.destination.hdfs.config.")
+ * @return Properties from core-site.xml and hdfs-site.xml with the specified prefix
+ */
+ public java.util.Properties getHadoopPropertiesWithPrefix(String prefix) {
+ LOG.debug("==> HdfsConsumerConfig.getHadoopPropertiesWithPrefix(prefix={})", prefix);
+
+ java.util.Properties prefixedProps = new java.util.Properties();
+ int propsAdded = 0;
+
+ try {
+ // Load core-site.xml separately to get pure Hadoop security properties
+ org.apache.hadoop.conf.Configuration coreSite = new org.apache.hadoop.conf.Configuration(false);
+ coreSite.addResource(CORE_SITE_FILE_PATH);
+
+ for (java.util.Map.Entry<String, String> entry : coreSite) {
+ String propName = entry.getKey();
+ String propValue = entry.getValue();
+
+ if (propValue != null && !propValue.trim().isEmpty()) {
+ prefixedProps.setProperty(prefix + propName, propValue);
+ LOG.trace("Added from core-site.xml: {} = {}", propName, propValue);
+ propsAdded++;
+ }
+ }
+
+ // Load hdfs-site.xml separately to get pure HDFS client properties
+ org.apache.hadoop.conf.Configuration hdfsSite = new org.apache.hadoop.conf.Configuration(false);
+ hdfsSite.addResource(HDFS_SITE_FILE_PATH);
+
+ for (java.util.Map.Entry<String, String> entry : hdfsSite) {
+ String propName = entry.getKey();
+ String propValue = entry.getValue();
+
+ if (propValue != null && !propValue.trim().isEmpty()) {
+ prefixedProps.setProperty(prefix + propName, propValue);
+ LOG.trace("Added from hdfs-site.xml: {} = {}", propName, propValue);
+ propsAdded++;
+ }
+ }
+
+ LOG.debug("<== HdfsConsumerConfig.getHadoopPropertiesWithPrefix(): Added {} Hadoop properties with prefix '{}'", propsAdded, prefix);
+ } catch (Exception e) {
+ LOG.error("Failed to load Hadoop properties from {} and {}", CORE_SITE_FILE_PATH, HDFS_SITE_FILE_PATH, e);
+ }
+
+ return prefixedProps;
+ }
+
+ /**
+ * Get core-site.xml Configuration for UGI initialization.
+ * @return Configuration loaded from core-site.xml
+ */
+ public org.apache.hadoop.conf.Configuration getCoreSiteConfiguration() {
+ LOG.debug("==> HdfsConsumerConfig.getCoreSiteConfiguration()");
+
+ org.apache.hadoop.conf.Configuration coreSite = new org.apache.hadoop.conf.Configuration(false);
+ coreSite.addResource(CORE_SITE_FILE_PATH);
+
+ LOG.debug("<== HdfsConsumerConfig.getCoreSiteConfiguration(): authentication={}", coreSite.get("hadoop.security.authentication"));
+
+ return coreSite;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml
new file mode 100644
index 000000000..3e8bc00a1
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/core-site.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+ <!--
+ Hadoop security settings for Ranger Audit HDFS Consumer Service
+ This service authenticates as the rangerauditserver principal
+ and writes all audits to HDFS as that user.
+ -->
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ <description>Authentication method for Hadoop services: simple or kerberos</description>
+ </property>
+
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[1:$1@$0](rangerauditserver@EXAMPLE\.COM)s/.*/rangerauditserver/
+ RULE:[1:$1/$2@$0](rangerauditserver/ranger-audit-consumer-hdfs.rangernw@EXAMPLE\.COM)s/.*/rangerauditserver/
+ DEFAULT
+ </value>
+ <description>
+ Kerberos principal to local user name mapping rules for Ranger Audit HDFS Consumer.
+ Note: rangerauditserver/hostname@REALM and rangerauditserver@REALM both map to 'ranger' OS user
+ </description>
+ </property>
+</configuration>
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml
new file mode 100644
index 000000000..26e604dcf
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/hdfs-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+ <!--
+ HDFS Configuration for Ranger Audit HDFS Consumer Service
+ These settings are required for Kerberos-authenticated HDFS writes.
+ -->
+
+ <!-- NameNode Kerberos Principal -->
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>nn/_HOST@EXAMPLE.COM</value>
+ <description>
+ HDFS NameNode Kerberos principal. Required for Kerberos RPC authentication to NameNode.
+ Common formats:
+ - hdfs/_HOST@REALM
+ - nn/_HOST@REALM
+ - hdfs/namenode.fqdn@REALM (fully qualified, for specific NameNode)
+ </description>
+ </property>
+
+ <!-- Optional: Additional HDFS Client Settings -->
+ <property>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>false</value>
+ <description>
+ Whether clients should use the datanode hostname when connecting.
+ Set to true if using hostname-based Kerberos principals.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.failover.proxy.provider</name>
+ <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+ <description>
+ Failover proxy provider for HA NameNode clusters.
+ Adjust if your cluster uses a different HA configuration.
+ </description>
+ </property>
+</configuration>
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml
new file mode 100644
index 000000000..e8a08e8ea
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/logback.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <!-- System properties for log directory and file -->
+ <property name="LOG_DIR" value="${audit.consumer.hdfs.log.dir:-./logs}" />
+ <property name="LOG_FILE" value="${audit.consumer.hdfs.log.file:-consumer-hdfs.log}" />
+
+ <!-- Console Appender -->
+ <appender name="LogToConsole" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Rolling File Appender -->
+ <appender name="LogToRollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${LOG_DIR}/${LOG_FILE}</file>
+ <encoder>
+ <pattern>%d %p %c{1} [%t] %m%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${LOG_DIR}/${LOG_FILE}-%d{MM-dd-yyyy}-%i.log.gz</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>200MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <maxHistory>10</maxHistory>
+ </rollingPolicy>
+ </appender>
+
+ <!-- Ranger Audit Logger -->
+ <logger name="org.apache.ranger.audit" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Spring Logger -->
+ <logger name="org.springframework" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Kafka Logger (reduce verbosity) -->
+ <logger name="org.apache.kafka" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Hadoop Logger (reduce verbosity) -->
+ <logger name="org.apache.hadoop" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Root Logger -->
+ <root level="INFO">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </root>
+</configuration>
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml
new file mode 100644
index 000000000..0cfb6688b
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/resources/conf/ranger-audit-consumer-hdfs-site.xml
@@ -0,0 +1,156 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+ <!-- HDFS CONSUMER SERVICE CONFIGURATION -->
+ <!-- Server Configuration (Health Check Endpoint) -->
+ <property>
+ <name>log.dir</name>
+ <value>${audit.consumer.hdfs.log.dir}</value>
+ <description>Log directory for HDFS consumer service</description>
+ </property>
+
+ <property>
+ <name>ranger.audit.consumer.hdfs.service.webapp.dir</name>
+ <value>webapp/ranger-audit-consumer-hdfs</value>
+ <description>Path to the extracted webapp directory</description>
+ </property>
+
+ <property>
+ <name>ranger.audit.consumer.hdfs.service.contextName</name>
+ <value>/</value>
+ </property>
+
+ <property>
+ <name>ranger.audit.consumer.hdfs.service.host</name>
+ <value>ranger-audit-consumer-hdfs.rangernw</value>
+ <description>Service hostname</description>
+ </property>
+
+ <property>
+ <name>ranger.audit.consumer.hdfs.service.http.port</name>
+ <value>7092</value>
+ <description>Health check endpoint port (different from Solr: 7091)</description>
+ </property>
+
+ <!-- Used for KERBEROS authentication into HDFS -->
+ <property>
+ <name>ranger.audit.service.authentication.method</name>
+ <value>KERBEROS</value>
+ </property>
+
+ <property>
+ <name>ranger.audit.service.host</name>
+ <value>ranger-audit-consumer-hdfs.rangernw</value>
+ <description>Hostname for resolving _HOST in Kerberos principal</description>
+ </property>
+
+ <property>
+ <name>ranger.audit.service.kerberos.principal</name>
+ <value>rangerauditserver/_HOST@EXAMPLE.COM</value>
+ <description>Principal for Kafka consumer and HDFS writes</description>
+ </property>
+
+ <property>
+ <name>ranger.audit.service.kerberos.keytab</name>
+ <value>/etc/keytabs/rangerauditserver.keytab</value>
+ </property>
+
+ <!-- KAFKA CONSUMER CONFIGURATION -->
+ <property>
+ <name>xasecure.audit.destination.kafka.bootstrap.servers</name>
+ <value>ranger-kafka:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.kafka.topic.name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.kafka.security.protocol</name>
+ <value>SASL_PLAINTEXT</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.kafka.sasl.mechanism</name>
+ <value>GSSAPI</value>
+ </property>
+
+ <!-- HDFS Consumer Threading Configuration -->
+ <property>
+ <name>xasecure.audit.destination.kafka.consumer.thread.count</name>
+ <value>3</value>
+ <description>Number of HDFS consumer worker threads</description>
+ </property>
+
+ <!-- Offset Management -->
+ <property>
+ <name>xasecure.audit.destination.kafka.consumer.offset.commit.strategy</name>
+ <value>batch</value>
+ <description>batch or manual</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.kafka.consumer.offset.commit.interval.ms</name>
+ <value>30000</value>
+ <description>Used only if strategy is 'manual'</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.kafka.consumer.max.poll.records</name>
+ <value>500</value>
+ <description>Maximum records per poll</description>
+ </property>
+
+ <!-- Consumer Classes (ONLY HDFS) -->
+ <property>
+ <name>xasecure.audit.destination.kafka.consumer.classes</name>
+ <value>org.apache.ranger.audit.consumer.kafka.AuditHDFSConsumer</value>
+ </property>
+
+ <!-- HDFS DESTINATION CONFIGURATION - MUST BE ENABLED -->
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <value>true</value>
+ <description>MUST be true for HDFS consumer to work</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <value>hdfs://ranger-hadoop:9000/ranger/audit</value>
+ <description>
+ Namenode host for HDFS audit destination.
+ example: hdfs://hdfs-namenode:8020/ranger/audit
+ In Docker, use the service name of the namenode: hdfs://ranger-hadoop:9000/ranger/audit
+ </description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
+ <value>/var/log/audit-consumer-hdfs/spool</value>
+ </property>
+
+ <!-- HDFS File Configuration -->
+ <property>
+ <name>xasecure.audit.destination.hdfs.batch.filequeue.filetype</name>
+ <value>json</value>
+ <description>File format: json or orc</description>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.hdfs.file.rollover.sec</name>
+ <value>86400</value>
+ <description>File rollover time in seconds (default: 24 hours)</description>
+ </property>
+</configuration>
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml
new file mode 100644
index 000000000..5a7696647
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/applicationContext.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-4.3.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-4.3.xsd">
+
+ <!-- Component scan for HDFS consumer -->
+ <context:component-scan base-package="org.apache.ranger.audit.consumer" />
+ <context:component-scan base-package="org.apache.ranger.audit.rest" />
+
+ <bean class="org.springframework.beans.factory.config.CustomScopeConfigurer">
+ <property name="scopes">
+ <map>
+ <entry key="request">
+ <bean class="org.springframework.web.context.request.RequestScope" />
+ </entry>
+ </map>
+ </property>
+ </bean>
+</beans>
diff --git a/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 000000000..995d91281
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-hdfs/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
+ <display-name>Apache Ranger - Audit Consumer HDFS Service</display-name>
+ <description>Apache Ranger - Audit Consumer HDFS Service</description>
+
+ <context-param>
+ <param-name>contextConfigLocation</param-name>
+ <param-value>
+ WEB-INF/applicationContext.xml
+ </param-value>
+ </context-param>
+
+ <listener>
+ <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
+ </listener>
+
+ <listener>
+ <listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>Health Check Service</servlet-name>
+ <servlet-class>com.sun.jersey.spi.spring.container.servlet.SpringServlet</servlet-class>
+
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.ranger.audit.rest</param-value>
+ </init-param>
+
+ <init-param>
+ <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>
+ <param-value>true</param-value>
+ </init-param>
+
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>Health Check Service</servlet-name>
+ <url-pattern>/api/*</url-pattern>
+ </servlet-mapping>
+
+</web-app>
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/pom.xml b/ranger-audit-server/ranger-audit-consumer-solr/pom.xml
new file mode 100644
index 000000000..1e5db3f1a
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/pom.xml
@@ -0,0 +1,336 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-server</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-consumer-solr</artifactId>
+ <packaging>war</packaging>
+ <name>Ranger Audit Consumer Solr</name>
+ <description>Kafka consumer service for indexing audits into Solr/OpenSearch</description>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+
+ <!-- Jackson -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ </dependency>
+
+ <!-- Jersey -->
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ <version>${jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-spring</artifactId>
+ <version>${jersey-spring.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax.servlet.version}</version>
+ </dependency>
+
+ <!-- Utilities -->
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+
+ <!-- HTTP Components -->
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ <version>${httpcomponents.httpasyncclient.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.httpclient.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-nio</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+
+ <!-- Kafka Consumer -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+
+ <!-- Solr/Lucene Dependencies -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-grouping</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-highlighter</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-join</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-misc</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queries</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-sandbox</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial-extras</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial3d</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-suggest</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <!-- Internal Dependency -->
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Ranger Dependencies -->
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-solr</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-annotations-api</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <!-- Embedded Tomcat -->
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-el</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-jasper</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+
+ <!-- Spring Framework-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <filtering>true</filtering>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh b/ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh
new file mode 100755
index 000000000..88f64e423
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/scripts/start-consumer-solr.sh
@@ -0,0 +1,163 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ========================================
+# Ranger Audit Consumer Solr Start Script
+# ========================================
+# This script starts the Solr consumer service
+# The service consumes audit events from Kafka and indexes them to Solr
+
+set -e
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SERVICE_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Default directories - can be overridden by environment variables
+AUDIT_CONSUMER_HOME_DIR="${AUDIT_CONSUMER_HOME_DIR:-${SERVICE_DIR}/target}"
+AUDIT_CONSUMER_CONF_DIR="${AUDIT_CONSUMER_CONF_DIR:-${SERVICE_DIR}/src/main/resources/conf}"
+AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}"
+
+# Create log directory if it doesn't exist
+mkdir -p "${AUDIT_CONSUMER_LOG_DIR}"
+
+echo "=========================================="
+echo "Starting Ranger Audit Consumer - Solr"
+echo "=========================================="
+echo "Service Directory: ${SERVICE_DIR}"
+echo "Home Directory: ${AUDIT_CONSUMER_HOME_DIR}"
+echo "Config Directory: ${AUDIT_CONSUMER_CONF_DIR}"
+echo "Log Directory: ${AUDIT_CONSUMER_LOG_DIR}"
+echo "=========================================="
+
+# Check if Java is available
+if [ -z "$JAVA_HOME" ]; then
+ JAVA_CMD=$(which java 2>/dev/null || true)
+ if [ -z "$JAVA_CMD" ]; then
+ echo "[ERROR] JAVA_HOME is not set and java is not in PATH"
+ exit 1
+ fi
+ JAVA_HOME=$(dirname $(dirname $(readlink -f "$JAVA_CMD")))
+ echo "[INFO] JAVA_HOME not set, detected: ${JAVA_HOME}"
+fi
+
+export JAVA_HOME
+export PATH=$JAVA_HOME/bin:$PATH
+
+echo "[INFO] Java version:"
+java -version
+
+# Set heap size (default: 512MB to 2GB)
+AUDIT_CONSUMER_HEAP="${AUDIT_CONSUMER_HEAP:--Xms512m -Xmx2g}"
+
+# Set JVM options
+if [ -z "$AUDIT_CONSUMER_OPTS" ]; then
+ AUDIT_CONSUMER_OPTS="-Dlogback.configurationFile=${AUDIT_CONSUMER_CONF_DIR}/logback.xml"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.dir=${AUDIT_CONSUMER_LOG_DIR}"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Daudit.consumer.solr.log.file=ranger-audit-consumer-solr.log"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.net.preferIPv4Stack=true -server"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:+UseG1GC -XX:MaxGCPauseMillis=200"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:InitiatingHeapOccupancyPercent=35"
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -XX:ConcGCThreads=4 -XX:ParallelGCThreads=8"
+fi
+
+# Add Kerberos configuration if needed
+if [ "${KERBEROS_ENABLED}" == "true" ]; then
+ AUDIT_CONSUMER_OPTS="${AUDIT_CONSUMER_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf"
+ echo "[INFO] Kerberos is enabled"
+fi
+
+export AUDIT_CONSUMER_OPTS
+export AUDIT_CONSUMER_LOG_DIR
+
+echo "[INFO] JAVA_HOME: ${JAVA_HOME}"
+echo "[INFO] HEAP: ${AUDIT_CONSUMER_HEAP}"
+echo "[INFO] JVM_OPTS: ${AUDIT_CONSUMER_OPTS}"
+
+# Find the WAR file
+WAR_FILE=$(find "${AUDIT_CONSUMER_HOME_DIR}" -name "ranger-audit-consumer-solr*.war" | head -1)
+
+if [ -z "$WAR_FILE" ] || [ ! -f "$WAR_FILE" ]; then
+ echo "[ERROR] WAR file not found in ${AUDIT_CONSUMER_HOME_DIR}"
+ echo "[ERROR] Please build the project first using: mvn clean package"
+ exit 1
+fi
+
+echo "[INFO] Using WAR file: ${WAR_FILE}"
+
+# Extract WAR if not already extracted
+WEBAPP_DIR="${AUDIT_CONSUMER_HOME_DIR}/webapp/ranger-audit-consumer-solr"
+if [ ! -d "${WEBAPP_DIR}" ]; then
+ echo "[INFO] Extracting WAR file..."
+ mkdir -p "${WEBAPP_DIR}"
+ cd "${WEBAPP_DIR}"
+ jar xf "${WAR_FILE}"
+ cd - > /dev/null
+fi
+
+# Build classpath
+RANGER_CLASSPATH="${WEBAPP_DIR}/WEB-INF/classes"
+for jar in "${WEBAPP_DIR}"/WEB-INF/lib/*.jar; do
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+done
+
+# Add libext directory if it exists
+if [ -d "${AUDIT_CONSUMER_HOME_DIR}/libext" ]; then
+ for jar in "${AUDIT_CONSUMER_HOME_DIR}"/libext/*.jar; do
+ if [ -f "${jar}" ]; then
+ RANGER_CLASSPATH="${RANGER_CLASSPATH}:${jar}"
+ fi
+ done
+fi
+
+export RANGER_CLASSPATH
+
+# Check if already running
+PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.pid"
+if [ -f "${PID_FILE}" ]; then
+ OLD_PID=$(cat "${PID_FILE}")
+ if kill -0 "$OLD_PID" 2>/dev/null; then
+ echo "[WARNING] Ranger Audit Consumer - Solr is already running (PID: ${OLD_PID})"
+ echo "[INFO] Use stop-consumer-solr.sh to stop it first"
+ exit 1
+ else
+ echo "[INFO] Removing stale PID file"
+ rm -f "${PID_FILE}"
+ fi
+fi
+
+# Start the service
+echo "[INFO] Starting Ranger Audit Consumer - Solr..."
+nohup java ${AUDIT_CONSUMER_HEAP} ${AUDIT_CONSUMER_OPTS} \
+ -Daudit.config=${AUDIT_CONSUMER_CONF_DIR}/ranger-audit-consumer-solr-site.xml \
+ -Dranger.audit.consumer.webapp.dir="${WAR_FILE}" \
+ -cp "${RANGER_CLASSPATH}" \
+ org.apache.ranger.audit.consumer.SolrConsumerApplication \
+ >> "${AUDIT_CONSUMER_LOG_DIR}/catalina.out" 2>&1 &
+
+PID=$!
+echo $PID > "${PID_FILE}"
+
+echo "[INFO] ✓ Ranger Audit Consumer - Solr started successfully"
+echo "[INFO] PID: ${PID}"
+echo "[INFO] Log file: ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.log"
+echo "[INFO] Catalina out: ${AUDIT_CONSUMER_LOG_DIR}/catalina.out"
+echo "[INFO] Health check: http://localhost:7091/api/health"
+echo ""
+echo "To monitor logs: tail -f ${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.log"
+echo "To stop service: ${SCRIPT_DIR}/stop-consumer-solr.sh"
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh b/ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh
new file mode 100755
index 000000000..8e2049ac4
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/scripts/stop-consumer-solr.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ========================================
+# Ranger Audit Consumer Solr Stop Script
+# ========================================
+
+set -e
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SERVICE_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Default log directory
+AUDIT_CONSUMER_LOG_DIR="${AUDIT_CONSUMER_LOG_DIR:-${SERVICE_DIR}/logs}"
+PID_FILE="${AUDIT_CONSUMER_LOG_DIR}/ranger-audit-consumer-solr.pid"
+
+echo "=========================================="
+echo "Stopping Ranger Audit Consumer - Solr"
+echo "=========================================="
+
+if [ ! -f "${PID_FILE}" ]; then
+ echo "[WARNING] PID file not found: ${PID_FILE}"
+ echo "[INFO] Service may not be running"
+ exit 0
+fi
+
+PID=$(cat "${PID_FILE}")
+
+if ! kill -0 "$PID" 2>/dev/null; then
+ echo "[WARNING] Process ${PID} is not running"
+ echo "[INFO] Removing stale PID file"
+ rm -f "${PID_FILE}"
+ exit 0
+fi
+
+echo "[INFO] Stopping process ${PID}..."
+kill "$PID"
+
+# Wait for process to stop (max 30 seconds)
+TIMEOUT=30
+COUNT=0
+while kill -0 "$PID" 2>/dev/null && [ $COUNT -lt $TIMEOUT ]; do
+ sleep 1
+ COUNT=$((COUNT + 1))
+ echo -n "."
+done
+echo ""
+
+if kill -0 "$PID" 2>/dev/null; then
+ echo "[WARNING] Process did not stop gracefully, forcing shutdown..."
+ kill -9 "$PID"
+ sleep 2
+fi
+
+rm -f "${PID_FILE}"
+echo "[INFO] ✓ Ranger Audit Consumer - Solr stopped successfully"
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java
new file mode 100644
index 000000000..601f37b47
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerApplication.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ranger.audit.server.EmbeddedServer;
+import org.apache.ranger.audit.server.SolrConsumerConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Main application class for Audit Consumer Solr Service.
+ * This service consumes audit events from Kafka and indexes them into Solr
+ * It uses embedded Tomcat server for lifecycle management and health check endpoints.
+ */
+public class SolrConsumerApplication {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrConsumerApplication.class);
+
+ private static final String APP_NAME = "ranger-audit-consumer-solr";
+ private static final String CONFIG_PREFIX = "ranger.audit.consumer.solr.service.";
+
+ private SolrConsumerApplication() {
+ }
+
+ public static void main(String[] args) {
+ LOG.info("==========================================================================");
+ LOG.info("==> Starting Ranger Audit Consumer Solr Service");
+ LOG.info("==========================================================================");
+
+ try {
+ // Load configuration
+ Configuration config = SolrConsumerConfig.getInstance();
+
+ LOG.info("Configuration loaded successfully");
+
+ // Create and start embedded server
+ // The server will load Spring context which initializes SolrConsumerManager
+ // which then creates and starts the Solr consumer threads
+ EmbeddedServer server = new EmbeddedServer(config, APP_NAME, CONFIG_PREFIX);
+ server.start();
+
+ LOG.info("==> Ranger Audit Consumer Solr Service Started Successfully");
+ } catch (Exception e) {
+ LOG.error("Failed to start Ranger Audit Consumer Solr Service", e);
+ System.exit(1);
+ }
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java
new file mode 100644
index 000000000..2024f2ddc
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/SolrConsumerManager.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer;
+
+import org.apache.ranger.audit.consumer.kafka.AuditConsumer;
+import org.apache.ranger.audit.consumer.kafka.AuditConsumerRegistry;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.server.SolrConsumerConfig;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Spring component that manages the lifecycle of Solr consumer threads.
+ * This manager:
+ * - Initializes the consumer registry
+ * - Creates Solr consumer instances
+ * - Starts consumer threads
+ * - Handles graceful shutdown
+ */
+@Component
+public class SolrConsumerManager {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrConsumerManager.class);
+
+ private final AuditConsumerRegistry consumerRegistry = AuditConsumerRegistry.getInstance();
+ private final List<AuditConsumer> consumers = new ArrayList<>();
+ private final List<Thread> consumerThreads = new ArrayList<>();
+
+ @PostConstruct
+ public void init() {
+ LOG.info("==> SolrConsumerManager.init()");
+
+ try {
+ SolrConsumerConfig config = SolrConsumerConfig.getInstance();
+ Properties props = config.getProperties();
+
+ if (props == null) {
+ LOG.error("Configuration properties are null");
+ throw new RuntimeException("Failed to load configuration");
+ }
+
+ // Initialize and register Solr Consumer
+ initializeConsumerClasses(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX);
+
+ // Create consumers from registry
+ List<AuditConsumer> createdConsumers = consumerRegistry.createConsumers(props, AuditServerConstants.PROP_KAFKA_PROP_PREFIX);
+ consumers.addAll(createdConsumers);
+
+ if (consumers.isEmpty()) {
+ LOG.warn("No consumers were created! Verify that xasecure.audit.destination.solr=true");
+ } else {
+ LOG.info("Created {} Solr consumer(s)", consumers.size());
+
+ // Start consumer threads
+ startConsumers();
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to initialize SolrConsumerManager", e);
+ throw new RuntimeException("Failed to initialize SolrConsumerManager", e);
+ }
+
+ LOG.info("<== SolrConsumerManager.init() - {} consumer thread(s) started", consumerThreads.size());
+ }
+
+ private void initializeConsumerClasses(Properties props, String propPrefix) {
+ LOG.info("==> SolrConsumerManager.initializeConsumerClasses()");
+
+ // Get consumer classes from configuration
+ String clsStr = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_CLASSES,
+ "org.apache.ranger.audit.consumer.kafka.AuditSolrConsumer");
+
+ String[] solrConsumerClasses = clsStr.split(",");
+
+ LOG.info("Initializing {} consumer class(es)", solrConsumerClasses.length);
+
+ for (String solrConsumerClassName : solrConsumerClasses) {
+ solrConsumerClassName = solrConsumerClassName.trim();
+
+ if (solrConsumerClassName.isEmpty()) {
+ continue;
+ }
+
+ try {
+ Class<?> consumerClass = Class.forName(solrConsumerClassName);
+ LOG.info("Successfully initialized consumer class: {}", consumerClass.getName());
+ } catch (ClassNotFoundException e) {
+ LOG.error("Consumer class not found: {}. Ensure the class is on the classpath.", solrConsumerClassName, e);
+ } catch (Exception e) {
+ LOG.error("Error initializing consumer class: {}", solrConsumerClassName, e);
+ }
+ }
+
+ LOG.info("Registered consumer factories: {}", consumerRegistry.getRegisteredDestinationTypes());
+ LOG.info("<== SolrConsumerManager.initializeConsumerClasses()");
+ }
+
+ /**
+ * Start all consumer threads
+ */
+ private void startConsumers() {
+ LOG.info("==> SolrConsumerManager.startConsumers()");
+
+ logSolrConsumerStartup();
+
+ for (AuditConsumer consumer : consumers) {
+ try {
+ String consumerName = consumer.getClass().getSimpleName();
+ Thread consumerThread = new Thread(consumer, consumerName);
+ consumerThread.setDaemon(true);
+ consumerThread.start();
+ consumerThreads.add(consumerThread);
+
+ LOG.info("Started {} thread [Thread-ID: {}, Thread-Name: '{}']",
+ consumerName, consumerThread.getId(), consumerThread.getName());
+ } catch (Exception e) {
+ LOG.error("Error starting consumer: {}", consumer.getClass().getSimpleName(), e);
+ }
+ }
+
+ LOG.info("<== SolrConsumerManager.startConsumers() - {} thread(s) started", consumerThreads.size());
+ }
+
+ /**
+ * Log startup banner with consumer information
+ */
+ private void logSolrConsumerStartup() {
+ LOG.info("################## SOLR CONSUMER SERVICE STARTUP ######################");
+
+ if (consumers.isEmpty()) {
+ LOG.warn("WARNING: No Solr consumers are enabled!");
+ LOG.warn("Verify: xasecure.audit.destination.solr=true in configuration");
+ } else {
+ AuditServerLogFormatter.LogBuilder builder = AuditServerLogFormatter.builder("Solr Consumer Status");
+
+ for (AuditConsumer consumer : consumers) {
+ String consumerType = consumer.getClass().getSimpleName();
+ builder.add(consumerType, "ENABLED");
+ builder.add("Topic", consumer.getTopicName());
+ }
+
+ builder.logInfo(LOG);
+ LOG.info("Starting {} Solr consumer thread(s)...", consumers.size());
+ }
+ LOG.info("########################################################################");
+ }
+
+ @PreDestroy
+ public void shutdown() {
+ LOG.info("==> SolrConsumerManager.shutdown()");
+
+ // Shutdown all consumers
+ for (AuditConsumer consumer : consumers) {
+ try {
+ LOG.info("Shutting down consumer: {}", consumer.getClass().getSimpleName());
+ consumer.shutdown();
+ LOG.info("Consumer shutdown completed: {}", consumer.getClass().getSimpleName());
+ } catch (Exception e) {
+ LOG.error("Error shutting down consumer: {}", consumer.getClass().getSimpleName(), e);
+ }
+ }
+
+ // Wait for threads to terminate
+ for (Thread thread : consumerThreads) {
+ if (thread.isAlive()) {
+ try {
+ LOG.info("Waiting for thread to terminate: {}", thread.getName());
+ thread.join(10000); // Wait up to 10 seconds
+ if (thread.isAlive()) {
+ LOG.warn("Thread did not terminate within 10 seconds: {}", thread.getName());
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for thread to terminate: {}", thread.getName(), e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ consumers.clear();
+ consumerThreads.clear();
+ consumerRegistry.clearActiveConsumers();
+
+ LOG.info("<== SolrConsumerManager.shutdown() - All Solr consumers stopped");
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java
new file mode 100644
index 000000000..043231f57
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/consumer/kafka/AuditSolrConsumer.java
@@ -0,0 +1,457 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.consumer.kafka;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.ranger.audit.destination.SolrAuditDestination;
+import org.apache.ranger.audit.provider.AuditProviderFactory;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Solr consumer that writes audits into Solr index using rangerauditserver user
+ */
+public class AuditSolrConsumer extends AuditConsumerBase implements AuditConsumer {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditSolrConsumer.class);
+ private static final String RANGER_AUDIT_SOLR_CONSUMER_GROUP = AuditServerConstants.DEFAULT_RANGER_AUDIT_SOLR_CONSUMER_GROUP;
+
+ // Register AuditSolrConsumer factory in the audit consumer registry
+ static {
+ try {
+ AuditConsumerRegistry.getInstance().registerFactory(AuditServerConstants.PROP_SOLR_DEST_PREFIX, (props, propPrefix) -> new AuditSolrConsumer(props, propPrefix));
+ LOG.info("Registered Solr consumer factory with AuditConsumerRegistry");
+ } catch (Exception e) {
+ LOG.error("Failed to register Solr consumer factory", e);
+ }
+ }
+
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final Map<String, ConsumerWorker> consumerWorkers = new ConcurrentHashMap<>();
+ private ExecutorService consumerThreadPool;
+ private int consumerThreadCount = 1;
+ // Offset management configuration (batch or manual only supported)
+ private String offsetCommitStrategy = AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY;
+ private long offsetCommitInterval = AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS;
+
+ public SolrAuditDestination solrAuditDestination;
+ public Properties props;
+ public String propPrefix;
+
+ public AuditSolrConsumer(Properties props, String propPrefix) throws Exception {
+ super(props, propPrefix, RANGER_AUDIT_SOLR_CONSUMER_GROUP);
+ this.props = props;
+ this.propPrefix = propPrefix;
+ init(props, propPrefix);
+ }
+
+ @Override
+ public void init(Properties props, String propPrefix) throws Exception {
+ LOG.info("==> AuditSolrConsumer.init()");
+
+ consumer = getConsumer();
+ solrAuditDestination = new SolrAuditDestination();
+ solrAuditDestination.init(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + AuditServerConstants.PROP_SOLR_DEST_PREFIX);
+
+ // Initialize consumer configuration
+ initConsumerConfig(props, propPrefix);
+
+ LOG.info("<== AuditSolrConsumer.init()");
+ }
+
+ private void initConsumerConfig(Properties props, String propPrefix) {
+ LOG.info("==> AuditSolrConsumer.initConsumerConfig()");
+
+ // Get consumer thread count
+ this.consumerThreadCount = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONSUMER_THREAD_COUNT, 1);
+ LOG.info("Consumer thread count: {}", consumerThreadCount);
+
+ // Initialize offset management configuration
+ initializeOffsetManagement(props, propPrefix);
+
+ LOG.info("<== AuditSolrConsumer.initConsumerConfig()");
+ }
+
+ private void initializeOffsetManagement(Properties props, String propPrefix) {
+ LOG.info("==> AuditSolrConsumer.initializeOffsetManagement()");
+
+ this.offsetCommitStrategy = MiscUtil.getStringProperty(props,
+ propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_STRATEGY,
+ AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY);
+
+ // Get offset commit interval (only used for manual strategy)
+ this.offsetCommitInterval = MiscUtil.getLongProperty(props,
+ propPrefix + "." + AuditServerConstants.PROP_CONSUMER_OFFSET_COMMIT_INTERVAL,
+ AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS);
+
+ AuditServerLogFormatter.builder("AuditSolrConsumer Offset Management Configuration")
+ .add("Commit Strategy", offsetCommitStrategy)
+ .add("Commit Interval (ms)", offsetCommitInterval + " (used in manual mode only)")
+ .logInfo(LOG);
+
+ LOG.info("<== AuditSolrConsumer.initializeOffsetManagement()");
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (solrAuditDestination == null) {
+ init(this.props, this.propPrefix);
+ }
+
+ startMultithreadedConsumption();
+
+ // Keep main thread alive while consumer threads are running
+ while (running.get()) {
+ Thread.sleep(1000);
+ }
+ } catch (Throwable e) {
+ LOG.error("Error in AuditSolrConsumer", e);
+ } finally {
+ shutdown();
+ }
+ }
+
+ /**
+ * Start multithreaded consumption with generic workers for horizontal scaling.
+ * Creates configured number of generic workers that can process ANY partition assigned by Kafka.
+ * This enables true horizontal scaling across multiple audit-server instances.
+ */
+ private void startMultithreadedConsumption() {
+ LOG.info("==> AuditSolrConsumer.startMultithreadedConsumption()");
+
+ if (running.compareAndSet(false, true)) {
+ startConsumerWorkers();
+ }
+
+ LOG.info("<== AuditSolrConsumer.startMultithreadedConsumption()");
+ }
+
+ /**
+ * Start consumer workers for horizontal scaling.
+ * Each worker subscribes to the topic and Kafka automatically assigns partitions.
+ * Workers can process messages from ANY appId.
+ */
+ private void startConsumerWorkers() {
+ int workerCount = consumerThreadCount;
+
+ LOG.info("==> AuditSolrConsumer.startConsumerWorkers(): Creating {} generic workers for horizontal scaling", workerCount);
+ LOG.info("Each worker will subscribe to topic '{}' and process partitions assigned by Kafka", topicName);
+
+ // Create thread pool sized for generic workers
+ consumerThreadPool = Executors.newFixedThreadPool(workerCount);
+ LOG.info("Created thread pool with {} threads for scalable SOLR consumption", workerCount);
+
+ // Create generic workers (no appId pre-assignment)
+ for (int i = 0; i < workerCount; i++) {
+ String workerId = "solr-worker-" + i;
+ // Pass empty list for partitions (Kafka will assign dynamically)
+ ConsumerWorker worker = new ConsumerWorker(workerId, new ArrayList<>());
+ consumerWorkers.put(workerId, worker);
+ consumerThreadPool.submit(worker);
+
+ LOG.info("Started SOLR consumer worker '{}' - will process ANY appId assigned by Kafka", workerId);
+ }
+
+ LOG.info("<== AuditSolrConsumer.startConsumerWorkers(): All {} workers started in SUBSCRIBE mode", workerCount);
+ }
+
+ private class ConsumerWorker implements Runnable {
+ private final String workerId;
+ private final List<Integer> assignedPartitions;
+ private KafkaConsumer<String, String> workerConsumer;
+
+ // Offset management
+ private final Map<TopicPartition, OffsetAndMetadata> pendingOffsets = new HashMap<>();
+ private final AtomicLong lastCommitTime = new AtomicLong(System.currentTimeMillis());
+ private final AtomicInteger messagesProcessedSinceLastCommit = new AtomicInteger(0);
+
+ public ConsumerWorker(String workerId, List<Integer> assignedPartitions) {
+ this.workerId = workerId;
+ this.assignedPartitions = assignedPartitions;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // Create consumer for this worker with offset management configuration
+ Properties workerConsumerProps = new Properties();
+ workerConsumerProps.putAll(consumerProps);
+
+ // Configure offset management based on strategy
+ configureOffsetManagement(workerConsumerProps);
+
+ workerConsumer = new KafkaConsumer<>(workerConsumerProps);
+
+ // Create re-balance listener
+ AuditConsumerRebalanceListener reBalanceListener = new AuditConsumerRebalanceListener(
+ workerId,
+ AuditServerConstants.DESTINATION_SOLR,
+ topicName,
+ offsetCommitStrategy,
+ consumerGroupId,
+ workerConsumer,
+ pendingOffsets,
+ messagesProcessedSinceLastCommit,
+ lastCommitTime,
+ assignedPartitions);
+
+ // Subscribe to topic with re-balance listener and let kafka automatically assign partitions
+ workerConsumer.subscribe(Collections.singletonList(topicName), reBalanceListener);
+
+ LOG.info("[SOLR-CONSUMER] Worker '{}' subscribed successfully, waiting for partition assignment from Kafka", workerId);
+ long threadId = Thread.currentThread().getId();
+ String threadName = Thread.currentThread().getName();
+ LOG.info("[SOLR-CONSUMER-STARTUP] Worker '{}' [Thread-ID: {}, Thread-Name: '{}'] started | Topic: '{}' | Consumer-Group: {} | Mode: SUBSCRIBE",
+ workerId, threadId, threadName, topicName, consumerGroupId);
+
+ // Consume messages
+ while (running.get()) {
+ ConsumerRecords<String, String> records = workerConsumer.poll(Duration.ofMillis(100));
+
+ if (!records.isEmpty()) {
+ processRecordBatch(records);
+
+ // Handle offset committing based on strategy
+ handleOffsetCommitting();
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error in consumer worker '{}'", workerId, e);
+ } finally {
+ // Final offset commit before shutdown
+ commitPendingOffsets(true);
+
+ if (workerConsumer != null) {
+ try {
+ // Unsubscribe before closing
+ LOG.info("Worker '{}': Unsubscribing from topic", workerId);
+ workerConsumer.unsubscribe();
+ } catch (Exception e) {
+ LOG.warn("Worker '{}': Error during unsubscribe", workerId, e);
+ }
+
+ try {
+ LOG.info("Worker '{}': Closing consumer", workerId);
+ workerConsumer.close();
+ } catch (Exception e) {
+ LOG.error("Error closing consumer for worker '{}'", workerId, e);
+ }
+ }
+ LOG.info("Consumer worker '{}' stopped", workerId);
+ }
+ }
+
+ /**
+ * Configure offset management for this worker's consumer (batch or manual)
+ */
+ private void configureOffsetManagement(Properties consumerProps) {
+ // Always disable auto commit - only batch or manual strategies supported
+ consumerProps.put("enable.auto.commit", "false");
+ LOG.debug("Worker '{}' configured for manual offset commit with strategy: {}", workerId, offsetCommitStrategy);
+ }
+
+ /**
+ * Process a batch of records.
+ * In generic worker mode, processes messages from ANY appId.
+ * Uses batch processing to send all records to Solr in one request for efficiency.
+ */
+ private void processRecordBatch(ConsumerRecords<String, String> records) {
+ // Collect all audit messages for batch processing
+ List<String> auditBatch = new ArrayList<>();
+ List<ConsumerRecord<String, String>> recordList = new ArrayList<>();
+
+ for (ConsumerRecord<String, String> record : records) {
+ LOG.debug("Worker '{}' consumed: partition={}, key={}, offset={}",
+ workerId, record.partition(), record.key(), record.offset());
+
+ auditBatch.add(record.value());
+ recordList.add(record);
+ }
+
+ // Process entire batch at once
+ try {
+ if (!auditBatch.isEmpty()) {
+ processMessageBatch(auditBatch);
+
+ // Track offsets for all successfully processed messages
+ for (ConsumerRecord<String, String> record : recordList) {
+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+ pendingOffsets.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ messagesProcessedSinceLastCommit.incrementAndGet();
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error processing batch in worker '{}', batch size: {}",
+ workerId, auditBatch.size(), e);
+
+ // On batch error, track offsets up to the first message to avoid reprocessing
+ // (Note: This is a simplistic approach - could be enhanced to track last successful offset)
+ if (!recordList.isEmpty()) {
+ ConsumerRecord<String, String> firstRecord = recordList.get(0);
+ TopicPartition partition = new TopicPartition(firstRecord.topic(), firstRecord.partition());
+ pendingOffsets.put(partition, new OffsetAndMetadata(firstRecord.offset()));
+ }
+ }
+ }
+
+ /**
+ * Handle offset committing based on the configured strategy (batch or manual only)
+ */
+ private void handleOffsetCommitting() {
+ boolean shouldCommit = false;
+ long currentTime = System.currentTimeMillis();
+
+ if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_BATCH.equals(offsetCommitStrategy)) {
+ // Commit after processing each batch
+ shouldCommit = !pendingOffsets.isEmpty();
+ } else if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_MANUAL.equals(offsetCommitStrategy)) {
+ // Commit based on time interval
+ shouldCommit = (currentTime - lastCommitTime.get()) >= offsetCommitInterval && !pendingOffsets.isEmpty();
+ }
+
+ if (shouldCommit) {
+ commitPendingOffsets(false);
+ }
+ }
+
+ /**
+ * Commit pending offsets
+ */
+ private void commitPendingOffsets(boolean isShutdown) {
+ if (pendingOffsets.isEmpty()) {
+ return;
+ }
+
+ try {
+ workerConsumer.commitSync(pendingOffsets);
+
+ LOG.debug("Worker '{}' committed {} offsets, processed {} messages",
+ workerId, pendingOffsets.size(), messagesProcessedSinceLastCommit.get());
+
+ // Clear committed offsets
+ pendingOffsets.clear();
+ lastCommitTime.set(System.currentTimeMillis());
+ messagesProcessedSinceLastCommit.set(0);
+ } catch (Exception e) {
+ LOG.error("Error committing offsets in worker '{}': {}", workerId, pendingOffsets, e);
+
+ if (isShutdown) {
+ // During shutdown, retry to avoid loss of any offsets
+ try {
+ Thread.sleep(1000);
+ workerConsumer.commitSync(pendingOffsets);
+ LOG.info("Successfully committed offsets on retry during shutdown for worker '{}'", workerId);
+ } catch (Exception retryException) {
+ LOG.error("Failed to commit offsets even on retry during shutdown for worker '{}'", workerId, retryException);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public KafkaConsumer<String, String> getKafkaConsumer() {
+ return consumer;
+ }
+
+ @Override
+ public void processMessage(String audit) throws Exception {
+ if (solrAuditDestination != null) {
+ solrAuditDestination.logJSON(audit);
+ }
+ }
+
+ /**
+ * Process a batch of audit messages.
+ * This method leverages SolrAuditDestination's batch processing capability
+ * to send multiple audits to Solr in a single request, improving performance.
+ *
+ * @param audits Collection of audit messages in JSON format
+ * @throws Exception if batch processing fails
+ */
+ public void processMessageBatch(Collection<String> audits) throws Exception {
+ if (solrAuditDestination != null && audits != null && !audits.isEmpty()) {
+ solrAuditDestination.logJSON(audits);
+ }
+ }
+
+ @Override
+ public String getTopicName() {
+ return topicName;
+ }
+
+ @Override
+ public void shutdown() {
+ LOG.info("Shutting down AuditSolrConsumer...");
+
+ running.set(false);
+
+ // Shutdown consumer workers
+ if (consumerThreadPool != null) {
+ consumerThreadPool.shutdownNow();
+ try {
+ if (!consumerThreadPool.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) {
+ LOG.warn("Consumer thread pool did not terminate within 30 seconds");
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for consumer thread pool to terminate", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ // Close main consumer
+ if (consumer != null) {
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ LOG.error("Error closing main consumer", e);
+ }
+ }
+
+ consumerWorkers.clear();
+ LOG.info("AuditSolrConsumer shutdown complete");
+ }
+
+ public String getConsumerGroupId() {
+ return consumerGroupId;
+ }
+}
diff --git a/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java
new file mode 100644
index 000000000..3245c1447
--- /dev/null
+++ b/ranger-audit-server/ranger-audit-consumer-solr/src/main/java/org/apache/ranger/audit/rest/HealthCheckREST.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.rest;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ranger.audit.consumer.SolrConsumerManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Health check REST endpoint for Solr Consumer Service
+ */
+@Path("/health")
+@Component
+@Scope("request")
+public class HealthCheckREST {
+ private static final Logger LOG = LoggerFactory.getLogger(HealthCheckREST.class);
+
+ @Autowired(required = false)
+ SolrConsumerManager solrConsumerManager;
+
+ /**
+ * Health check endpoint
+ */
+ @GET
+ @Produces("application/json")
+ public Response healthCheck() {
+ LOG.debug("==> HealthCheckREST.healthCheck()");
+ Response ret;
+ String jsonString;
+
+ try {
+ // Check if consumer manager is available and healthy
... 5339 lines suppressed ...