You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ra...@apache.org on 2026/09/20 05:19:24 UTC
(ranger) branch backport-audit-pr1-server-2.10 updated (30a5e2751 -> 98fae3b0b)
This is an automated email from the ASF dual-hosted git repository.
ramackri pushed a change to branch backport-audit-pr1-server-2.10
in repository https://gitbox.apache.org/repos/asf/ranger.git
discard 30a5e2751 RANGER-5482/5520: Audit Server stack for ranger-2.10 (PR-1)
new 98fae3b0b RANGER-5482/5520: Audit Server stack for ranger-2.10 (PR-1)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (30a5e2751)
\
N -- N -- N refs/heads/backport-audit-pr1-server-2.10 (98fae3b0b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
(ranger) 01/01: RANGER-5482/5520: Audit Server stack for ranger-2.10 (PR-1)
Posted by ra...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ramackri pushed a commit to branch backport-audit-pr1-server-2.10
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 98fae3b0b6a79d43b938dfed63f0619546c02197
Author: ramk <ra...@cloudera.com>
AuthorDate: Sun Sep 20 10:21:33 2026 +0530
RANGER-5482/5520: Audit Server stack for ranger-2.10 (PR-1)
Add audit-server (ingestor + all dispatchers), agents-audit dest-os,
core Kerberos/utils, and distro audit assemblies. Excludes plugin
wiring, dest-auditserver, admin OpenSearch UI, and docker/CI (PR-2–4).
Co-authored-by: Cursor <cu...@cursor.com>
---
.gitignore | 6 +
.../apache/ranger/audit/model/AuthzAuditEvent.java | 1091 ++++++++++----------
.../apache/ranger/audit/provider/AuditHandler.java | 30 +-
.../audit/provider/AuditProviderFactory.java | 196 ++--
.../ranger/audit/provider/BaseAuditHandler.java | 938 ++++++++---------
.../ranger/audit/utils/AbstractKerberosUser.java | 155 ++-
.../audit/utils/AbstractRangerAuditWriter.java | 321 +++---
.../apache/ranger/audit/utils/KerberosAction.java | 47 +-
.../ranger/audit/utils/KerberosJAASConfigUser.java | 39 +-
.../ranger/audit/utils/RangerJSONAuditWriter.java | 181 ++--
agents-audit/dest-os/pom.xml | 81 ++
.../destination/OpenSearchAuditDestination.java | 323 ++++++
.../TestOpenSearchAuditDestination.java | 192 ++++
.../audit/destination/SolrAuditDestination.java | 898 ++++++++--------
agents-audit/pom.xml | 1 +
.../audit-common}/pom.xml | 52 +-
.../apache/ranger/audit/server/AuditConfig.java | 126 +++
.../ranger/audit/server/AuditServerConstants.java | 145 +++
.../ranger/audit/utils/AuditMessageQueueUtils.java | 396 +++++++
.../audit/utils/AuditServerLogFormatter.java | 101 ++
.../ranger/audit/utils/AuditServerUtils.java | 84 ++
.../audit/utils/AuditMessageQueueUtilsTest.java | 68 ++
.../audit-dispatcher/dispatcher-app/pom.xml | 93 ++
.../dispatcher/AuditDispatcherApplication.java | 106 ++
.../audit/rest/AuditDispatcherHealthREST.java | 117 +++
.../src/main/webapp/WEB-INF/applicationContext.xml | 37 +
.../dispatcher-app/src/main/webapp/WEB-INF/web.xml | 59 ++
.../audit-dispatcher/dispatcher-common/pom.xml | 146 +++
.../audit/dispatcher/AuditDispatcherLauncher.java | 107 ++
.../ranger/audit/dispatcher/EmbeddedServer.java | 784 ++++++++++++++
.../audit/dispatcher/kafka/AuditDispatcher.java | 56 +
.../dispatcher/kafka/AuditDispatcherBase.java | 358 +++++++
.../dispatcher/kafka/AuditDispatcherFactory.java | 38 +
.../kafka/AuditDispatcherRebalanceListener.java | 119 +++
.../dispatcher/kafka/AuditDispatcherTracker.java | 71 ++
.../audit-dispatcher/dispatcher-hdfs/pom.xml | 452 ++++++++
.../audit/dispatcher/HdfsDispatcherManager.java | 204 ++++
.../dispatcher/kafka/AuditHDFSDispatcher.java | 322 ++++++
.../audit/dispatcher/kafka/AuditRouterHDFS.java | 322 ++++++
.../src/main/resources/conf/core-site.xml | 39 +
.../src/main/resources/conf/hdfs-site.xml | 52 +
.../src/main/resources/conf/logback.xml | 74 ++
.../conf/ranger-audit-dispatcher-hdfs-site.xml | 186 ++++
.../audit-dispatcher/dispatcher-opensearch/pom.xml | 240 +++++
.../dispatcher/OpenSearchDispatcherManager.java | 239 +++++
.../kafka/AuditOpenSearchDispatcher.java | 155 +++
.../src/main/resources/conf/logback.xml | 55 +
.../ranger-audit-dispatcher-opensearch-site.xml | 228 ++++
.../TestOpenSearchDispatcherManager.java | 80 ++
.../kafka/TestAuditOpenSearchDispatcher.java | 98 ++
.../audit-dispatcher/dispatcher-solr/pom.xml | 440 ++++++++
.../audit/dispatcher/SolrDispatcherManager.java | 211 ++++
.../dispatcher/kafka/AuditSolrDispatcher.java | 194 ++++
.../src/main/resources/conf/logback.xml | 79 ++
.../conf/ranger-audit-dispatcher-solr-site.xml | 236 +++++
.../audit-dispatcher}/pom.xml | 34 +-
.../scripts/start-audit-dispatcher.sh | 161 +++
.../scripts/stop-audit-dispatcher.sh | 62 ++
audit-server/audit-ingestor/pom.xml | 433 ++++++++
.../audit-ingestor/scripts/start-audit-ingestor.sh | 162 +++
.../audit-ingestor/scripts/stop-audit-ingestor.sh | 72 ++
.../java/javax/ws/rs/core/NoContentException.java | 43 +
.../ranger/audit/producer/AuditDestinationMgr.java | 113 ++
.../audit/producer/kafka/AuditMessageQueue.java | 381 +++++++
.../audit/producer/kafka/AuditPartitioner.java | 187 ++++
.../ranger/audit/producer/kafka/AuditProducer.java | 317 ++++++
.../audit/producer/kafka/AuditRecoveryManager.java | 222 ++++
.../audit/producer/kafka/AuditRecoveryRetry.java | 362 +++++++
.../audit/producer/kafka/AuditRecoveryWriter.java | 319 ++++++
.../org/apache/ranger/audit/rest/AuditREST.java | 373 +++++++
.../ranger/audit/rest/RangerJsonProvider.java | 44 +
.../ranger/audit/security/AuditAuthEntryPoint.java | 52 +
.../audit/security/AuditDelegationTokenFilter.java | 314 ++++++
.../ranger/audit/security/AuditJwtAuthFilter.java | 196 ++++
.../ranger/audit/security/FilterChainWrapper.java | 159 +++
.../ranger/audit/security/NullServletContext.java | 265 +++++
.../audit/server/AuditServerApplication.java | 62 ++
.../ranger/audit/server/AuditServerConfig.java | 73 ++
.../apache/ranger/audit/server/EmbeddedServer.java | 781 ++++++++++++++
.../src/main/resources/conf/logback.xml | 71 ++
.../resources/conf/ranger-audit-ingestor-site.xml | 575 +++++++++++
.../src/main/webapp/WEB-INF/applicationContext.xml | 54 +
.../webapp/WEB-INF/security-applicationContext.xml | 63 ++
.../audit-ingestor/src/main/webapp/WEB-INF/web.xml | 72 ++
.../audit/producer/kafka/AuditProducerTest.java | 106 ++
audit-server/pom.xml | 480 +++++++++
audit-server/scripts/README.md | 463 +++++++++
audit-server/scripts/start-all-services.sh | 84 ++
audit-server/scripts/stop-all-services.sh | 66 ++
distro/pom.xml | 73 +-
distro/src/main/assembly/audit-dispatcher.xml | 158 +++
distro/src/main/assembly/audit-ingestor.xml | 83 ++
pom.xml | 51 +
93 files changed, 17170 insertions(+), 1884 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5bd45411b..92e70d43f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,9 @@ winpkg/target
.python-version
/security-admin/src/main/webapp/react-webapp/node_modules
**/target
+
+# Runtime logs and process files
+logs/
+*.log
+*.pid
+catalina.out
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
index 6e00bcef8..de91ba164 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
@@ -19,567 +19,574 @@
package org.apache.ranger.audit.model;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Set;
-
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.commons.lang3.StringUtils;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize
public class AuthzAuditEvent extends AuditEventBase {
- protected static String FIELD_SEPARATOR = ";";
+ protected static final int MAX_ACTION_FIELD_SIZE = 1800;
+ protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800;
+ protected static String FIELD_SEPARATOR = ";";
+
+ @JsonProperty("repoType")
+ protected int repositoryType;
+
+ @JsonProperty("repo")
+ protected String repositoryName;
- protected static final int MAX_ACTION_FIELD_SIZE = 1800;
- protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800;
+ @JsonProperty("reqUser")
+ protected String user;
- @JsonProperty("repoType")
- protected int repositoryType = 0;
+ @JsonProperty("evtTime")
+ protected Date eventTime = new Date();
- @JsonProperty("repo")
- protected String repositoryName = null;
+ @JsonProperty("access")
+ protected String accessType;
- @JsonProperty("reqUser")
- protected String user = null;
+ @JsonProperty("resource")
+ protected String resourcePath;
- @JsonProperty("evtTime")
- protected Date eventTime = new Date();
+ @JsonProperty("resType")
+ protected String resourceType;
- @JsonProperty("access")
- protected String accessType = null;
+ @JsonProperty("action")
+ protected String action;
- @JsonProperty("resource")
- protected String resourcePath = null;
+ @JsonProperty("result")
+ protected short accessResult; // 0 - DENIED; 1 - ALLOWED; HTTP return code
- @JsonProperty("resType")
- protected String resourceType = null;
+ @JsonProperty("agent")
+ protected String agentId;
- @JsonProperty("action")
- protected String action = null;
+ @JsonProperty("policy")
+ protected long policyId;
- @JsonProperty("result")
- protected short accessResult = 0; // 0 - DENIED; 1 - ALLOWED; HTTP return
- // code
+ @JsonProperty("reason")
+ protected String resultReason;
- @JsonProperty("agent")
- protected String agentId = null;
+ @JsonProperty("enforcer")
+ protected String aclEnforcer;
- @JsonProperty("policy")
- protected long policyId = 0;
+ @JsonProperty("sess")
+ protected String sessionId;
- @JsonProperty("reason")
- protected String resultReason = null;
+ @JsonProperty("cliType")
+ protected String clientType;
- @JsonProperty("enforcer")
- protected String aclEnforcer = null;
+ @JsonProperty("cliIP")
+ protected String clientIP;
+
+ @JsonProperty("reqData")
+ protected String requestData;
- @JsonProperty("sess")
- protected String sessionId = null;
-
- @JsonProperty("cliType")
- protected String clientType = null;
-
- @JsonProperty("cliIP")
- protected String clientIP = null;
-
- @JsonProperty("reqData")
- protected String requestData = null;
-
- @JsonProperty("agentHost")
- protected String agentHostname = null;
-
- @JsonProperty("logType")
- protected String logType = null;
-
- @JsonProperty("id")
- protected String eventId = null;
-
- /**
- * This to ensure order within a session. Order not guaranteed across
- * processes and hosts
- */
- @JsonProperty("seq_num")
- protected long seqNum = 0;
-
- @JsonProperty("event_count")
- protected long eventCount = 1;
-
- @JsonProperty("event_dur_ms")
- protected long eventDurationMS = 0;
-
- @JsonProperty("tags")
- protected Set<String> tags = new HashSet<>();
-
- @JsonProperty("datasets")
- protected Set<String> datasets = null;
-
- @JsonProperty("projects")
- protected Set<String> projects = null;
-
- @JsonProperty("additional_info")
- protected String additionalInfo;
-
- @JsonProperty("cluster_name")
- protected String clusterName;
-
- @JsonProperty("zone_name")
- protected String zoneName;
-
- @JsonProperty("policy_version")
- protected Long policyVersion;
-
- public AuthzAuditEvent() {
- super();
-
- this.repositoryType = 0;
- }
-
- public AuthzAuditEvent(int repositoryType, String repositoryName,
- String user, Date eventTime, String accessType,
- String resourcePath, String resourceType, String action,
- short accessResult, String agentId, long policyId,
- String resultReason, String aclEnforcer, String sessionId,
- String clientType, String clientIP, String requestData, String clusterName) {
- this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
- policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, null);
- }
-
- public AuthzAuditEvent(int repositoryType, String repositoryName,
- String user, Date eventTime, String accessType,
- String resourcePath, String resourceType, String action,
- short accessResult, String agentId, long policyId,
- String resultReason, String aclEnforcer, String sessionId,
- String clientType, String clientIP, String requestData, String clusterName, String zoneName) {
- this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
- policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, zoneName, null);
-
- }
-
- public AuthzAuditEvent(int repositoryType, String repositoryName,
- String user, Date eventTime, String accessType,
- String resourcePath, String resourceType, String action,
- short accessResult, String agentId, long policyId,
- String resultReason, String aclEnforcer, String sessionId,
- String clientType, String clientIP, String requestData, String clusterName, String zoneName, Long policyVersion) {
- this.repositoryType = repositoryType;
- this.repositoryName = repositoryName;
- this.user = user;
- this.eventTime = eventTime;
- this.accessType = accessType;
- this.resourcePath = resourcePath;
- this.resourceType = resourceType;
- this.action = action;
- this.accessResult = accessResult;
- this.agentId = agentId;
- this.policyId = policyId;
- this.resultReason = resultReason;
- this.aclEnforcer = aclEnforcer;
- this.sessionId = sessionId;
- this.clientType = clientType;
- this.clientIP = clientIP;
- this.requestData = requestData;
- this.clusterName = clusterName;
- this.zoneName = zoneName;
- this.policyVersion = policyVersion;
- }
-
- /**
- * @return the repositoryType
- */
- public int getRepositoryType() {
- return repositoryType;
- }
-
- /**
- * @param repositoryType
- * the repositoryType to set
- */
- public void setRepositoryType(int repositoryType) {
- this.repositoryType = repositoryType;
- }
-
- /**
- * @return the repositoryName
- */
- public String getRepositoryName() {
- return repositoryName;
- }
-
- /**
- * @param repositoryName
- * the repositoryName to set
- */
- public void setRepositoryName(String repositoryName) {
- this.repositoryName = repositoryName;
- }
-
- /**
- * @return the user
- */
- public String getUser() {
- return user;
- }
-
- /**
- * @param user
- * the user to set
- */
- public void setUser(String user) {
- this.user = user;
- }
-
- /**
- * @return the timeStamp
- */
- public Date getEventTime() {
- return eventTime;
- }
-
- /**
- * @param eventTime
- * the eventTime to set
- */
- public void setEventTime(Date eventTime) {
- this.eventTime = eventTime;
- }
-
- /**
- * @return the accessType
- */
- public String getAccessType() {
- return accessType;
- }
-
- /**
- * @param accessType
- * the accessType to set
- */
- public void setAccessType(String accessType) {
- this.accessType = accessType;
- }
-
- /**
- * @return the resourcePath
- */
- public String getResourcePath() {
- return resourcePath;
- }
-
- /**
- * @param resourcePath
- * the resourcePath to set
- */
- public void setResourcePath(String resourcePath) {
- this.resourcePath = resourcePath;
- }
-
- /**
- * @return the resourceType
- */
- public String getResourceType() {
- return resourceType;
- }
-
- /**
- * @param resourceType
- * the resourceType to set
- */
- public void setResourceType(String resourceType) {
- this.resourceType = resourceType;
- }
-
- /**
- * @return the action
- */
- public String getAction() { return action; }
-
- /**
- * @param action
- * the action to set
- */
- public void setAction(String action) {
- this.action = action;
- }
-
- /**
- * @return the accessResult
- */
- public short getAccessResult() {
- return accessResult;
- }
-
- /**
- * @param accessResult
- * the accessResult to set
- */
- public void setAccessResult(short accessResult) {
- this.accessResult = accessResult;
- }
-
- /**
- * @return the agentId
- */
- public String getAgentId() {
- return agentId;
- }
-
- /**
- * @param agentId
- * the agentId to set
- */
- public void setAgentId(String agentId) {
- this.agentId = agentId;
- }
-
- /**
- * @return the policyId
- */
- public long getPolicyId() {
- return policyId;
- }
-
- /**
- * @param policyId
- * the policyId to set
- */
- public void setPolicyId(long policyId) {
- this.policyId = policyId;
- }
-
- /**
- * @return the resultReason
- */
- public String getResultReason() {
- return resultReason;
- }
-
- /**
- * @param resultReason
- * the resultReason to set
- */
- public void setResultReason(String resultReason) {
- this.resultReason = resultReason;
- }
-
- /**
- * @return the aclEnforcer
- */
- public String getAclEnforcer() {
- return aclEnforcer;
- }
-
- /**
- * @param aclEnforcer
- * the aclEnforcer to set
- */
- public void setAclEnforcer(String aclEnforcer) {
- this.aclEnforcer = aclEnforcer;
- }
-
- /**
- * @return the sessionId
- */
- public String getSessionId() {
- return sessionId;
- }
-
- /**
- * @param sessionId
- * the sessionId to set
- */
- public void setSessionId(String sessionId) {
- this.sessionId = sessionId;
- }
-
- /**
- * @return the clientType
- */
- public String getClientType() {
- return clientType;
- }
-
- /**
- * @param clientType
- * the clientType to set
- */
- public void setClientType(String clientType) {
- this.clientType = clientType;
- }
-
- /**
- * @return the clientIP
- */
- public String getClientIP() {
- return clientIP;
- }
-
- /**
- * @param clientIP
- * the clientIP to set
- */
- public void setClientIP(String clientIP) {
- this.clientIP = clientIP;
- }
-
- /**
- * @return the requestData
- */
- public String getRequestData() { return requestData; }
-
- /**
- * @param requestData
- * the requestData to set
- */
- public void setRequestData(String requestData) {
- this.requestData = requestData;
- }
-
- public String getAgentHostname() {
- return agentHostname;
- }
-
- public void setAgentHostname(String agentHostname) {
- this.agentHostname = agentHostname;
- }
-
- public String getLogType() {
- return logType;
- }
-
- public void setLogType(String logType) {
- this.logType = logType;
- }
-
- public String getEventId() {
- return eventId;
- }
-
- public void setEventId(String eventId) {
- this.eventId = eventId;
- }
-
- public long getSeqNum() {
- return seqNum;
- }
-
- public void setSeqNum(long seqNum) {
- this.seqNum = seqNum;
- }
-
- public long getEventCount() {
- return eventCount;
- }
-
- public void setEventCount(long frequencyCount) {
- this.eventCount = frequencyCount;
- }
-
- public long getEventDurationMS() {
- return eventDurationMS;
- }
-
- public Set<String> getTags() {
- return tags;
- }
-
- public void setEventDurationMS(long frequencyDurationMS) {
- this.eventDurationMS = frequencyDurationMS;
- }
-
- public void setTags(Set<String> tags) {
- this.tags = tags;
- }
-
- public String getClusterName() {
- return clusterName;
- }
-
- public void setZoneName(String zoneName) {
- this.zoneName = zoneName;
- }
-
- public String getZoneName() {
- return zoneName;
- }
-
- public void setPolicyVersion(Long policyVersion) {
- this.policyVersion = policyVersion;
- }
-
- public Long getPolicyVersion() {
- return policyVersion;
- }
-
- public void setClusterName(String clusterName) {
- this.clusterName = clusterName;
- }
-
- public String getAdditionalInfo() { return this.additionalInfo; }
-
- public void setAdditionalInfo(String additionalInfo) { this.additionalInfo = additionalInfo; }
-
- @JsonIgnore
- @Override
- public String getEventKey() {
- String key = user + "^" + accessType + "^" + resourcePath + "^"
- + resourceType + "^" + action + "^" + accessResult + "^"
- + sessionId + "^" + clientIP;
- return key;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
-
- sb.append("AuthzAuditEvent{");
- toString(sb);
- sb.append("}");
-
- return sb.toString();
- }
-
- protected StringBuilder toString(StringBuilder sb) {
- sb.append("repositoryType=").append(repositoryType)
- .append(FIELD_SEPARATOR).append("repositoryName=")
- .append(repositoryName).append(FIELD_SEPARATOR).append("user=")
- .append(user).append(FIELD_SEPARATOR).append("eventTime=")
- .append(eventTime).append(FIELD_SEPARATOR)
- .append("accessType=").append(accessType)
- .append(FIELD_SEPARATOR).append("resourcePath=")
- .append(resourcePath).append(FIELD_SEPARATOR)
- .append("resourceType=").append(resourceType)
- .append(FIELD_SEPARATOR).append("action=").append(action)
- .append(FIELD_SEPARATOR).append("accessResult=")
- .append(accessResult).append(FIELD_SEPARATOR)
- .append("agentId=").append(agentId).append(FIELD_SEPARATOR)
- .append("policyId=").append(policyId).append(FIELD_SEPARATOR)
- .append("resultReason=").append(resultReason)
- .append(FIELD_SEPARATOR).append("aclEnforcer=")
- .append(aclEnforcer).append(FIELD_SEPARATOR)
- .append("sessionId=").append(sessionId).append(FIELD_SEPARATOR)
- .append("clientType=").append(clientType)
- .append(FIELD_SEPARATOR).append("clientIP=").append(clientIP)
- .append(FIELD_SEPARATOR).append("requestData=")
- .append(requestData).append(FIELD_SEPARATOR)
- .append("agentHostname=").append(agentHostname)
- .append(FIELD_SEPARATOR).append("logType=").append(logType)
- .append(FIELD_SEPARATOR).append("eventId=").append(eventId)
- .append(FIELD_SEPARATOR).append("seq_num=").append(seqNum)
- .append(FIELD_SEPARATOR).append("event_count=")
- .append(eventCount).append(FIELD_SEPARATOR)
- .append("event_dur_ms=").append(eventDurationMS)
- .append(FIELD_SEPARATOR)
- .append("tags=").append("[")
- .append(StringUtils.join(tags, ", "))
- .append("]")
- .append(FIELD_SEPARATOR).append("clusterName=").append(clusterName)
- .append(FIELD_SEPARATOR).append("zoneName=").append(zoneName)
- .append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion)
- .append(FIELD_SEPARATOR).append("additionalInfo=").append(additionalInfo);
-
- return sb;
- }
+ @JsonProperty("agentHost")
+ protected String agentHostname;
+
+ @JsonProperty("logType")
+ protected String logType;
+
+ @JsonProperty("id")
+ protected String eventId;
+
+ /**
+ * This to ensure order within a session. Order not guaranteed across
+ * processes and hosts
+ */
+ @JsonProperty("seq_num")
+ protected long seqNum;
+
+ @JsonProperty("event_count")
+ protected long eventCount = 1;
+
+ @JsonProperty("event_dur_ms")
+ protected long eventDurationMS;
+
+ @JsonProperty("tags")
+ protected Set<String> tags = new HashSet<>();
+
+ @JsonProperty("datasets")
+ protected Set<String> datasets;
+
+ @JsonProperty("projects")
+ protected Set<String> projects;
+
+ @JsonProperty("datasetIds")
+ protected Set<Long> datasetIds;
+
+ @JsonProperty("additional_info")
+ protected String additionalInfo;
+
+ @JsonProperty("cluster_name")
+ protected String clusterName;
+
+ @JsonProperty("zone_name")
+ protected String zoneName;
+
+ @JsonProperty("policy_version")
+ protected Long policyVersion;
+
+ public AuthzAuditEvent() {
+ super();
+
+ this.repositoryType = 0;
+ }
+
+ public AuthzAuditEvent(int repositoryType, String repositoryName, String user, Date eventTime, String accessType,
+ String resourcePath, String resourceType, String action, short accessResult, String agentId, long policyId,
+ String resultReason, String aclEnforcer, String sessionId, String clientType, String clientIP, String requestData, String clusterName) {
+ this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
+ policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, null);
+ }
+
+ public AuthzAuditEvent(int repositoryType, String repositoryName, String user, Date eventTime, String accessType,
+ String resourcePath, String resourceType, String action, short accessResult, String agentId, long policyId,
+ String resultReason, String aclEnforcer, String sessionId, String clientType, String clientIP, String requestData, String clusterName, String zoneName) {
+ this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
+ policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, zoneName, null);
+ }
+
+ public AuthzAuditEvent(int repositoryType, String repositoryName, String user, Date eventTime, String accessType,
+ String resourcePath, String resourceType, String action, short accessResult, String agentId, long policyId,
+ String resultReason, String aclEnforcer, String sessionId, String clientType, String clientIP, String requestData, String clusterName, String zoneName, Long policyVersion) {
+ this.repositoryType = repositoryType;
+ this.repositoryName = repositoryName;
+ this.user = user;
+ this.eventTime = eventTime;
+ this.accessType = accessType;
+ this.resourcePath = resourcePath;
+ this.resourceType = resourceType;
+ this.action = action;
+ this.accessResult = accessResult;
+ this.agentId = agentId;
+ this.policyId = policyId;
+ this.resultReason = resultReason;
+ this.aclEnforcer = aclEnforcer;
+ this.sessionId = sessionId;
+ this.clientType = clientType;
+ this.clientIP = clientIP;
+ this.requestData = requestData;
+ this.clusterName = clusterName;
+ this.zoneName = zoneName;
+ this.policyVersion = policyVersion;
+ }
+
+ /**
+ * @return the repositoryType
+ */
+ public int getRepositoryType() {
+ return repositoryType;
+ }
+
+ /**
+ * @param repositoryType the repositoryType to set
+ */
+ public void setRepositoryType(int repositoryType) {
+ this.repositoryType = repositoryType;
+ }
+
+ /**
+ * @return the repositoryName
+ */
+ public String getRepositoryName() {
+ return repositoryName;
+ }
+
+ /**
+ * @param repositoryName the repositoryName to set
+ */
+ public void setRepositoryName(String repositoryName) {
+ this.repositoryName = repositoryName;
+ }
+
+ /**
+ * @return the user
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * @param user the user to set
+ */
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ /**
+ * @return the accessType
+ */
+ public String getAccessType() {
+ return accessType;
+ }
+
+ /**
+ * @param accessType the accessType to set
+ */
+ public void setAccessType(String accessType) {
+ this.accessType = accessType;
+ }
+
+ /**
+ * @return the resourcePath
+ */
+ public String getResourcePath() {
+ return resourcePath;
+ }
+
+ /**
+ * @param resourcePath the resourcePath to set
+ */
+ public void setResourcePath(String resourcePath) {
+ this.resourcePath = resourcePath;
+ }
+
+ /**
+ * @return the resourceType
+ */
+ public String getResourceType() {
+ return resourceType;
+ }
+
+ /**
+ * @param resourceType the resourceType to set
+ */
+ public void setResourceType(String resourceType) {
+ this.resourceType = resourceType;
+ }
+
+ /**
+ * @return the action
+ */
+ public String getAction() {
+ return action;
+ }
+
+ /**
+ * @param action the action to set
+ */
+ public void setAction(String action) {
+ this.action = action;
+ }
+
+ /**
+ * @return the accessResult
+ */
+ public short getAccessResult() {
+ return accessResult;
+ }
+
+ /**
+ * @param accessResult the accessResult to set
+ */
+ public void setAccessResult(short accessResult) {
+ this.accessResult = accessResult;
+ }
+
+ /**
+ * @return the agentId
+ */
+ public String getAgentId() {
+ return agentId;
+ }
+
+ /**
+ * @param agentId the agentId to set
+ */
+ public void setAgentId(String agentId) {
+ this.agentId = agentId;
+ }
+
+ /**
+ * @return the policyId
+ */
+ public long getPolicyId() {
+ return policyId;
+ }
+
+ /**
+ * @param policyId the policyId to set
+ */
+ public void setPolicyId(long policyId) {
+ this.policyId = policyId;
+ }
+
+ /**
+ * @return the resultReason
+ */
+ public String getResultReason() {
+ return resultReason;
+ }
+
+ /**
+ * @param resultReason the resultReason to set
+ */
+ public void setResultReason(String resultReason) {
+ this.resultReason = resultReason;
+ }
+
+ /**
+ * @return the aclEnforcer
+ */
+ public String getAclEnforcer() {
+ return aclEnforcer;
+ }
+
+ /**
+ * @param aclEnforcer the aclEnforcer to set
+ */
+ public void setAclEnforcer(String aclEnforcer) {
+ this.aclEnforcer = aclEnforcer;
+ }
+
+ /**
+ * @return the sessionId
+ */
+ public String getSessionId() {
+ return sessionId;
+ }
+
+ /**
+ * @param sessionId the sessionId to set
+ */
+ public void setSessionId(String sessionId) {
+ this.sessionId = sessionId;
+ }
+
+ /**
+ * @return the clientType
+ */
+ public String getClientType() {
+ return clientType;
+ }
+
+ /**
+ * @param clientType the clientType to set
+ */
+ public void setClientType(String clientType) {
+ this.clientType = clientType;
+ }
+
+ /**
+ * @return the clientIP
+ */
+ public String getClientIP() {
+ return clientIP;
+ }
+
+ /**
+ * @param clientIP the clientIP to set
+ */
+ public void setClientIP(String clientIP) {
+ this.clientIP = clientIP;
+ }
+
+ /**
+ * @return the requestData
+ */
+ public String getRequestData() {
+ return requestData;
+ }
+
+ /**
+ * @param requestData the requestData to set
+ */
+ public void setRequestData(String requestData) {
+ this.requestData = requestData;
+ }
+
+ public String getAgentHostname() {
+ return agentHostname;
+ }
+
+ public void setAgentHostname(String agentHostname) {
+ this.agentHostname = agentHostname;
+ }
+
+ public String getLogType() {
+ return logType;
+ }
+
+ public void setLogType(String logType) {
+ this.logType = logType;
+ }
+
+ public String getEventId() {
+ return eventId;
+ }
+
+ public void setEventId(String eventId) {
+ this.eventId = eventId;
+ }
+
+ public long getSeqNum() {
+ return seqNum;
+ }
+
+ public void setSeqNum(long seqNum) {
+ this.seqNum = seqNum;
+ }
+
+ public long getEventCount() {
+ return eventCount;
+ }
+
+ public void setEventCount(long frequencyCount) {
+ this.eventCount = frequencyCount;
+ }
+
+ public long getEventDurationMS() {
+ return eventDurationMS;
+ }
+
+ public void setEventDurationMS(long frequencyDurationMS) {
+ this.eventDurationMS = frequencyDurationMS;
+ }
+
+ public Set<String> getTags() {
+ return tags;
+ }
+
+ public void setTags(Set<String> tags) {
+ this.tags = tags;
+ }
+
+ public Set<String> getDatasets() {
+ return datasets;
+ }
+
+ public void setDatasets(Set<String> datasets) {
+ this.datasets = datasets;
+ }
+
+ public Set<String> getProjects() {
+ return projects;
+ }
+
+ public void setProjects(Set<String> projects) {
+ this.projects = projects;
+ }
+
+ public Set<Long> getDatasetIds() {
+ return datasetIds;
+ }
+
+ public void setDatasetIds(Set<Long> datasetIds) {
+ this.datasetIds = datasetIds;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public String getZoneName() {
+ return zoneName;
+ }
+
+ public void setZoneName(String zoneName) {
+ this.zoneName = zoneName;
+ }
+
+ public Long getPolicyVersion() {
+ return policyVersion;
+ }
+
+ public void setPolicyVersion(Long policyVersion) {
+ this.policyVersion = policyVersion;
+ }
+
+ public String getAdditionalInfo() {
+ return this.additionalInfo;
+ }
+
+ public void setAdditionalInfo(String additionalInfo) {
+ this.additionalInfo = additionalInfo;
+ }
+
+ @JsonIgnore
+ @Override
+ public String getEventKey() {
+ return user + "^" + accessType + "^" + resourcePath + "^" + resourceType + "^" + action + "^" + accessResult + "^" + sessionId + "^" + clientIP;
+ }
+
+ /**
+ * @return the timeStamp
+ */
+ public Date getEventTime() {
+ return eventTime;
+ }
+
+ /**
+ * @param eventTime the eventTime to set
+ */
+ public void setEventTime(Date eventTime) {
+ this.eventTime = eventTime;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("AuthzAuditEvent{");
+ toString(sb);
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+ protected StringBuilder toString(StringBuilder sb) {
+ sb.append("repositoryType=").append(repositoryType)
+ .append(FIELD_SEPARATOR).append("repositoryName=")
+ .append(repositoryName).append(FIELD_SEPARATOR).append("user=")
+ .append(user).append(FIELD_SEPARATOR).append("eventTime=")
+ .append(eventTime).append(FIELD_SEPARATOR)
+ .append("accessType=").append(accessType)
+ .append(FIELD_SEPARATOR).append("resourcePath=")
+ .append(resourcePath).append(FIELD_SEPARATOR)
+ .append("resourceType=").append(resourceType)
+ .append(FIELD_SEPARATOR).append("action=").append(action)
+ .append(FIELD_SEPARATOR).append("accessResult=")
+ .append(accessResult).append(FIELD_SEPARATOR)
+ .append("agentId=").append(agentId).append(FIELD_SEPARATOR)
+ .append("policyId=").append(policyId).append(FIELD_SEPARATOR)
+ .append("resultReason=").append(resultReason)
+ .append(FIELD_SEPARATOR).append("aclEnforcer=")
+ .append(aclEnforcer).append(FIELD_SEPARATOR)
+ .append("sessionId=").append(sessionId).append(FIELD_SEPARATOR)
+ .append("clientType=").append(clientType)
+ .append(FIELD_SEPARATOR).append("clientIP=").append(clientIP)
+ .append(FIELD_SEPARATOR).append("requestData=")
+ .append(requestData).append(FIELD_SEPARATOR)
+ .append("agentHostname=").append(agentHostname)
+ .append(FIELD_SEPARATOR).append("logType=").append(logType)
+ .append(FIELD_SEPARATOR).append("eventId=").append(eventId)
+ .append(FIELD_SEPARATOR).append("seq_num=").append(seqNum)
+ .append(FIELD_SEPARATOR).append("event_count=")
+ .append(eventCount).append(FIELD_SEPARATOR)
+ .append("event_dur_ms=").append(eventDurationMS)
+ .append(FIELD_SEPARATOR).append("tags=").append("[").append(StringUtils.join(tags, ", ")).append("]")
+ .append(FIELD_SEPARATOR).append("datasets=").append("[").append(datasets != null ? StringUtils.join(datasets, ", ") : "").append("]")
+ .append(FIELD_SEPARATOR).append("projects=").append("[").append(projects != null ? StringUtils.join(projects, ", ") : "").append("]")
+ .append(FIELD_SEPARATOR).append("clusterName=").append(clusterName)
+ .append(FIELD_SEPARATOR).append("zoneName=").append(zoneName)
+ .append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion)
+ .append(FIELD_SEPARATOR).append("additionalInfo=").append(additionalInfo);
+
+ return sb;
+ }
}
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java
index dd02255fc..c74805495 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,25 +18,37 @@
package org.apache.ranger.audit.provider;
+import org.apache.ranger.audit.model.AuditEventBase;
+
import java.io.File;
import java.util.Collection;
import java.util.Properties;
-import org.apache.ranger.audit.model.AuditEventBase;
-
public interface AuditHandler {
- boolean log(AuditEventBase event);
- boolean log(Collection<AuditEventBase> events);
+ boolean log(AuditEventBase event);
+
+ boolean log(Collection<AuditEventBase> events);
+
+ default boolean log(Collection<AuditEventBase> events, String batchKey) {
+ return log(events);
+ }
+
+ boolean logJSON(String event);
+
+ boolean logJSON(Collection<String> events);
- boolean logJSON(String event);
- boolean logJSON(Collection<String> events);
boolean logFile(File file);
void init(Properties prop);
+
void init(Properties prop, String basePropertyName);
+
void start();
+
void stop();
+
void waitToComplete();
+
void waitToComplete(long timeout);
/**
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index f6e29bad8..cd3b671d3 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -18,14 +18,6 @@
package org.apache.ranger.audit.provider;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.ranger.audit.destination.AuditDestination;
import org.apache.ranger.audit.queue.AuditAsyncQueue;
@@ -36,6 +28,14 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
/*
* TODO:
* 1) Flag to enable/disable audit logging
@@ -44,94 +44,94 @@
*/
public class AuditProviderFactory {
- private static final Logger LOG = LoggerFactory.getLogger(AuditProviderFactory.class);
-
- public static final String AUDIT_IS_ENABLED_PROP = "xasecure.audit.is.enabled";
- public static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
- public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled";
- public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
- public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
- public static final String AUDIT_DEST_BASE = "xasecure.audit.destination";
- public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds";
- public static final String AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP = "xasecure.audit.provider.filecache.is.enabled";
- public static final String AUDIT_HDFS_IS_ASYNC_PROP = "xasecure.audit.hdfs.is.async";
- public static final String AUDIT_HDFS_MAX_QUEUE_SIZE_PROP = "xasecure.audit.hdfs.async.max.queue.size";
- public static final String AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.hdfs.async.max.flush.interval.ms";
- public static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async";
- public static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size";
- public static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms";
- public static final String AUDIT_KAFKA_IS_ASYNC_PROP = "xasecure.audit.kafka.is.async";
- public static final String AUDIT_SOLR_IS_ASYNC_PROP = "xasecure.audit.solr.is.async";
-
- public static final String FILE_QUEUE_TYPE = "filequeue";
- public static final String DEFAULT_QUEUE_TYPE = "memoryqueue";
- public static final int AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT = 30;
- public static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024;
- public static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000;
-
- private static final int RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY = 30;
-
- private volatile static AuditProviderFactory sFactory = null;
-
- private AuditHandler mProvider = null;
- private String componentAppType = "";
- private boolean mInitDone = false;
- private JVMShutdownHook jvmShutdownHook = null;
- private ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster","hbaseRegional"));
-
- public AuditProviderFactory() {
- LOG.info("AuditProviderFactory: creating..");
-
- mProvider = getDefaultProvider();
- }
-
- public static AuditProviderFactory getInstance() {
- AuditProviderFactory ret = sFactory;
-
- if(ret == null) {
- synchronized(AuditProviderFactory.class) {
- ret = sFactory;
-
- if(ret == null) {
- ret = sFactory = new AuditProviderFactory();
- }
- }
- }
-
- return ret;
- }
-
- public AuditHandler getAuditProvider() {
- return mProvider;
- }
-
- public boolean isInitDone() {
- return mInitDone;
- }
-
- /**
- * call shutdown hook to provide a way to
- * shutdown gracefully in addition to the ShutdownHook mechanism
- */
- public void shutdown() {
- if (isInitDone() && jvmShutdownHook != null) {
- jvmShutdownHook.run();
- }
- }
-
- public synchronized void init(Properties props, String appType) {
- LOG.info("AuditProviderFactory: initializing..");
-
- if (mInitDone) {
- LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
- }
-
- mInitDone = true;
- componentAppType = appType;
-
- MiscUtil.setApplicationType(appType);
-
- boolean isEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_ENABLED_PROP, true);
+ private static final Logger LOG = LoggerFactory.getLogger(AuditProviderFactory.class);
+
+ public static final String AUDIT_IS_ENABLED_PROP = "xasecure.audit.is.enabled";
+ public static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
+ public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled";
+ public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
+ public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
+ public static final String AUDIT_DEST_BASE = "xasecure.audit.destination";
+ public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds";
+ public static final String AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP = "xasecure.audit.provider.filecache.is.enabled";
+ public static final String AUDIT_HDFS_IS_ASYNC_PROP = "xasecure.audit.hdfs.is.async";
+ public static final String AUDIT_HDFS_MAX_QUEUE_SIZE_PROP = "xasecure.audit.hdfs.async.max.queue.size";
+ public static final String AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.hdfs.async.max.flush.interval.ms";
+ public static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async";
+ public static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size";
+ public static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms";
+ public static final String AUDIT_KAFKA_IS_ASYNC_PROP = "xasecure.audit.kafka.is.async";
+ public static final String AUDIT_SOLR_IS_ASYNC_PROP = "xasecure.audit.solr.is.async";
+
+ public static final String FILE_QUEUE_TYPE = "filequeue";
+ public static final String DEFAULT_QUEUE_TYPE = "memoryqueue";
+ public static final int AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT = 30;
+ public static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024;
+ public static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000;
+
+ private static final int RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY = 30;
+
+ private volatile static AuditProviderFactory sFactory = null;
+
+ private AuditHandler mProvider = null;
+ private String componentAppType = "";
+ private boolean mInitDone = false;
+ private JVMShutdownHook jvmShutdownHook = null;
+ private ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster","hbaseRegional"));
+
+ public AuditProviderFactory() {
+ LOG.info("AuditProviderFactory: creating..");
+
+ mProvider = getDefaultProvider();
+ }
+
+ public static AuditProviderFactory getInstance() {
+ AuditProviderFactory ret = sFactory;
+
+ if(ret == null) {
+ synchronized(AuditProviderFactory.class) {
+ ret = sFactory;
+
+ if(ret == null) {
+ ret = sFactory = new AuditProviderFactory();
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ public AuditHandler getAuditProvider() {
+ return mProvider;
+ }
+
+ public boolean isInitDone() {
+ return mInitDone;
+ }
+
+ /**
+ * call shutdown hook to provide a way to
+ * shutdown gracefully in addition to the ShutdownHook mechanism
+ */
+ public void shutdown() {
+ if (isInitDone() && jvmShutdownHook != null) {
+ jvmShutdownHook.run();
+ }
+ }
+
+ public synchronized void init(Properties props, String appType) {
+ LOG.info("AuditProviderFactory: initializing..");
+
+ if (mInitDone) {
+ LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
+ }
+
+ mInitDone = true;
+ componentAppType = appType;
+
+ MiscUtil.setApplicationType(appType);
+
+ boolean isEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_ENABLED_PROP, true);
if (!isEnabled) {
LOG.info("AuditProviderFactory: Audit not enabled..");
@@ -433,12 +433,16 @@ private AuditHandler getProviderFromConfig(Properties props, String propPrefix,
provider = createDestination("org.apache.ranger.audit.destination.SolrAuditDestination");
} else if (providerName.equalsIgnoreCase("elasticsearch")) {
provider = createDestination("org.apache.ranger.audit.destination.ElasticSearchAuditDestination");
+ } else if (providerName.equalsIgnoreCase("opensearch")) {
+ provider = createDestination("org.apache.ranger.audit.destination.OpenSearchAuditDestination");
} else if (providerName.equalsIgnoreCase("amazon_cloudwatch")) {
provider = createDestination("org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination");
} else if (providerName.equalsIgnoreCase("kafka")) {
provider = createDestination("org.apache.ranger.audit.provider.kafka.KafkaAuditProvider");
} else if (providerName.equalsIgnoreCase("log4j")) {
provider = createDestination("org.apache.ranger.audit.destination.Log4JAuditDestination");
+ } else if (providerName.equalsIgnoreCase("auditserver")) {
+ provider = createDestination("org.apache.ranger.audit.destination.RangerAuditServerDestination");
} else if (providerName.equalsIgnoreCase("batch")) {
provider = getAuditProvider(props, propPrefix, consumer);
} else if (providerName.equalsIgnoreCase("async")) {
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index 94c6d754b..4dc35cb45 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -23,472 +23,484 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicLong;
-
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicLong;
+
public abstract class BaseAuditHandler implements AuditHandler {
- private static final Logger LOG = LoggerFactory.getLogger(BaseAuditHandler.class);
-
- static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms";
-
- static final String AUDIT_LOG_STATUS_LOG_ENABLED = "xasecure.audit.log.status.log.enabled";
- static final String AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = "xasecure.audit.log.status.log.interval.sec";
- static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED = false;
- static final long DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5L * 60; // 5 minutes
-
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks";
-
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks";
-
- public static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm();
- public static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm();
- public static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2";
-
- public static final String PROP_CONFIG = "config";
- public static final String FAILED_TO_LOG_AUDIT_EVENT = "failed to log audit event: {}";
- private int mLogFailureReportMinIntervalInMs = 60 * 1000;
-
- private AtomicLong mFailedLogLastReportTime = new AtomicLong(0);
- private AtomicLong mFailedLogCountSinceLastReport = new AtomicLong(0);
- private AtomicLong mFailedLogCountLifeTime = new AtomicLong(0);
-
- public static final String PROP_NAME = "name";
- public static final String PROP_CLASS_NAME = "classname";
-
- public static final String PROP_DEFAULT_PREFIX = "xasecure.audit.provider";
-
- protected String propPrefix = PROP_DEFAULT_PREFIX;
-
- protected String providerName = null;
- protected String parentPath = null;
-
- protected int failedRetryTimes = 3;
- protected int failedRetrySleep = 3 * 1000;
-
- int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
- long lastErrorLogMS = 0;
-
- long totalCount = 0;
- long totalSuccessCount = 0;
- long totalFailedCount = 0;
- long totalStashedCount = 0;
- long totalDeferredCount = 0;
-
- long lastIntervalCount = 0;
- long lastIntervalSuccessCount = 0;
- long lastIntervalFailedCount = 0;
- long lastStashedCount = 0;
- long lastDeferredCount = 0;
-
- boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
- long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
- long lastStatusLogTime = System.currentTimeMillis();
- long nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
-
- protected Properties props = null;
- protected Map<String, String> configProps = new HashMap<>();
-
- @Override
- public void init(Properties props) {
- init(props, null);
- }
-
- @Override
- public void init(Properties props, String basePropertyName) {
- LOG.info("BaseAuditProvider.init()");
- this.props = props;
- if (basePropertyName != null) {
- propPrefix = basePropertyName;
- }
- LOG.info("propPrefix=" + propPrefix);
-
- String name = MiscUtil.getStringProperty(props, basePropertyName + "."
- + PROP_NAME);
- if (name != null && !name.isEmpty()) {
- setName(name);
- }
- // Get final token
- if (providerName == null) {
- List<String> tokens = MiscUtil.toArray(propPrefix, ".");
- if (!tokens.isEmpty()) {
- String finalToken = tokens.get(tokens.size() - 1);
- setName(finalToken);
- LOG.info("Using providerName from property prefix. providerName="
- + getName());
- }
- }
- LOG.info("providerName=" + getName());
-
- mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props,
- AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
-
- boolean globalStatusLogEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED, DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED);
- long globalStatusLogIntervalSec = MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC);
-
- statusLogEnabled = MiscUtil.getBooleanProperty(props, basePropertyName + ".status.log.enabled", globalStatusLogEnabled);
- statusLogIntervalMS = MiscUtil.getLongProperty(props, basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) * 1000;
-
- nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
-
- LOG.info(AUDIT_LOG_STATUS_LOG_ENABLED + "=" + globalStatusLogEnabled);
- LOG.info(AUDIT_LOG_STATUS_LOG_INTERVAL_SEC + "=" + globalStatusLogIntervalSec);
- LOG.info(basePropertyName + ".status.log.enabled=" + statusLogEnabled);
- LOG.info(basePropertyName + ".status.log.interval.sec=" + (statusLogIntervalMS / 1000));
-
- String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + ".";
- for (Object propNameObj : props.keySet()) {
- String propName = propNameObj.toString();
-
- if (!propName.startsWith(configPropsNamePrefix)) {
- continue;
- }
- String configName = propName.substring(configPropsNamePrefix.length());
- String configValue = props.getProperty(propName);
- configProps.put(configName, configValue);
- LOG.info("Found Config property: " + configName + " => " + configValue);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
- * audit.model.AuditEventBase)
- */
- @Override
- public boolean log(AuditEventBase event) {
- return log(Collections.singletonList(event));
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.lang.String)
- */
- @Override
- public boolean logJSON(String event) {
- AuditEventBase eventObj = MiscUtil.fromJson(event,
- AuthzAuditEvent.class);
- return log(eventObj);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.util.Collection
- * )
- */
- @Override
- public boolean logJSON(Collection<String> events) {
- List<AuditEventBase> eventList = new ArrayList<>(events.size());
- for (String event : events) {
- eventList.add(MiscUtil.fromJson(event, AuthzAuditEvent.class));
- }
- return log(eventList);
- }
-
- @Override
- public boolean logFile(File file) {
- return logFile(file);
- }
-
- public String getParentPath() {
- return parentPath;
- }
-
- public void setParentPath(String parentPath) {
- this.parentPath = parentPath;
- }
-
- public String getFinalPath() {
- return getName();
- }
-
- public void setName(String name) {
- providerName = name;
- }
-
- @Override
- public String getName() {
- if (parentPath != null) {
- return parentPath + "." + providerName;
- }
- return providerName;
- }
-
- public long addTotalCount(int count) {
- totalCount += count;
- return totalCount;
- }
-
- public long addSuccessCount(int count) {
- totalSuccessCount += count;
- return totalSuccessCount;
- }
-
- public long addFailedCount(int count) {
- totalFailedCount += count;
- return totalFailedCount;
- }
-
- public long addStashedCount(int count) {
- totalStashedCount += count;
- return totalStashedCount;
- }
-
- public long addDeferredCount(int count) {
- totalDeferredCount += count;
- return totalDeferredCount;
- }
-
- public long getTotalCount() {
- return totalCount;
- }
-
- public long getTotalSuccessCount() {
- return totalSuccessCount;
- }
-
- public long getTotalFailedCount() {
- return totalFailedCount;
- }
-
- public long getTotalStashedCount() {
- return totalStashedCount;
- }
-
- public long getLastStashedCount() {
- return lastStashedCount;
- }
-
- public long getTotalDeferredCount() {
- return totalDeferredCount;
- }
-
- public long getLastDeferredCount() {
- return lastDeferredCount;
- }
-
- public boolean isStatusLogEnabled() { return statusLogEnabled; }
-
- public void logStatusIfRequired() {
- if (System.currentTimeMillis() > nextStatusLogTime) {
- logStatus();
- }
- }
-
- public void logStatus() {
- try {
- long currTime = System.currentTimeMillis();
- long diffTime = currTime - lastStatusLogTime;
-
- lastStatusLogTime = currTime;
- nextStatusLogTime = currTime + statusLogIntervalMS;
-
- long diffCount = totalCount - lastIntervalCount;
- long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
- long diffFailed = totalFailedCount - lastIntervalFailedCount;
- long diffStashed = totalStashedCount - lastStashedCount;
- long diffDeferred = totalDeferredCount - lastDeferredCount;
-
- if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0
- && diffStashed == 0 && diffDeferred == 0) {
- return;
- }
-
- lastIntervalCount = totalCount;
- lastIntervalSuccessCount = totalSuccessCount;
- lastIntervalFailedCount = totalFailedCount;
- lastStashedCount = totalStashedCount;
- lastDeferredCount = totalDeferredCount;
-
- if (statusLogEnabled) {
- String finalPath = "";
- String tFinalPath = getFinalPath();
- if (!getName().equals(tFinalPath)) {
- finalPath = ", finalDestination=" + tFinalPath;
- }
-
- logAuditStatus(diffTime, diffCount, diffSuccess, diffFailed, diffStashed, diffDeferred, finalPath);
- }
- } catch (Exception t) {
- LOG.error("Error while printing stats. auditProvider=" + getName());
- }
- }
- private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) {
- String msg = "Audit Status Log: name="
- + getName()
- + finalPath
- + ", interval="
- + formatIntervalForLog(diffTime)
- + ", events="
- + diffCount
- + (diffSuccess > 0 ? (", succcessCount=" + diffSuccess)
- : "")
- + (diffFailed > 0 ? (", failedCount=" + diffFailed) : "")
- + (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "")
- + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
- : "")
- + ", totalEvents="
- + totalCount
- + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount)
- : "")
- + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount)
- : "")
- + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount)
- : "")
- + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
- : "");
- LOG.info(msg);
- }
-
- public void logError(String msg, Object arg) {
- long currTimeMS = System.currentTimeMillis();
- if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
- LOG.error(msg, arg);
- lastErrorLogMS = currTimeMS;
- }
- }
-
- public void logError(String msg, Throwable ex) {
- long currTimeMS = System.currentTimeMillis();
- if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
- LOG.error(msg, ex);
- lastErrorLogMS = currTimeMS;
- }
- }
-
- public String getTimeDiffStr(long time1, long time2) {
- long timeInMs = Math.abs(time1 - time2);
- return formatIntervalForLog(timeInMs);
- }
-
- public String formatIntervalForLog(long timeInMs) {
- long hours = timeInMs / (60 * 60 * 1000);
- long minutes = (timeInMs / (60 * 1000)) % 60;
- long seconds = (timeInMs % (60 * 1000)) / 1000;
- long mSeconds = (timeInMs % (1000));
-
- if (hours > 0)
- return String.format("%02d:%02d:%02d.%03d hours", hours, minutes,
- seconds, mSeconds);
- else if (minutes > 0)
- return String.format("%02d:%02d.%03d minutes", minutes, seconds,
- mSeconds);
- else if (seconds > 0)
- return String.format("%02d.%03d seconds", seconds, mSeconds);
- else
- return String.format("%03d milli-seconds", mSeconds);
- }
-
- public void logFailedEvent(AuditEventBase event) {
- logFailedEvent(event, "");
- }
-
- public void logFailedEvent(AuditEventBase event, Throwable excp) {
- long now = System.currentTimeMillis();
-
- long timeSinceLastReport = now - mFailedLogLastReportTime.get();
- long countSinceLastReport = mFailedLogCountSinceLastReport
- .incrementAndGet();
- long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
-
- if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
- mFailedLogLastReportTime.set(now);
- mFailedLogCountSinceLastReport.set(0);
-
- if (excp != null) {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event), excp);
- } else {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event));
- }
-
- if (countLifeTime > 1) { // no stats to print for the 1st failure
- LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
- }
- }
- }
-
- public void logFailedEvent(Collection<AuditEventBase> events) {
- logFailedEvent(events, "");
- }
-
- public void logFailedEvent(Collection<AuditEventBase> events, Throwable excp) {
- for (AuditEventBase event : events) {
- logFailedEvent(event, excp);
- }
- }
-
- public void logFailedEvent(AuditEventBase event, String message) {
- long now = System.currentTimeMillis();
-
- long timeSinceLastReport = now - mFailedLogLastReportTime.get();
- long countSinceLastReport = mFailedLogCountSinceLastReport
- .incrementAndGet();
- long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
-
- if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
- mFailedLogLastReportTime.set(now);
- mFailedLogCountSinceLastReport.set(0);
-
- LOG.warn("failed to log audit event: {} , errorMessage={}", MiscUtil.stringify(event), message);
-
- if (countLifeTime > 1) { // no stats to print for the 1st failure
- LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
- }
- }
- }
-
- public void logFailedEvent(Collection<AuditEventBase> events,
- String errorMessage) {
- for (AuditEventBase event : events) {
- logFailedEvent(event, errorMessage);
- }
- }
-
- public void logFailedEventJSON(String event, Throwable excp) {
- long now = System.currentTimeMillis();
-
- long timeSinceLastReport = now - mFailedLogLastReportTime.get();
- long countSinceLastReport = mFailedLogCountSinceLastReport
- .incrementAndGet();
- long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
-
- if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
- mFailedLogLastReportTime.set(now);
- mFailedLogCountSinceLastReport.set(0);
-
- if (excp != null) {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event, excp);
- } else {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event);
- }
-
- if (countLifeTime > 1) { // no stats to print for the 1st failure
- LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
- }
- }
- }
-
- public void logFailedEventJSON(Collection<String> events, Throwable excp) {
- for (String event : events) {
- logFailedEventJSON(event, excp);
- }
- }
+ private static final Logger LOG = LoggerFactory.getLogger(BaseAuditHandler.class);
+
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks";
+ public static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm();
+ public static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm();
+ public static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2";
+ public static final String PROP_CONFIG = "config";
+ public static final String FAILED_TO_LOG_AUDIT_EVENT = "failed to log audit event: {}";
+ public static final String PROP_NAME = "name";
+ public static final String PROP_CLASS_NAME = "classname";
+ public static final String PROP_DEFAULT_PREFIX = "xasecure.audit.provider";
+
+ static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms";
+ static final String AUDIT_LOG_STATUS_LOG_ENABLED = "xasecure.audit.log.status.log.enabled";
+ static final String AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = "xasecure.audit.log.status.log.interval.sec";
+ static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED = false;
+ static final long DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5L * 60; // 5 minutes
+
+ protected String propPrefix = PROP_DEFAULT_PREFIX;
+ protected String providerName;
+ protected String parentPath;
+ protected int failedRetryTimes = 3;
+ protected int failedRetrySleep = 3 * 1000;
+ protected Map<String, String> configProps = new HashMap<>();
+ protected Properties props;
+
+ int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
+ long lastErrorLogMS;
+ long lastIntervalCount;
+ long lastIntervalSuccessCount;
+ long lastIntervalFailedCount;
+ long lastStashedCount;
+ long lastDeferredCount;
+ AtomicLong totalCount = new AtomicLong(0);
+ AtomicLong totalSuccessCount = new AtomicLong(0);
+ AtomicLong totalFailedCount = new AtomicLong(0);
+ AtomicLong totalStashedCount = new AtomicLong(0);
+ AtomicLong totalDeferredCount = new AtomicLong(0);
+
+ boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
+ long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
+ long lastStatusLogTime = System.currentTimeMillis();
+ long nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
+
+ private int mLogFailureReportMinIntervalInMs = 60 * 1000;
+ private final AtomicLong mFailedLogLastReportTime = new AtomicLong(0);
+ private final AtomicLong mFailedLogCountSinceLastReport = new AtomicLong(0);
+ private final AtomicLong mFailedLogCountLifeTime = new AtomicLong(0);
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
+ * audit.model.AuditEventBase)
+ */
+ @Override
+ public boolean log(AuditEventBase event) {
+ return log(Collections.singletonList(event));
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.lang.String)
+ */
+ @Override
+ public boolean logJSON(String event) {
+ AuditEventBase eventObj = MiscUtil.fromJson(event, AuthzAuditEvent.class);
+
+ return log(eventObj);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.util.Collection)
+ */
+ @Override
+ public boolean logJSON(Collection<String> events) {
+ List<AuditEventBase> eventList = new ArrayList<>(events.size());
+
+ for (String event : events) {
+ eventList.add(MiscUtil.fromJson(event, AuthzAuditEvent.class));
+ }
+
+ return log(eventList);
+ }
+
+ @Override
+ public boolean logFile(File file) {
+ return false;
+ }
+
+ @Override
+ public void init(Properties props) {
+ init(props, null);
+ }
+
+ @Override
+ public void init(Properties props, String basePropertyName) {
+ LOG.info("BaseAuditProvider.init()");
+
+ this.props = props;
+
+ if (basePropertyName != null) {
+ propPrefix = basePropertyName;
+ }
+
+ LOG.info("propPrefix={}", propPrefix);
+
+ String name = MiscUtil.getStringProperty(props, basePropertyName + "." + PROP_NAME);
+
+ if (name != null && !name.isEmpty()) {
+ setName(name);
+ }
+
+ // Get final token
+ if (providerName == null) {
+ List<String> tokens = MiscUtil.toArray(propPrefix, ".");
+
+ if (!tokens.isEmpty()) {
+ String finalToken = tokens.get(tokens.size() - 1);
+
+ setName(finalToken);
+
+ LOG.info("Using providerName from property prefix. providerName={}", getName());
+ }
+ }
+
+ LOG.info("providerName={}", getName());
+
+ mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
+
+ boolean globalStatusLogEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED, DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED);
+ long globalStatusLogIntervalSec = MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC);
+
+ statusLogEnabled = MiscUtil.getBooleanProperty(props, basePropertyName + ".status.log.enabled", globalStatusLogEnabled);
+ statusLogIntervalMS = MiscUtil.getLongProperty(props, basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) * 1000;
+ nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
+
+ LOG.info("{}={}", AUDIT_LOG_STATUS_LOG_ENABLED, globalStatusLogEnabled);
+ LOG.info("{}={}", AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, globalStatusLogIntervalSec);
+ LOG.info("{}.status.log.enabled={}", basePropertyName, statusLogEnabled);
+ LOG.info("{}.status.log.interval.sec={}", basePropertyName, (statusLogIntervalMS / 1000));
+
+ String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + ".";
+
+ for (Object propNameObj : props.keySet()) {
+ String propName = propNameObj.toString();
+
+ if (!propName.startsWith(configPropsNamePrefix)) {
+ continue;
+ }
+
+ String configName = propName.substring(configPropsNamePrefix.length());
+ String configValue = props.getProperty(propName);
+
+ configProps.put(configName, configValue);
+
+ LOG.info("Found Config property: {} => {}", configName, configValue);
+ }
+ }
+
+ @Override
+ public String getName() {
+ if (parentPath != null) {
+ return parentPath + "." + providerName;
+ }
+
+ return providerName;
+ }
+
+ public void setName(String name) {
+ providerName = name;
+ }
+
+ public String getParentPath() {
+ return parentPath;
+ }
+
+ public void setParentPath(String parentPath) {
+ this.parentPath = parentPath;
+ }
+
+ public String getFinalPath() {
+ return getName();
+ }
+
+ public long addTotalCount(int count) {
+ return totalCount.addAndGet(count);
+ }
+
+ public long addSuccessCount(int count) {
+ return totalSuccessCount.addAndGet(count);
+ }
+
+ public long addFailedCount(int count) {
+ return totalFailedCount.addAndGet(count);
+ }
+
+ public long addStashedCount(int count) {
+ return totalStashedCount.addAndGet(count);
+ }
+
+ public long addDeferredCount(int count) {
+ return totalDeferredCount.addAndGet(count);
+ }
+
+ public long getTotalCount() {
+ return totalCount.get();
+ }
+
+ public long getTotalSuccessCount() {
+ return totalSuccessCount.get();
+ }
+
+ public long getTotalFailedCount() {
+ return totalFailedCount.get();
+ }
+
+ public long getTotalStashedCount() {
+ return totalStashedCount.get();
+ }
+
+ public long getLastStashedCount() {
+ return lastStashedCount;
+ }
+ public long getTotalDeferredCount() {
+ return totalDeferredCount.get();
+ }
+
+ public long getLastDeferredCount() {
+ return lastDeferredCount;
+ }
+
+ public boolean isStatusLogEnabled() {
+ return statusLogEnabled;
+ }
+
+ public void logStatusIfRequired() {
+ if (System.currentTimeMillis() > nextStatusLogTime) {
+ logStatus();
+ }
+ }
+
+ public void logStatus() {
+ try {
+ long currTime = System.currentTimeMillis();
+ long diffTime = currTime - lastStatusLogTime;
+
+ lastStatusLogTime = currTime;
+ nextStatusLogTime = currTime + statusLogIntervalMS;
+
+ long currentTotalCount = totalCount.get();
+ long currentSuccessCount = totalSuccessCount.get();
+ long currentFailedCount = totalFailedCount.get();
+ long currentStashedCount = totalStashedCount.get();
+ long currentDeferredCount = totalDeferredCount.get();
+
+ long diffCount = currentTotalCount - lastIntervalCount;
+ long diffSuccess = currentSuccessCount - lastIntervalSuccessCount;
+ long diffFailed = currentFailedCount - lastIntervalFailedCount;
+ long diffStashed = currentStashedCount - lastStashedCount;
+ long diffDeferred = currentDeferredCount - lastDeferredCount;
+
+ if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 && diffStashed == 0 && diffDeferred == 0) {
+ return;
+ }
+
+ lastIntervalCount = currentTotalCount;
+ lastIntervalSuccessCount = currentSuccessCount;
+ lastIntervalFailedCount = currentFailedCount;
+ lastStashedCount = currentStashedCount;
+ lastDeferredCount = currentDeferredCount;
+
+ if (statusLogEnabled) {
+ String finalPath = "";
+ String tFinalPath = getFinalPath();
+
+ if (!getName().equals(tFinalPath)) {
+ finalPath = ", finalDestination=" + tFinalPath;
+ }
+
+ logAuditStatus(diffTime, diffCount, diffSuccess, diffFailed, diffStashed, diffDeferred, finalPath);
+ }
+ } catch (Exception t) {
+ LOG.error("Error while printing stats. auditProvider={}", getName());
+ }
+ }
+
+ public void logError(String msg, Object...args) {
+ long currTimeMS = System.currentTimeMillis();
+
+ if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
+ LOG.error(msg, args);
+
+ lastErrorLogMS = currTimeMS;
+ }
+ }
+
+ public void logError(String msg, Throwable ex) {
+ long currTimeMS = System.currentTimeMillis();
+
+ if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
+ LOG.error(msg, ex);
+
+ lastErrorLogMS = currTimeMS;
+ }
+ }
+
+ public String getTimeDiffStr(long time1, long time2) {
+ long timeInMs = Math.abs(time1 - time2);
+
+ return formatIntervalForLog(timeInMs);
+ }
+
+ public String formatIntervalForLog(long timeInMs) {
+ long hours = timeInMs / (60 * 60 * 1000);
+ long minutes = (timeInMs / (60 * 1000)) % 60;
+ long seconds = (timeInMs % (60 * 1000)) / 1000;
+ long mSeconds = (timeInMs % (1000));
+
+ if (hours > 0) {
+ return String.format("%02d:%02d:%02d.%03d hours", hours, minutes, seconds, mSeconds);
+ } else if (minutes > 0) {
+ return String.format("%02d:%02d.%03d minutes", minutes, seconds, mSeconds);
+ } else if (seconds > 0) {
+ return String.format("%02d.%03d seconds", seconds, mSeconds);
+ } else {
+ return String.format("%03d milli-seconds", mSeconds);
+ }
+ }
+
+ public void logFailedEvent(AuditEventBase event) {
+ logFailedEvent(event, "");
+ }
+
+ public void logFailedEvent(AuditEventBase event, Throwable excp) {
+ long now = System.currentTimeMillis();
+ long timeSinceLastReport = now - mFailedLogLastReportTime.get();
+ long countSinceLastReport = mFailedLogCountSinceLastReport.incrementAndGet();
+ long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
+
+ if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
+ mFailedLogLastReportTime.set(now);
+ mFailedLogCountSinceLastReport.set(0);
+
+ if (excp != null) {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event), excp);
+ } else {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event));
+ }
+
+ if (countLifeTime > 1) { // no stats to print for the 1st failure
+ LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
+ }
+ }
+ }
+
+ public void logFailedEvent(Collection<AuditEventBase> events) {
+ logFailedEvent(events, "");
+ }
+
+ public void logFailedEvent(Collection<AuditEventBase> events, Throwable excp) {
+ for (AuditEventBase event : events) {
+ logFailedEvent(event, excp);
+ }
+ }
+
+ public void logFailedEvent(AuditEventBase event, String message) {
+ long now = System.currentTimeMillis();
+ long timeSinceLastReport = now - mFailedLogLastReportTime.get();
+ long countSinceLastReport = mFailedLogCountSinceLastReport.incrementAndGet();
+ long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
+
+ if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
+ mFailedLogLastReportTime.set(now);
+ mFailedLogCountSinceLastReport.set(0);
+
+ LOG.warn("failed to log audit event: {} , errorMessage={}", MiscUtil.stringify(event), message);
+
+ if (countLifeTime > 1) { // no stats to print for the 1st failure
+ LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
+ }
+ }
+ }
+
+ public void logFailedEvent(Collection<AuditEventBase> events, String errorMessage) {
+ for (AuditEventBase event : events) {
+ logFailedEvent(event, errorMessage);
+ }
+ }
+
+ public void logFailedEventJSON(String event, Throwable excp) {
+ long now = System.currentTimeMillis();
+ long timeSinceLastReport = now - mFailedLogLastReportTime.get();
+ long countSinceLastReport = mFailedLogCountSinceLastReport.incrementAndGet();
+ long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
+
+ if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
+ mFailedLogLastReportTime.set(now);
+ mFailedLogCountSinceLastReport.set(0);
+
+ if (excp != null) {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event, excp);
+ } else {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event);
+ }
+
+ if (countLifeTime > 1) { // no stats to print for the 1st failure
+ LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
+ }
+ }
+ }
+
+ public void logFailedEventJSON(Collection<String> events, Throwable excp) {
+ for (String event : events) {
+ logFailedEventJSON(event, excp);
+ }
+ }
+
+ private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) {
+ long currentTotalCount = totalCount.get();
+ long currentTotalSuccessCount = totalSuccessCount.get();
+ long currentTotalFailedCount = totalFailedCount.get();
+ long currentTotalStashedCount = totalStashedCount.get();
+ long currentTotalDeferredCount = totalDeferredCount.get();
+
+ String msg = "Audit Status Log: name="
+ + getName()
+ + finalPath
+ + ", interval="
+ + formatIntervalForLog(diffTime)
+ + ", events="
+ + diffCount
+ + (diffSuccess > 0 ? (", succcessCount=" + diffSuccess)
+ : "")
+ + (diffFailed > 0 ? (", failedCount=" + diffFailed) : "")
+ + (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "")
+ + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
+ : "")
+ + ", totalEvents="
+ + currentTotalCount
+ + (currentTotalSuccessCount > 0 ? (", totalSuccessCount=" + currentTotalSuccessCount)
+ : "")
+ + (currentTotalFailedCount > 0 ? (", totalFailedCount=" + currentTotalFailedCount)
+ : "")
+ + (currentTotalStashedCount > 0 ? (", totalStashedCount=" + currentTotalStashedCount)
+ : "")
+ + (currentTotalDeferredCount > 0 ? (", totalDeferredCount=" + currentTotalDeferredCount)
+ : "");
+ LOG.info(msg);
+ }
}
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractKerberosUser.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractKerberosUser.java
index fd1c96e90..b78d64d5b 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractKerberosUser.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractKerberosUser.java
@@ -27,6 +27,7 @@
import javax.security.auth.kerberos.KerberosTicket;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
+
import java.security.PrivilegedAction;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
@@ -36,11 +37,9 @@
import java.util.concurrent.atomic.AtomicBoolean;
public abstract class AbstractKerberosUser implements KerberosUser {
-
private static final Logger LOG = LoggerFactory.getLogger(AbstractKerberosUser.class);
static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
-
/**
* Percentage of the ticket window to use before we renew the TGT.
*/
@@ -48,7 +47,7 @@ public abstract class AbstractKerberosUser implements KerberosUser {
protected final AtomicBoolean loggedIn = new AtomicBoolean(false);
- protected Subject subject;
+ protected Subject subject;
protected LoginContext loginContext;
public AbstractKerberosUser() {
@@ -68,31 +67,32 @@ public synchronized void login() throws LoginException {
try {
// If it's the first time ever calling login then we need to initialize a new context
if (loginContext == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Initializing new login context...");
- }
+ LOG.debug("Initializing new login context...");
+
if (this.subject == null) {
// only create a new subject if a current one does not exist
// other classes may be referencing an existing subject and replacing it may break functionality of those other classes after relogin
this.subject = new Subject();
}
+
this.loginContext = createLoginContext(subject);
}
loginContext.login();
loggedIn.set(true);
+
if (LOG.isDebugEnabled()) {
- LOG.debug("Successful login for {}", new Object[]{getPrincipal()});
+ LOG.debug("Successful login for {}", getPrincipal());
}
} catch (LoginException le) {
LoginException loginException = new LoginException("Unable to login with " + getPrincipal() + " due to: " + le.getMessage());
+
loginException.setStackTrace(le.getStackTrace());
+
throw loginException;
}
}
- protected abstract LoginContext createLoginContext(final Subject subject) throws LoginException;
-
/**
* Performs a logout of the current user.
*
@@ -107,7 +107,8 @@ public synchronized void logout() throws LoginException {
try {
loginContext.logout();
loggedIn.set(false);
- LOG.debug("Successful logout for {}", new Object[]{getPrincipal()});
+
+ LOG.debug("Successful logout for {}", getPrincipal());
loginContext = null;
} catch (LoginException e) {
@@ -142,8 +143,7 @@ public <T> T doAs(final PrivilegedAction<T> action) throws IllegalStateException
* @throws PrivilegedActionException if an exception is thrown from the action
*/
@Override
- public <T> T doAs(final PrivilegedExceptionAction<T> action)
- throws IllegalStateException, PrivilegedActionException {
+ public <T> T doAs(final PrivilegedExceptionAction<T> action) throws IllegalStateException, PrivilegedActionException {
if (!isLoggedIn()) {
throw new IllegalStateException("Must login before executing actions");
}
@@ -152,34 +152,122 @@ public <T> T doAs(final PrivilegedExceptionAction<T> action)
}
/**
- * Re-login a user from keytab if TGT is expired or is close to expiry.
+ * Proactively renew credentials when the TGT is missing or has passed
+ * {@link #TICKET_RENEW_WINDOW} (80%) of its lifetime.
+ *
+ * <p>Called on every {@link KerberosAction#execute()} before the privileged
+ * action runs, so long-lived Solr/Kafka dispatchers refresh before the TGT
+ * expires.
*
- * @throws LoginException if an error happens performing the re-login
+ * @return {@code true} if relogin was performed, {@code false} if the TGT
+ * is still valid
+ * @throws LoginException if relogin fails
*/
@Override
public synchronized boolean checkTGTAndRelogin() throws LoginException {
final KerberosTicket tgt = getTGT();
+
if (tgt == null) {
LOG.debug("TGT was not found");
}
if (tgt != null && System.currentTimeMillis() < getRefreshTime(tgt)) {
LOG.debug("TGT was found, but has not reached expiration window");
+
return false;
}
- LOG.debug("Performing relogin for {}", new Object[]{getPrincipal()});
+ LOG.debug("Performing relogin for {}", getPrincipal());
+
+ performRelogin();
+
+ return true;
+ }
+
+ /**
+ * Renews JAAS credentials for proactive TGT refresh and for error recovery.
+ *
+ * <p>With {@code useKeyTab=true} and {@code useTicketCache=true}
+ * (shipped Solr dispatcher default), the previous {@code logout(); login()}
+ * sequence at the 80% TGT renewal window leaves {@code Krb5LoginModule}
+ * with no encryption key for the ticket cache and fails with
+ * {@code "No key to store"}, stopping audit indexing until process restart.
+ * For keytab-backed users, calling {@code loginContext.login()} on the
+ * existing {@link LoginContext} and {@link Subject} renews from the keytab
+ * without clearing credentials first. The {@link Subject} is preserved
+ * across relogin because other components may hold references to it.
+ *
+ * <p>Non-keytab principals, or callers that are not yet logged in, use the
+ * traditional {@code logout(); login()} path. In-place keytab relogin
+ * failures also fall back to that path.
+ *
+ * @throws LoginException if relogin fails
+ * @see KerberosJAASConfigUser#useKeytabRelogin()
+ */
+ public synchronized void performRelogin() throws LoginException {
+ if (useKeytabRelogin() && isLoggedIn() && loginContext != null) {
+ try {
+ // In-place keytab relogin without logout() (useTicketCache=true).
+ loginContext.login();
+ loggedIn.set(true);
+
+ LOG.debug("Successful in-place keytab relogin for {}", getPrincipal());
+
+ return;
+ } catch (LoginException e) {
+ LOG.warn("In-place keytab relogin failed for {}, falling back to logout/login: {}", getPrincipal(), e.getMessage());
+ }
+ }
+
logout();
login();
- return true;
}
/**
- * Get the Kerberos TGT.
+ * Whether {@link #performRelogin()} should renew from keytab in place (no
+ * {@code logout()} first). Default {@code false};
+ * {@link KerberosJAASConfigUser} returns {@code true} when JAAS
+ * {@code useKeyTab} is enabled. Gated on keytab capability, not on
+ * {@code useTicketCache}, because only keytab principals can safely skip
+ * logout.
+ *
+ * @return {@code true} to use in-place keytab relogin
+ */
+ protected boolean useKeytabRelogin() {
+ return false;
+ }
+
+ /**
+ * @return true if this user is currently logged in, false otherwise
+ */
+ @Override
+ public boolean isLoggedIn() {
+ return loggedIn.get();
+ }
+
+ @Override
+ public String toString() {
+ return "KerberosUser{" +
+ "principal='" + getPrincipal() + '\'' +
+ ", loggedIn=" + loggedIn +
+ '}';
+ }
+
+ protected abstract LoginContext createLoginContext(Subject subject) throws LoginException;
+
+ /**
+ * Returns the Kerberos TGT from the JAAS {@link Subject}, if present.
*
- * @return the user's TGT or null if none was found
+ * @return the user's TGT, or {@code null} if the subject is uninitialized
+ * or has no TGT
*/
private synchronized KerberosTicket getTGT() {
+ // Subject is created lazily on first login(); null before that is
+ // expected.
+ if (subject == null) {
+ return null;
+ }
+
final Set<KerberosTicket> tickets = subject.getPrivateCredentials(KerberosTicket.class);
for (KerberosTicket ticket : tickets) {
@@ -204,8 +292,9 @@ private boolean isTGSPrincipal(final KerberosPrincipal principal) {
if (principal.getName().equals("krbtgt/" + principal.getRealm() + "@" + principal.getRealm())) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Found TGT principal: " + principal.getName());
+ LOG.trace("Found TGT principal: {}", principal.getName());
}
+
return true;
}
@@ -214,33 +303,17 @@ private boolean isTGSPrincipal(final KerberosPrincipal principal) {
private long getRefreshTime(final KerberosTicket tgt) {
long start = tgt.getStartTime().getTime();
- long end = tgt.getEndTime().getTime();
+ long end = tgt.getEndTime().getTime();
if (LOG.isTraceEnabled()) {
final SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- final String startDate = dateFormat.format(new Date(start));
- final String endDate = dateFormat.format(new Date(end));
- LOG.trace("TGT valid starting at: " + startDate);
- LOG.trace("TGT expires at: " + endDate);
+ final String startDate = dateFormat.format(new Date(start));
+ final String endDate = dateFormat.format(new Date(end));
+
+ LOG.trace("TGT valid starting at: {}", startDate);
+ LOG.trace("TGT expires at: {}", endDate);
}
return start + (long) ((end - start) * TICKET_RENEW_WINDOW);
}
-
- /**
- * @return true if this user is currently logged in, false otherwise
- */
- @Override
- public boolean isLoggedIn() {
- return loggedIn.get();
- }
-
- @Override
- public String toString() {
- return "KerberosUser{" +
- "principal='" + getPrincipal() + '\'' +
- ", loggedIn=" + loggedIn +
- '}';
- }
}
-
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
index 3352ea787..d1ac21051 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
@@ -21,7 +21,12 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.CommonPathCapabilities;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.ranger.audit.provider.MiscUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,122 +53,169 @@ public abstract class AbstractRangerAuditWriter implements RangerAuditWriter {
public static final String PROP_FILESYSTEM_FILE_EXTENSION = ".log";
public static final String PROP_IS_APPEND_ENABLED = "file.append.enabled";
- public Configuration conf = null;
- public FileSystem fileSystem = null;
- public Map<String, String> auditConfigs = null;
- public Path auditPath = null;
- public volatile PrintWriter logWriter = null;
- public RollingTimeUtil rollingTimeUtil = null;
- public String auditProviderName = null;
- public String fullPath = null;
- public String parentFolder = null;
- public String currentFileName = null;
- public String logFileNameFormat = null;
- public String logFolder = null;
- public String fileExtension = null;
- public String rolloverPeriod = null;
- public String fileSystemScheme = null;
- public Date nextRollOverTime = null;
- public int fileRolloverSec = 24 * 60 * 60; // In seconds
- public boolean rollOverByDuration = false;
- public volatile FSDataOutputStream ostream = null; // output stream wrapped in logWriter
- private boolean isHFlushCapableStream = false;
- protected boolean reUseLastLogFile = false;
+ public Configuration conf;
+ public FileSystem fileSystem;
+ public Map<String, String> auditConfigs;
+ public Path auditPath;
+ public RollingTimeUtil rollingTimeUtil;
+ public String auditProviderName;
+ public String fullPath;
+ public String parentFolder;
+ public String currentFileName;
+ public String logFileNameFormat;
+ public String logFolder;
+ public String fileExtension;
+ public String rolloverPeriod;
+ public String fileSystemScheme;
+ public Date nextRollOverTime;
+ public int fileRolloverSec = 24 * 60 * 60; // In seconds
+ public boolean rollOverByDuration;
+
+ public volatile PrintWriter logWriter;
+ public volatile FSDataOutputStream ostream; // output stream wrapped in logWriter
+
+ protected boolean reUseLastLogFile;
+ private boolean isHFlushCapableStream;
@Override
- public void init(Properties props, String propPrefix, String auditProviderName, Map<String,String> auditConfigs) {
+ public void init(Properties props, String propPrefix, String auditProviderName, Map<String, String> auditConfigs) {
// Initialize properties for this class
// Initial folder and file properties
logger.info("==> AbstractRangerAuditWriter.init()");
+
this.auditProviderName = auditProviderName;
- this.auditConfigs = auditConfigs;
+ this.auditConfigs = auditConfigs;
- init(props,propPrefix);
+ init(props, propPrefix);
logger.info("<== AbstractRangerAuditWriter.init()");
}
- public void createFileSystemFolders() throws Exception {
+ @Override
+ public void flush() {
+ logger.debug("==> AbstractRangerAuditWriter.flush() {}", fileSystemScheme);
+
+ if (ostream != null) {
+ try {
+ synchronized (this) {
+ if (ostream != null) {
+ // 1) PrinterWriter does not have bufferring of its own so
+ // we need to flush its underlying stream
+ // 2) HDFS flush() does not really flush all the way to disk.
+ if (isHFlushCapableStream) {
+ //Checking HFLUSH capability of the stream because of HADOOP-13327.
+ //For S3 filesysttem, hflush throws UnsupportedOperationException and hence we call flush.
+ ostream.hflush();
+ } else {
+ ostream.flush();
+ }
+ }
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.createFileSystemFolders()");
+ logger.debug("Flush {} audit logs completed.....", fileSystemScheme);
+ }
+ } catch (IOException e) {
+ logger.error("Error on flushing log writer: {}\nException will be ignored. name={}, fileName={}", e.getMessage(), auditProviderName, currentFileName);
+ }
}
+
+ logger.debug("<== AbstractRangerAuditWriter.flush()");
+ }
+
+ public void createFileSystemFolders() throws Exception {
+ logger.debug("==> AbstractRangerAuditWriter.createFileSystemFolders()");
+
// Create a new file
- Date currentTime = new Date();
- String fileName = MiscUtil.replaceTokens(logFileNameFormat, currentTime.getTime());
- parentFolder = MiscUtil.replaceTokens(logFolder, currentTime.getTime());
- fullPath = parentFolder + Path.SEPARATOR + fileName;
- String defaultPath = fullPath;
- conf = createConfiguration();
- URI uri = URI.create(fullPath);
- fileSystem = FileSystem.get(uri, conf);
- auditPath = new Path(fullPath);
- fileSystemScheme = getFileSystemScheme();
- logger.info("Checking whether log file exists. "+ fileSystemScheme + "Path= " + fullPath + ", UGI=" + MiscUtil.getUGILoginUser());
+ Date currentTime = new Date();
+ String fileName = MiscUtil.replaceTokens(logFileNameFormat, currentTime.getTime());
+
+ parentFolder = MiscUtil.replaceTokens(logFolder, currentTime.getTime());
+ fullPath = parentFolder + Path.SEPARATOR + fileName;
+
+ String defaultPath = fullPath;
+
+ fileSystemScheme = getFileSystemScheme();
+
+ conf = createConfiguration();
+
+ URI uri = URI.create(fullPath);
+
+ fileSystem = FileSystem.get(uri, conf);
+
+ auditPath = new Path(fullPath);
+
+ logger.info("Checking whether log file exists. {} Path={}, UGI={}", fileSystemScheme, fullPath, MiscUtil.getUGILoginUser());
+
int i = 0;
+
while (fileSystem.exists(auditPath)) {
i++;
+
int lastDot = defaultPath.lastIndexOf('.');
String baseName = defaultPath.substring(0, lastDot);
String extension = defaultPath.substring(lastDot);
- fullPath = baseName + "." + i + extension;
- auditPath = new Path(fullPath);
- logger.info("Checking whether log file exists. "+ fileSystemScheme + "Path= " + fullPath);
+
+ fullPath = baseName + "." + i + extension;
+ auditPath = new Path(fullPath);
+
+ logger.info("Checking whether log file exists. {} Path={}", fileSystemScheme, fullPath);
}
- logger.info("Log file doesn't exists. Will create and use it. "+ fileSystemScheme + "Path= " + fullPath);
+
+ logger.info("Log file doesn't exists. Will create and use it. {} Path={}", fileSystemScheme, fullPath);
// Create parent folders
createParents(auditPath, fileSystem);
currentFileName = fullPath;
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.createFileSystemFolders()");
- }
+ logger.debug("<== AbstractRangerAuditWriter.createFileSystemFolders()");
}
- public Configuration createConfiguration() {
+ public Configuration createConfiguration() {
Configuration conf = new Configuration();
+
for (Map.Entry<String, String> entry : auditConfigs.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
+
// for ease of install config file may contain properties with empty value, skip those
if (StringUtils.isNotEmpty(value)) {
conf.set(key, value);
}
- logger.info("Adding property to "+ fileSystemScheme + " + config: " + key + " => " + value);
+
+ logger.info("Adding property to {} + config: {} => {}", fileSystemScheme, key, value);
}
- logger.info("Returning " + fileSystemScheme + "Filesystem Config: " + conf.toString());
+ logger.info("Returning {} Filesystem Config: {}", fileSystemScheme, conf);
+
return conf;
}
public void createParents(Path pathLogfile, FileSystem fileSystem) throws Exception {
- logger.info("Creating parent folder for " + pathLogfile);
+ logger.info("Creating parent folder for {}", pathLogfile);
+
Path parentPath = pathLogfile != null ? pathLogfile.getParent() : null;
- if (parentPath != null && fileSystem != null
- && !fileSystem.exists(parentPath)) {
+ if (parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) {
fileSystem.mkdirs(parentPath);
+ logger.info("Successfully created parent folder: {}", parentPath);
+ } else {
+ logger.info("Parent folder already exists or not required: {}", parentPath);
}
}
public void init(Properties props, String propPrefix) {
+ logger.debug("==> AbstractRangerAuditWriter.init()");
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.init()");
- }
+ String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_FILESYSTEM_DIR);
- String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_FILESYSTEM_DIR);
if (StringUtils.isEmpty(logFolderProp)) {
- logger.error("File destination folder is not configured. Please set "
- + propPrefix + "."
- + PROP_FILESYSTEM_DIR + ". name="
- + auditProviderName);
+ logger.error("File destination folder is not configured. Please set {}.{}. name={}", propPrefix, PROP_FILESYSTEM_DIR, auditProviderName);
+
return;
}
String logSubFolder = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_FILESYSTEM_SUBDIR);
+
if (StringUtils.isEmpty(logSubFolder)) {
logSubFolder = "%app-type%/%time:yyyyMMdd%";
}
@@ -180,75 +232,68 @@ public void init(Properties props, String propPrefix) {
}
reUseLastLogFile = MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_IS_APPEND_ENABLED, false);
- logFolder = logFolderProp + "/" + logSubFolder;
+ logFolder = logFolderProp + "/" + logSubFolder;
- logger.info("logFolder={}, destName={}", logFolder, auditProviderName);
- logger.info("logFileNameFormat={}, destName={}", logFileNameFormat, auditProviderName);
- logger.info("config={}", auditConfigs.toString());
+ logger.info("logFolder = {}, destName = {}", logFolder, auditProviderName);
+ logger.info("logFileNameFormat = {}, destName = {}", logFileNameFormat, auditProviderName);
+ logger.info("config = {}", auditConfigs);
logger.info("isAppendEnabled = {}", reUseLastLogFile);
rolloverPeriod = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_FILESYSTEM_ROLLOVER_PERIOD);
rollingTimeUtil = RollingTimeUtil.getInstance();
- //file.rollover.period is used for rolling over. If it could compute the next roll over time using file.rollover.period
- //it fall back to use file.rollover.sec for find next rollover time. If still couldn't find default will be 1day window
- //for rollover.
- if(StringUtils.isEmpty(rolloverPeriod) ) {
+ //file.rollover.period is used for rolling over. If it could compute the next rollover time using file.rollover.period
+ //it fallbacks to use file.rollover.sec for find next rollover time. If still couldn't find default will be 1day window for rollover.
+ if (StringUtils.isEmpty(rolloverPeriod)) {
rolloverPeriod = rollingTimeUtil.convertRolloverSecondsToRolloverPeriod(fileRolloverSec);
}
try {
nextRollOverTime = rollingTimeUtil.computeNextRollingTime(rolloverPeriod);
- } catch ( Exception e) {
- logger.warn("Rollover by file.rollover.period failed...will be using the file.rollover.sec for "+ fileSystemScheme + " audit file rollover...", e);
+ } catch (Exception e) {
+ logger.warn("Rollover by file.rollover.period failed...will be using the file.rollover.sec for {} audit file rollover...", fileSystemScheme, e);
+
rollOverByDuration = true;
nextRollOverTime = rollOverByDuration();
}
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.init()");
- }
-
+ logger.debug("<== AbstractRangerAuditWriter.init()");
}
public void closeFileIfNeeded() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.closeFileIfNeeded()");
- }
+ logger.debug("==> AbstractRangerAuditWriter.closeFileIfNeeded()");
if (logWriter == null) {
- if (logger.isDebugEnabled()){
- logger.debug("Log writer is null, aborting rollover condition check!");
- }
+ logger.debug("Log writer is null, aborting rollover condition check!");
+
return;
}
- if ( System.currentTimeMillis() >= nextRollOverTime.getTime() ) {
+ if (System.currentTimeMillis() >= nextRollOverTime.getTime()) {
logger.info("Closing file. Rolling over. name = {}, fileName = {}", auditProviderName, currentFileName);
+
logWriter.flush();
+
closeWriter();
resetWriter();
setNextRollOverTime();
- currentFileName = null;
+ currentFileName = null;
auditPath = null;
fullPath = null;
}
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.closeFileIfNeeded()");
- }
+ logger.debug("<== AbstractRangerAuditWriter.closeFileIfNeeded()");
}
- public Date rollOverByDuration() {
- long rollOverTime = rollingTimeUtil.computeNextRollingTime(fileRolloverSec,nextRollOverTime);
+ public Date rollOverByDuration() {
+ long rollOverTime = rollingTimeUtil.computeNextRollingTime(fileRolloverSec, nextRollOverTime);
+
return new Date(rollOverTime);
}
public PrintWriter createWriter() throws Exception {
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.createWriter()");
- }
+ logger.debug("==> AbstractRangerAuditWriter.createWriter()");
if (logWriter == null) {
boolean appendMode = false;
@@ -256,30 +301,33 @@ public PrintWriter createWriter() throws Exception {
// if append is supported and enabled via config param, reuse last log file
if (auditPath != null && reUseLastLogFile && isAppendEnabled()) {
try {
- ostream = fileSystem.append(auditPath);
+ ostream = fileSystem.append(auditPath);
appendMode = true;
logger.info("Appending to last log file. auditPath = {}", fullPath);
- } catch (Exception e){
+ } catch (Exception e) {
logger.error("Failed to append to file {} due to {}", fullPath, e.getMessage());
logger.info("Falling back to create a new log file!");
- appendMode = false;
}
}
+
if (!appendMode) {
// Create the file to write
- logger.info("Creating new log file. auditPath = {}", fullPath);
createFileSystemFolders();
+
+ logger.info("Creating new log file. fullPath = {}", fullPath);
+
ostream = fileSystem.create(auditPath);
+ logger.info("Successfully created {} output stream for file: {}", fileSystemScheme, fullPath);
}
logWriter = new PrintWriter(ostream);
isHFlushCapableStream = ostream.hasCapability(StreamCapabilities.HFLUSH);
- }
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.createWriter()");
+ logger.info("{} audit writer initialized successfully. File: {}, HFlush capable: {}", fileSystemScheme, fullPath, isHFlushCapableStream);
}
+ logger.debug("<== AbstractRangerAuditWriter.createWriter()");
+
return logWriter;
}
@@ -287,9 +335,7 @@ public PrintWriter createWriter() throws Exception {
* Closes the writer after writing audits
**/
public void closeWriter() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.closeWriter()");
- }
+ logger.debug("==> AbstractRangerAuditWriter.closeWriter()");
if (ostream != null) {
try {
@@ -298,89 +344,50 @@ public void closeWriter() {
logger.error("Error closing the stream {}", e.getMessage());
}
}
- if (logWriter != null)
- logWriter.close();
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.closeWriter()");
+ if (logWriter != null) {
+ logWriter.close();
}
+
+ logger.debug("<== AbstractRangerAuditWriter.closeWriter()");
}
public void resetWriter() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.resetWriter()");
- }
+ logger.debug("==> AbstractRangerAuditWriter.resetWriter()");
logWriter = null;
- ostream = null;
+ ostream = null;
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.resetWriter()");
- }
- }
-
- @Override
- public void flush() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.flush() " + fileSystemScheme);
- }
- if (ostream != null) {
- try {
- synchronized (this) {
- if (ostream != null)
- // 1) PrinterWriter does not have bufferring of its own so
- // we need to flush its underlying stream
- // 2) HDFS flush() does not really flush all the way to disk.
- if (isHFlushCapableStream) {
- //Checking HFLUSH capability of the stream because of HADOOP-13327.
- //For S3 filesysttem, hflush throws UnsupportedOperationException and hence we call flush.
- ostream.hflush();
- } else {
- ostream.flush();
- }
- if (logger.isDebugEnabled()) {
- logger.debug("Flush " + fileSystemScheme + " audit logs completed.....");
- }
- }
- } catch (IOException e) {
- logger.error("Error on flushing log writer: " + e.getMessage() +
- "\nException will be ignored. name=" + auditProviderName + ", fileName=" + currentFileName);
- }
- }
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.flush()");
- }
+ logger.debug("<== AbstractRangerAuditWriter.resetWriter()");
}
public boolean logFileToHDFS(File file) throws Exception {
+ logger.debug("==> AbstractRangerAuditWriter.logFileToHDFS()");
+
boolean ret = false;
- if (logger.isDebugEnabled()) {
- logger.debug("==> AbstractRangerAuditWriter.logFileToHDFS()");
- }
if (logWriter == null) {
// Create the file to write
createFileSystemFolders();
- logger.info("Copying the Audit File" + file.getName() + " to HDFS Path" + fullPath);
+
+ logger.info("Copying the Audit File {} to HDFS Path {}", file.getName(), fullPath);
+
Path destPath = new Path(fullPath);
- ret = FileUtil.copy(file,fileSystem,destPath,false,conf);
- }
- if (logger.isDebugEnabled()) {
- logger.debug("<== AbstractRangerAuditWriter.logFileToHDFS()");
+ ret = FileUtil.copy(file, fileSystem, destPath, false, conf);
}
+
+ logger.debug("<== AbstractRangerAuditWriter.logFileToHDFS()");
+
return ret;
}
public String getFileSystemScheme() {
- String ret = null;
- ret = logFolder.substring(0, (logFolder.indexOf(":")));
- ret = ret.toUpperCase();
- return ret;
+ return logFolder.substring(0, (logFolder.indexOf(":"))).toUpperCase();
}
public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
+ this.fileExtension = fileExtension;
}
private void setNextRollOverTime() {
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosAction.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosAction.java
index 1bbbca8d1..e53eb077c 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosAction.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosAction.java
@@ -23,6 +23,7 @@
import org.slf4j.Logger;
import javax.security.auth.login.LoginException;
+
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
@@ -30,42 +31,54 @@
* Helper class for processors to perform an action as a KerberosUser.
*/
public class KerberosAction<T> {
-
- private final KerberosUser kerberosUser;
+ private final KerberosUser kerberosUser;
private final PrivilegedExceptionAction<T> action;
- private final Logger logger;
+ private final Logger logger;
- public KerberosAction(final KerberosUser kerberosUser,
- final PrivilegedExceptionAction<T> action,
- final Logger logger) {
+ public KerberosAction(final KerberosUser kerberosUser, final PrivilegedExceptionAction<T> action, final Logger logger) {
this.kerberosUser = kerberosUser;
- this.action = action;
- this.logger = logger;
+ this.action = action;
+ this.logger = logger;
+
Validate.notNull(this.kerberosUser);
Validate.notNull(this.action);
Validate.notNull(this.logger);
}
+ /**
+ * Runs {@code action} as {@code kerberosUser}: lazy login, proactive TGT
+ * refresh at 80% lifetime ({@link KerberosUser#checkTGTAndRelogin()}),
+ * then the privileged action. On {@link SecurityException}, relogin once via
+ * {@link AbstractKerberosUser#performRelogin()} (keytab-safe) and retry.
+ *
+ * @return the result of {@code action}
+ * @throws Exception if login, relogin, or the privileged action fails
+ */
public T execute() throws Exception {
T result;
+
// lazily login the first time the processor executes
if (!kerberosUser.isLoggedIn()) {
try {
kerberosUser.login();
- logger.info("Successful login for " + kerberosUser.getPrincipal());
+
+ logger.info("Successful login for {}", kerberosUser.getPrincipal());
} catch (LoginException e) {
throw new Exception("Login failed due to: " + e.getMessage(), e);
}
}
- // check if we need to re-login, will only happen if re-login window is reached (80% of TGT life)
+ // check if we need to re-login, will only happen if re-login window is
+ // reached (80% of TGT life)
try {
kerberosUser.checkTGTAndRelogin();
} catch (LoginException e) {
throw new Exception("Relogin check failed due to: " + e.getMessage(), e);
}
- // attempt to execute the action, if an exception is caught attempt to logout/login and retry
+ // On SecurityException, relogin and retry once. Use performRelogin()
+ // for AbstractKerberosUser so keytab clients skip logout first; bare
+ // logout/login reproduces "No key to store" when useTicketCache is true.
try {
result = kerberosUser.doAs(action);
} catch (SecurityException se) {
@@ -73,11 +86,17 @@ public T execute() throws Exception {
logger.debug("", se);
try {
- kerberosUser.logout();
- kerberosUser.login();
+ if (kerberosUser instanceof AbstractKerberosUser) {
+ ((AbstractKerberosUser) kerberosUser).performRelogin();
+ } else {
+ kerberosUser.logout();
+ kerberosUser.login();
+ }
+
result = kerberosUser.doAs(action);
} catch (Exception e) {
- throw new Exception("Retrying privileged action failed due to: " + e.getMessage(), e);
+ throw new Exception(
+ "Retrying privileged action failed due to: " + e.getMessage(), e);
}
} catch (PrivilegedActionException pae) {
final Exception cause = pae.getException();
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosJAASConfigUser.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosJAASConfigUser.java
index 266772160..d8b1c00e6 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosJAASConfigUser.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/KerberosJAASConfigUser.java
@@ -35,12 +35,16 @@
public class KerberosJAASConfigUser extends AbstractKerberosUser {
private static final Logger LOG = LoggerFactory.getLogger(KerberosJAASConfigUser.class);
+ private static final String JAAS_USE_KEYTAB = "useKeyTab";
+
private final String configName;
private final Configuration config;
+ private final boolean optionUseKeyTab;
public KerberosJAASConfigUser(final String configName, final Configuration config) {
- this.configName = configName;
- this.config = config;
+ this.configName = configName;
+ this.config = config;
+ this.optionUseKeyTab = getBooleanOption(JAAS_USE_KEYTAB);
}
@@ -62,6 +66,36 @@ public String getPrincipal() {
return ret;
}
+ /**
+ * Solr/Kafka outbound JAAS clients (audit dispatcher, plugin Solr
+ * destination) use {@code useKeyTab=true}. Opt those principals into
+ * in-place keytab relogin so shipped {@code useTicketCache=true} does not
+ * fail at TGT renewal with {@code "No key to store"}.
+ */
+ @Override
+ protected boolean useKeytabRelogin() {
+ return optionUseKeyTab;
+ }
+
+ private boolean getBooleanOption(String optionName) {
+ boolean ret = false;
+ AppConfigurationEntry[] entries = config.getAppConfigurationEntry(configName);
+
+ if (entries != null) {
+ for (AppConfigurationEntry entry : entries) {
+ Object value = entry.getOptions().get(optionName);
+
+ if (value != null && Boolean.parseBoolean(value.toString())) {
+ ret = true;
+
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
+
@Override
protected LoginContext createLoginContext(Subject subject) throws LoginException {
if (LOG.isDebugEnabled()) {
@@ -75,4 +109,3 @@ protected LoginContext createLoginContext(Subject subject) throws LoginException
return new LoginContext(configName, subject, null, config);
}
}
-
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
index 9161b76ed..c8b98fa75 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
@@ -27,14 +27,14 @@
import java.io.File;
import java.io.PrintWriter;
import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
/**
* Writes the Ranger audit to HDFS as JSON text
@@ -42,10 +42,10 @@
public class RangerJSONAuditWriter extends AbstractRangerAuditWriter {
private static final Logger logger = LoggerFactory.getLogger(RangerJSONAuditWriter.class);
- public static final String PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER = "file.rollover.enable.periodic.rollover";
- public static final String PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME = "file.rollover.periodic.rollover.check.sec";
+ public static final String PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER = "file.rollover.enable.periodic.rollover";
+ public static final String PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME = "file.rollover.periodic.rollover.check.sec";
- protected String JSON_FILE_EXTENSION = ".log";
+ protected static final String JSON_FILE_EXTENSION = ".log";
/*
Time frequency of next occurrence of periodic rollover check. By Default every 60 seconds the check is done if enabled
@@ -53,77 +53,87 @@ public class RangerJSONAuditWriter extends AbstractRangerAuditWriter {
private long periodicRollOverCheckTimeinSec;
public void init(Properties props, String propPrefix, String auditProviderName, Map<String, String> auditConfigs) {
- if (logger.isDebugEnabled()) {
- logger.debug("==> RangerJSONAuditWriter.init()");
- }
+ logger.debug("==> RangerJSONAuditWriter.init()");
+
init();
+
super.init(props, propPrefix, auditProviderName, auditConfigs);
// start AuditFilePeriodicRollOverTask if enabled.
boolean enableAuditFilePeriodicRollOver = MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER, false);
if (enableAuditFilePeriodicRollOver) {
periodicRollOverCheckTimeinSec = MiscUtil.getLongProperty(props, propPrefix + "." + PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME, 60L);
+
try {
- if (logger.isDebugEnabled()) {
- logger.debug("rolloverPeriod: " + rolloverPeriod + " nextRollOverTime: " + nextRollOverTime + " periodicRollOverTimeinSec: " + periodicRollOverCheckTimeinSec);
- }
+ logger.debug("rolloverPeriod: {} nextRollOverTime: {} periodicRollOverTimeinSec: {}", rolloverPeriod, nextRollOverTime, periodicRollOverCheckTimeinSec);
+
startAuditFilePeriodicRollOverTask();
} catch (Exception e) {
logger.warn("Error enabling audit file perodic rollover..! Default behavior will be");
}
}
- if (logger.isDebugEnabled()) {
- logger.debug("<== RangerJSONAuditWriter.init()");
- }
+ logger.debug("<== RangerJSONAuditWriter.init()");
+ }
+
+ public void flush() {
+ logger.debug("==> JSONWriter.flush() called. name={}", auditProviderName);
+
+ super.flush();
+
+ logger.debug("<== JSONWriter.flush()");
}
public void init() {
setFileExtension(JSON_FILE_EXTENSION);
}
- synchronized public boolean logJSON(final Collection<String> events) throws Exception {
+ public synchronized boolean logJSON(final Collection<String> events) throws Exception {
PrintWriter out = null;
+
try {
- if (logger.isDebugEnabled()) {
- logger.debug("UGI = {}, will write to HDFS file = {}", MiscUtil.getUGILoginUser(), currentFileName);
- }
- out = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction<PrintWriter>() {
- @Override
- public PrintWriter run() throws Exception {
- PrintWriter out = null;
-
- if (CollectionUtils.isEmpty(events)) {
- closeFileIfNeeded();
- } else {
- out = getLogFileStream();
-
- for (String event : events) {
- out.println(event);
- }
+ logger.debug("UGI = {}, will write to HDFS file = {}", MiscUtil.getUGILoginUser(), currentFileName);
+
+ out = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<PrintWriter>) () -> {
+ PrintWriter out1 = null;
+
+ if (CollectionUtils.isEmpty(events)) {
+ closeFileIfNeeded();
+ } else {
+ out1 = getLogFileStream();
+
+ logger.debug("Writing {} audit events to HDFS file: {}", events.size(), currentFileName);
+
+ for (String event : events) {
+ out1.println(event);
}
- return out;
- };
+ logger.debug("Successfully wrote {} audit events to HDFS", events.size());
+ }
+
+ return out1;
});
+
// flush and check the stream for errors
if (out != null && out.checkError()) {
// In theory, this count may NOT be accurate as part of the messages may have been successfully written.
// However, in practice, since client does buffering, either all or none would succeed.
logger.error("Stream encountered errors while writing audits to HDFS!");
+
closeWriter();
resetWriter();
+
return false;
}
} catch (Exception e) {
logger.error("Exception encountered while writing audits to HDFS!", e);
closeWriter();
resetWriter();
+
return false;
} finally {
- if (logger.isDebugEnabled()) {
- logger.debug("Flushing HDFS audit. Event Size:" + events.size());
- }
+ logger.debug("Flushing HDFS audit. Event Size:{}", events.size());
+
if (out != null) {
out.flush();
}
@@ -133,111 +143,88 @@ public PrintWriter run() throws Exception {
}
@Override
- public boolean log(Collection<String> events) throws Exception {
+ public boolean log(Collection<String> events) throws Exception {
return logJSON(events);
}
- synchronized public boolean logAsFile(final File file) throws Exception {
- boolean ret = false;
- if (logger.isDebugEnabled()) {
- logger.debug("UGI=" + MiscUtil.getUGILoginUser()
- + ". Will write to HDFS file=" + currentFileName);
- }
- Boolean retVal = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction<Boolean>() {
- @Override
- public Boolean run() throws Exception {
- boolean ret = logFileToHDFS(file);
- return Boolean.valueOf(ret);
- };
- });
- ret = retVal.booleanValue();
- logger.info("Flushing HDFS audit File :" + file.getAbsolutePath() + file.getName());
-
- return ret;
- }
-
@Override
public boolean logFile(File file) throws Exception {
return logAsFile(file);
}
- synchronized public PrintWriter getLogFileStream() throws Exception {
- closeFileIfNeeded();
-
- // Either there are no open log file or the previous one has been rolled over
- PrintWriter logWriter = createWriter();
- return logWriter;
- }
-
- public void flush() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> JSONWriter.flush() called. name=" + auditProviderName);
- }
- super.flush();
- if (logger.isDebugEnabled()) {
- logger.debug("<== JSONWriter.flush()");
- }
- }
-
@Override
public void start() {
// nothing to start
}
@Override
- synchronized public void stop() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> JSONWriter.stop()");
- }
+ public synchronized void stop() {
+ logger.debug("==> JSONWriter.stop()");
+
if (logWriter != null) {
try {
logWriter.flush();
logWriter.close();
} catch (Throwable t) {
- logger.error("Error on closing log writter. Exception will be ignored. name="
- + auditProviderName + ", fileName=" + currentFileName);
+ logger.error("Error on closing log writter. Exception will be ignored. name={}, fileName={}", auditProviderName, currentFileName);
}
+
logWriter = null;
- ostream = null;
- }
- if (logger.isDebugEnabled()) {
- logger.debug("<== JSONWriter.stop()");
+ ostream = null;
}
+
+ logger.debug("<== JSONWriter.stop()");
+ }
+
+ public synchronized boolean logAsFile(final File file) throws Exception {
+ logger.debug("UGI={}. Will write to HDFS file={}", MiscUtil.getUGILoginUser(), currentFileName);
+
+ boolean ret = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<Boolean>) () -> logFileToHDFS(file));
+
+ logger.info("Flushing HDFS audit File :{}{}", file.getAbsolutePath(), file.getName());
+
+ return ret;
+ }
+
+ public synchronized PrintWriter getLogFileStream() throws Exception {
+ closeFileIfNeeded();
+
+ // Either there are no open log file or the previous one has been rolled over
+ return createWriter();
}
private void startAuditFilePeriodicRollOverTask() {
ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new AuditFilePeriodicRollOverTaskThreadFactory());
- if (logger.isDebugEnabled()) {
- logger.debug("HDFSAuditDestination.startAuditFilePeriodicRollOverTask() strated.." + "Audit File rollover happens every " + rolloverPeriod );
- }
+ logger.debug("HDFSAuditDestination.startAuditFilePeriodicRollOverTask() strated..Audit File rollover happens every {}", rolloverPeriod);
executorService.scheduleAtFixedRate(new AuditFilePeriodicRollOverTask(), 0, periodicRollOverCheckTimeinSec, TimeUnit.SECONDS);
}
- class AuditFilePeriodicRollOverTaskThreadFactory implements ThreadFactory {
+ static class AuditFilePeriodicRollOverTaskThreadFactory implements ThreadFactory {
//Threadfactory to create a daemon Thread.
+ @Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, "AuditFilePeriodicRollOverTask");
+
t.setDaemon(true);
+
return t;
}
}
private class AuditFilePeriodicRollOverTask implements Runnable {
+ @Override
public void run() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> AuditFilePeriodicRollOverTask.run()");
- }
+ logger.debug("==> AuditFilePeriodicRollOverTask.run()");
+
try {
logJSON(Collections.emptyList());
} catch (Exception excp) {
logger.error("AuditFilePeriodicRollOverTask Failed. Aborting..", excp);
}
- if (logger.isDebugEnabled()) {
- logger.debug("<== AuditFilePeriodicRollOverTask.run()");
- }
+
+ logger.debug("<== AuditFilePeriodicRollOverTask.run()");
}
}
-
}
diff --git a/agents-audit/dest-os/pom.xml b/agents-audit/dest-os/pom.xml
new file mode 100644
index 000000000..fd14f9bea
--- /dev/null
+++ b/agents-audit/dest-os/pom.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>2.10.0-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-dest-os</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Ranger - Audit Destination OpenSearch</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ <version>${httpcomponents.httpasyncclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-nio</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j-api.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/agents-audit/dest-os/src/main/java/org/apache/ranger/audit/destination/OpenSearchAuditDestination.java b/agents-audit/dest-os/src/main/java/org/apache/ranger/audit/destination/OpenSearchAuditDestination.java
new file mode 100644
index 000000000..f397dfa81
--- /dev/null
+++ b/agents-audit/dest-os/src/main/java/org/apache/ranger/audit/destination/OpenSearchAuditDestination.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.destination;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.config.Lookup;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.nio.entity.NStringEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.authorization.credutils.CredentialsProviderUtil;
+import org.apache.ranger.authorization.credutils.kerberos.KerberosCredentialsProvider;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TimeZone;
+import java.util.UUID;
+
+public class OpenSearchAuditDestination extends AuditDestination {
+ private static final Logger LOG = LoggerFactory.getLogger(OpenSearchAuditDestination.class);
+
+ public static final String CONFIG_PREFIX = "ranger.audit.opensearch";
+ public static final String CONFIG_URLS = "urls";
+ public static final String CONFIG_PORT = "port";
+ public static final String CONFIG_USER = "user";
+ public static final String CONFIG_PASSWORD = "password";
+ public static final String CONFIG_PROTOCOL = "protocol";
+ public static final String CONFIG_INDEX = "index";
+ public static final String CONFIG_AUTH_TYPE = "authentication.type";
+ public static final String CONFIG_KERBEROS_PRINCIPAL = "kerberos.principal";
+ public static final String CONFIG_KERBEROS_KEYTAB = "kerberos.keytab";
+ public static final String DEFAULT_INDEX = "ranger_audits";
+
+ public static final String AUTH_TYPE_KERBEROS = "kerberos";
+ public static final String AUTH_TYPE_BASIC = "basic";
+ public static final String AUTH_TYPE_NONE = "none";
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final ThreadLocal<SimpleDateFormat> DATE_FORMAT = ThreadLocal.withInitial(() -> {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ return sdf;
+ });
+
+ private volatile RestClient client;
+
+ private String index;
+ private String user;
+ private String password;
+ private String protocol;
+ private String urls;
+ private int port;
+ private String authType;
+ private String kerberosPrincipal;
+ private String kerberosKeytab;
+
+ public OpenSearchAuditDestination() {
+ propPrefix = CONFIG_PREFIX;
+ }
+
+ @Override
+ public void init(Properties props, String propPrefix) {
+ super.init(props, propPrefix);
+
+ this.urls = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_URLS, "localhost");
+ this.port = MiscUtil.getIntProperty(props, propPrefix + "." + CONFIG_PORT, 9200);
+ this.protocol = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_PROTOCOL, "http");
+ this.user = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_USER, "");
+ this.password = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_PASSWORD, "");
+ this.index = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_INDEX, DEFAULT_INDEX);
+
+ this.authType = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_AUTH_TYPE, "");
+ this.kerberosPrincipal = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_KERBEROS_PRINCIPAL, "");
+ this.kerberosKeytab = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_KERBEROS_KEYTAB, "");
+
+ LOG.info("OpenSearchAuditDestination.init(): urls={}, port={}, index={}", urls, port, index);
+
+ getClient();
+ }
+
+ @Override
+ public void stop() {
+ logStatus();
+
+ if (client != null) {
+ try {
+ client.close();
+ } catch (Exception e) {
+ LOG.error("Error closing OpenSearch client", e);
+ }
+ }
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ if (events == null || events.isEmpty()) {
+ return true;
+ }
+
+ boolean ret = false;
+ RestClient currentClient = getClient();
+
+ if (currentClient == null) {
+ LOG.error("OpenSearch client is null. Cannot write audit events.");
+ } else {
+ try {
+ StringBuilder bulk = new StringBuilder();
+
+ for (AuditEventBase event : events) {
+ AuthzAuditEvent auditEvent = (AuthzAuditEvent) event;
+ Map<String, Object> doc = toDoc(auditEvent);
+ String id = (String) doc.get("id");
+
+ if (StringUtils.isBlank(id)) {
+ id = UUID.randomUUID().toString();
+
+ doc.put("id", id);
+ }
+
+ Map<String, Object> indexProps = new HashMap<>();
+
+ indexProps.put("_index", index);
+ indexProps.put("_id", id);
+
+ Map<String, Object> indexAction = new HashMap<>();
+ indexAction.put("index", indexProps);
+ bulk.append(MAPPER.writeValueAsString(indexAction)).append('\n');
+ bulk.append(MAPPER.writeValueAsString(doc)).append('\n');
+ }
+
+ Request request = new Request("POST", "/_bulk");
+
+ request.setEntity(new NStringEntity(bulk.toString(), ContentType.create("application/x-ndjson", StandardCharsets.UTF_8)));
+
+ Response response = currentClient.performRequest(request);
+ int statusCode = response.getStatusLine().getStatusCode();
+
+ if (statusCode >= 400) {
+ LOG.error("OpenSearch bulk request failed: HTTP {}", statusCode);
+ } else {
+ String responseBody = EntityUtils.toString(response.getEntity());
+
+ @SuppressWarnings("unchecked")
+ Map<String, Object> responseMap = MAPPER.readValue(responseBody, Map.class);
+
+ if (Boolean.TRUE.equals(responseMap.get("errors"))) {
+ LOG.error("OpenSearch bulk response contains item-level errors");
+ } else {
+ ret = true;
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to write audit events to OpenSearch", e);
+ }
+ }
+
+ if (ret) {
+ addSuccessCount(events.size());
+ } else {
+ addFailedCount(events.size());
+ }
+
+ return ret;
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+ synchronized RestClient getClient() {
+ if (client == null) {
+ if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls)) {
+ LOG.error("OpenSearch URLs not configured");
+
+ return null;
+ }
+
+ HttpHost[] hosts = Arrays.stream(urls.split(",")).map(String::trim).filter(h -> !h.isEmpty()).map(h -> new HttpHost(h, port, protocol)).toArray(HttpHost[]::new);
+ RestClientBuilder builder = RestClient.builder(hosts);
+ String resolvedAuthType = resolveAuthType(authType, user, password);
+
+ if (AUTH_TYPE_KERBEROS.equals(resolvedAuthType)) {
+ String principal = isConfigured(kerberosPrincipal) ? kerberosPrincipal : user;
+ String keytab = isConfigured(kerberosKeytab) ? kerberosKeytab : password;
+
+ KerberosCredentialsProvider credentialsProvider = CredentialsProviderUtil.getKerberosCredentials(principal, keytab);
+ Lookup<AuthSchemeProvider> authRegistry = RegistryBuilder.<AuthSchemeProvider>create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
+
+ builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setDefaultAuthSchemeRegistry(authRegistry));
+
+ LOG.info("OpenSearch client configured with Kerberos authentication for principal: {}", principal);
+ } else if (AUTH_TYPE_BASIC.equals(resolvedAuthType)) {
+ CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+
+ credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password));
+
+ builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+
+ LOG.info("OpenSearch client configured with basic authentication for user: {}", user);
+ } else {
+ LOG.info("OpenSearch client configured without authentication");
+ }
+
+ client = builder.build();
+ }
+
+ return client;
+ }
+
+ public static boolean isConfigured(final String value) {
+ return StringUtils.isNotBlank(value) && !"NONE".equalsIgnoreCase(value.trim());
+ }
+
+ /**
+ * Resolves the authentication scheme. When {@code authentication.type} is set explicitly
+ * ({@value AUTH_TYPE_KERBEROS}/{@value AUTH_TYPE_BASIC}/{@value AUTH_TYPE_NONE}) it is honored as-is.
+ * When unset, the type is inferred for backward compatibility: a password pointing to an existing
+ * keytab file selects Kerberos, a user+password pair selects basic auth, otherwise no authentication.
+ */
+ public static String resolveAuthType(final String authType, final String user, final String password) {
+ if (StringUtils.isNotBlank(authType)) {
+ return authType.trim().toLowerCase(Locale.ROOT);
+ }
+
+ if (isConfigured(user) && isConfigured(password)) {
+ if (password.contains("keytab") && new File(password).exists()) {
+ return AUTH_TYPE_KERBEROS;
+ }
+
+ return AUTH_TYPE_BASIC;
+ }
+
+ return AUTH_TYPE_NONE;
+ }
+
+ Map<String, Object> toDoc(AuthzAuditEvent event) {
+ Map<String, Object> doc = new HashMap<>();
+
+ doc.put("id", event.getEventId());
+ doc.put("access", event.getAccessType());
+ doc.put("enforcer", event.getAclEnforcer());
+ doc.put("agent", event.getAgentId());
+ doc.put("repo", event.getRepositoryName());
+ doc.put("sess", event.getSessionId());
+ doc.put("reqUser", event.getUser());
+ doc.put("reqData", event.getRequestData());
+ doc.put("resource", event.getResourcePath());
+ doc.put("cliIP", event.getClientIP());
+ doc.put("logType", event.getLogType());
+ doc.put("result", event.getAccessResult());
+ doc.put("policy", event.getPolicyId());
+ doc.put("repoType", event.getRepositoryType());
+ doc.put("resType", event.getResourceType());
+ doc.put("reason", event.getResultReason());
+ doc.put("action", event.getAction());
+ doc.put("evtTime", formatDate(event.getEventTime()));
+ doc.put("seq_num", event.getSeqNum());
+ doc.put("event_count", event.getEventCount());
+ doc.put("event_dur_ms", event.getEventDurationMS());
+ doc.put("tags", event.getTags());
+ doc.put("datasets", event.getDatasets());
+ doc.put("projects", event.getProjects());
+ doc.put("cluster", event.getClusterName());
+ doc.put("zoneName", event.getZoneName());
+ doc.put("agentHost", event.getAgentHostname());
+ doc.put("policyVersion", event.getPolicyVersion());
+
+ return doc;
+ }
+
+ private static String formatDate(Date date) {
+ return date != null ? DATE_FORMAT.get().format(date) : null;
+ }
+}
diff --git a/agents-audit/dest-os/src/test/java/org/apache/ranger/audit/destination/TestOpenSearchAuditDestination.java b/agents-audit/dest-os/src/test/java/org/apache/ranger/audit/destination/TestOpenSearchAuditDestination.java
new file mode 100644
index 000000000..cffea80cc
--- /dev/null
+++ b/agents-audit/dest-os/src/test/java/org/apache/ranger/audit/destination/TestOpenSearchAuditDestination.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.destination;
+
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Date;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class TestOpenSearchAuditDestination {
+ private OpenSearchAuditDestination destination;
+
+ @BeforeEach
+ void setUp() {
+ destination = new OpenSearchAuditDestination();
+ }
+
+ @Test
+ void toDoc_mapsAllFieldsCorrectly() {
+ AuthzAuditEvent event = new AuthzAuditEvent();
+ event.setEventId("test-event-001");
+ event.setAccessType("read");
+ event.setAclEnforcer("ranger-acl");
+ event.setAgentId("hdfs-agent");
+ event.setRepositoryName("dev_hdfs");
+ event.setSessionId("sess-123");
+ event.setUser("testuser");
+ event.setRequestData("/tmp/testfile");
+ event.setResourcePath("/data/warehouse");
+ event.setClientIP("192.168.1.10");
+ event.setLogType("RangerAudit");
+ event.setAccessResult((short) 1);
+ event.setPolicyId(42L);
+ event.setRepositoryType(1);
+ event.setResourceType("path");
+ event.setResultReason("Allowed by policy");
+ event.setAction("read");
+ event.setEventTime(new Date(1700000000000L));
+ event.setSeqNum(10L);
+ event.setEventCount(1L);
+ event.setEventDurationMS(50L);
+ event.setClusterName("test-cluster");
+ event.setZoneName("zone1");
+ event.setAgentHostname("host1.example.com");
+ event.setPolicyVersion(5L);
+
+ Map<String, Object> doc = destination.toDoc(event);
+
+ assertEquals("test-event-001", doc.get("id"));
+ assertEquals("read", doc.get("access"));
+ assertEquals("ranger-acl", doc.get("enforcer"));
+ assertEquals("hdfs-agent", doc.get("agent"));
+ assertEquals("dev_hdfs", doc.get("repo"));
+ assertEquals("sess-123", doc.get("sess"));
+ assertEquals("testuser", doc.get("reqUser"));
+ assertEquals("/tmp/testfile", doc.get("reqData"));
+ assertEquals("/data/warehouse", doc.get("resource"));
+ assertEquals("192.168.1.10", doc.get("cliIP"));
+ assertEquals("RangerAudit", doc.get("logType"));
+ assertEquals((short) 1, doc.get("result"));
+ assertEquals(42L, doc.get("policy"));
+ assertEquals(1, doc.get("repoType"));
+ assertEquals("path", doc.get("resType"));
+ assertEquals("Allowed by policy", doc.get("reason"));
+ assertEquals("read", doc.get("action"));
+ assertNotNull(doc.get("evtTime"));
+ assertTrue(doc.get("evtTime").toString().contains("2023-11-14"));
+ assertEquals(10L, doc.get("seq_num"));
+ assertEquals(1L, doc.get("event_count"));
+ assertEquals(50L, doc.get("event_dur_ms"));
+ assertEquals("test-cluster", doc.get("cluster"));
+ assertEquals("zone1", doc.get("zoneName"));
+ assertEquals("host1.example.com", doc.get("agentHost"));
+ assertEquals(5L, doc.get("policyVersion"));
+ }
+
+ @Test
+ void toDoc_nullEventId_remainsNull() {
+ AuthzAuditEvent event = new AuthzAuditEvent();
+ event.setUser("testuser");
+
+ Map<String, Object> doc = destination.toDoc(event);
+
+ assertNull(doc.get("id"));
+ }
+
+ @Test
+ void toDoc_nullEventTime_formatsAsNull() {
+ AuthzAuditEvent event = new AuthzAuditEvent();
+ event.setEventId("evt-1");
+ event.setEventTime(null);
+
+ Map<String, Object> doc = destination.toDoc(event);
+
+ assertNull(doc.get("evtTime"));
+ }
+
+ @Test
+ void getClient_noneUrls_returnsNull() {
+ java.util.Properties props = new java.util.Properties();
+ props.setProperty(OpenSearchAuditDestination.CONFIG_PREFIX + ".urls", "NONE");
+
+ destination.init(props, OpenSearchAuditDestination.CONFIG_PREFIX);
+
+ assertNull(destination.getClient());
+ }
+
+ @Test
+ void log_nullClient_returnsFalse() {
+ java.util.Properties props = new java.util.Properties();
+ props.setProperty(OpenSearchAuditDestination.CONFIG_PREFIX + ".urls", "NONE");
+ destination.init(props, OpenSearchAuditDestination.CONFIG_PREFIX);
+
+ AuthzAuditEvent event = new AuthzAuditEvent();
+ event.setEventId("test-1");
+ event.setUser("user1");
+
+ java.util.Collection<org.apache.ranger.audit.model.AuditEventBase> events = java.util.Collections.singletonList(event);
+ boolean result = destination.log(events);
+
+ assertFalse(result);
+ }
+
+ @Test
+ void log_emptyEvents_returnsTrue() {
+ java.util.Collection<org.apache.ranger.audit.model.AuditEventBase> events = java.util.Collections.emptyList();
+ boolean result = destination.log(events);
+
+ assertTrue(result);
+ }
+
+ @Test
+ void log_nullEvents_returnsTrue() {
+ boolean result = destination.log((java.util.Collection<org.apache.ranger.audit.model.AuditEventBase>) null);
+
+ assertTrue(result);
+ }
+
+ @Test
+ void configConstants_matchExpectedValues() {
+ assertEquals("ranger.audit.opensearch", OpenSearchAuditDestination.CONFIG_PREFIX);
+ assertEquals("urls", OpenSearchAuditDestination.CONFIG_URLS);
+ assertEquals("port", OpenSearchAuditDestination.CONFIG_PORT);
+ assertEquals("user", OpenSearchAuditDestination.CONFIG_USER);
+ assertEquals("password", OpenSearchAuditDestination.CONFIG_PASSWORD);
+ assertEquals("protocol", OpenSearchAuditDestination.CONFIG_PROTOCOL);
+ assertEquals("index", OpenSearchAuditDestination.CONFIG_INDEX);
+ assertEquals("authentication.type", OpenSearchAuditDestination.CONFIG_AUTH_TYPE);
+ assertEquals("kerberos.principal", OpenSearchAuditDestination.CONFIG_KERBEROS_PRINCIPAL);
+ assertEquals("kerberos.keytab", OpenSearchAuditDestination.CONFIG_KERBEROS_KEYTAB);
+ assertEquals("ranger_audits", OpenSearchAuditDestination.DEFAULT_INDEX);
+ }
+
+ @Test
+ void resolveAuthType_explicitValuesAreHonored() {
+ assertEquals("kerberos", OpenSearchAuditDestination.resolveAuthType("kerberos", "", ""));
+ assertEquals("basic", OpenSearchAuditDestination.resolveAuthType("BASIC", "", ""));
+ assertEquals("none", OpenSearchAuditDestination.resolveAuthType("none", "u", "p"));
+ }
+
+ @Test
+ void resolveAuthType_inferredWhenTypeUnset() {
+ assertEquals("basic", OpenSearchAuditDestination.resolveAuthType("", "admin", "secret"));
+ assertEquals("none", OpenSearchAuditDestination.resolveAuthType("", "", ""));
+ assertEquals("none", OpenSearchAuditDestination.resolveAuthType("", "NONE", "NONE"));
+ }
+}
diff --git a/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index 6a60154f5..61ec23692 100644
--- a/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/dest-solr/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -25,26 +25,32 @@
import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.audit.utils.InMemoryJAASConfiguration;
import org.apache.ranger.audit.utils.KerberosAction;
-import org.apache.ranger.audit.utils.KerberosUser;
import org.apache.ranger.audit.utils.KerberosJAASConfigUser;
+import org.apache.ranger.audit.utils.KerberosUser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
-import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.security.auth.login.LoginException;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Field;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
@@ -54,439 +60,469 @@
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Properties;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.security.auth.login.LoginException;
+public class SolrAuditDestination extends AuditDestination {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrAuditDestination.class);
-import java.util.Arrays;
-import java.util.Optional;
+ public static final String PROP_SOLR_URLS = "urls";
+ public static final String PROP_SOLR_ZK = "zookeepers";
+ public static final String PROP_SOLR_COLLECTION = "collection";
+ public static final String PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG = "force.use.inmemory.jaas.config";
+ public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
+ public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
+ private volatile SolrClient solrClient;
+ private volatile KerberosUser kerberosUser;
-public class SolrAuditDestination extends AuditDestination {
- private static final Logger LOG = LoggerFactory
- .getLogger(SolrAuditDestination.class);
-
- public static final String PROP_SOLR_URLS = "urls";
- public static final String PROP_SOLR_ZK = "zookeepers";
- public static final String PROP_SOLR_COLLECTION = "collection";
- public static final String PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG = "force.use.inmemory.jaas.config";
-
- public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
- public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
-
- private volatile SolrClient solrClient = null;
- private volatile KerberosUser kerberosUser = null;
-
- public SolrAuditDestination() {
- }
-
- @Override
- public void init(Properties props, String propPrefix) {
- LOG.info("init() called");
- super.init(props, propPrefix);
- init();
- connect();
- }
-
- @Override
- public void stop() {
- LOG.info("SolrAuditDestination.stop() called..");
- logStatus();
-
- if (solrClient != null) {
- try {
- solrClient.close();
- } catch (IOException ioe) {
- LOG.error("Error while stopping slor!", ioe);
- } finally {
- solrClient = null;
- }
- }
-
- if (kerberosUser != null) {
- try {
- kerberosUser.logout();
- } catch (LoginException excp) {
- LOG.error("Error logging out keytab user", excp);
- } finally {
- kerberosUser = null;
- }
- }
- }
-
- synchronized void connect() {
- SolrClient me = solrClient;
- if (me == null) {
- synchronized(SolrAuditDestination.class) {
- me = solrClient;
- if (solrClient == null) {
- KeyManager[] kmList = getKeyManagers();
- TrustManager[] tmList = getTrustManagers();
- SSLContext sslContext = getSSLContext(kmList, tmList);
- if(sslContext != null) {
- SSLContext.setDefault(sslContext);
- }
- String urls = MiscUtil.getStringProperty(props, propPrefix
- + "." + PROP_SOLR_URLS);
- if (urls != null) {
- urls = urls.trim();
- }
- if (urls != null && urls.equalsIgnoreCase("NONE")) {
- urls = null;
- }
- List<String> solrURLs = new ArrayList<String>();
- String zkHosts = null;
- solrURLs = MiscUtil.toArray(urls, ",");
- zkHosts = MiscUtil.getStringProperty(props, propPrefix + "."
- + PROP_SOLR_ZK);
- if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) {
- zkHosts = null;
- }
- String collectionName = MiscUtil.getStringProperty(props,
- propPrefix + "." + PROP_SOLR_COLLECTION);
- if (collectionName == null
- || collectionName.equalsIgnoreCase("none")) {
- collectionName = DEFAULT_COLLECTION_NAME;
- }
-
- LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls
- + ", collectionName=" + collectionName);
-
- if (zkHosts != null && !zkHosts.isEmpty()) {
- LOG.info("Connecting to solr cloud using zkHosts="
- + zkHosts);
- try {
- Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder();
- SolrHttpClientBuilder kb = krbBuild.getBuilder();
- HttpClientUtil.setHttpClientBuilder(kb);
-
- final List<String> zkhosts = new ArrayList<String>(Arrays.asList(zkHosts.split(",")));
- final CloudSolrClient solrCloudClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction<CloudSolrClient>() {
- @Override
- public CloudSolrClient run() throws Exception {
- CloudSolrClient solrCloudClient = new CloudSolrClient.Builder(zkhosts, Optional.empty()).build();
- return solrCloudClient;
- }
- });
-
- solrCloudClient.setDefaultCollection(collectionName);
- me = solrClient = solrCloudClient;
- } catch (Throwable t) {
- LOG.error("Can't connect to Solr server. ZooKeepers="
- + zkHosts, t);
- }
- } else if (solrURLs != null && !solrURLs.isEmpty()) {
- try {
- Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder();
- LOG.info("Connecting to Solr using URLs=" + solrURLs);
- SolrHttpClientBuilder kb = krbBuild.getBuilder();
- HttpClientUtil.setHttpClientBuilder(kb);
- final List<String> solrUrls = solrURLs;
- final LBHttpSolrClient lbSolrClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction<LBHttpSolrClient>() {
- @Override
- public LBHttpSolrClient run() throws Exception {
- LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
- builder.withBaseSolrUrl(solrUrls.get(0));
- builder.withConnectionTimeout(1000);
- LBHttpSolrClient lbSolrClient = builder.build();
- return lbSolrClient;
- }
- });
-
- for (int i = 1; i < solrURLs.size(); i++) {
- lbSolrClient.addSolrServer(solrURLs.get(i));
- }
- me = solrClient = lbSolrClient;
- } catch (Throwable t) {
- LOG.error("Can't connect to Solr server. URL="
- + solrURLs, t);
- }
- }
- }
- }
- }
- }
-
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- boolean ret = false;
- try {
- logStatusIfRequired();
- addTotalCount(events.size());
-
- if (solrClient == null) {
- connect();
- if (solrClient == null) {
- // Solr is still not initialized. So need return error
- addDeferredCount(events.size());
- return ret;
- }
- }
-
- final Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
- for (AuditEventBase event : events) {
- AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
- // Convert AuditEventBase to Solr document
- SolrInputDocument document = toSolrDoc(authzEvent);
- docs.add(document);
- }
- try {
- final UpdateResponse response = addDocsToSolr(solrClient, docs);
-
- if (response.getStatus() != 0) {
- addFailedCount(events.size());
- logFailedEvent(events, response.toString());
- } else {
- addSuccessCount(events.size());
- ret = true;
- }
- } catch (SolrException ex) {
- addFailedCount(events.size());
- logFailedEvent(events, ex);
- }
- } catch (Throwable t) {
- addDeferredCount(events.size());
- logError("Error sending message to Solr", t);
- }
- return ret;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#flush()
- */
- @Override
- public void flush() {
-
- }
-
- SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", auditEvent.getEventId());
- doc.addField("access", auditEvent.getAccessType());
- doc.addField("enforcer", auditEvent.getAclEnforcer());
- doc.addField("agent", auditEvent.getAgentId());
- doc.addField("repo", auditEvent.getRepositoryName());
- doc.addField("sess", auditEvent.getSessionId());
- doc.addField("reqUser", auditEvent.getUser());
- doc.addField("reqData", auditEvent.getRequestData());
- doc.addField("resource", auditEvent.getResourcePath());
- doc.addField("cliIP", auditEvent.getClientIP());
- doc.addField("logType", auditEvent.getLogType());
- doc.addField("result", auditEvent.getAccessResult());
- doc.addField("policy", auditEvent.getPolicyId());
- doc.addField("repoType", auditEvent.getRepositoryType());
- doc.addField("resType", auditEvent.getResourceType());
- doc.addField("reason", auditEvent.getResultReason());
- doc.addField("action", auditEvent.getAction());
- doc.addField("evtTime", auditEvent.getEventTime());
- doc.addField("seq_num", auditEvent.getSeqNum());
- doc.setField("event_count", auditEvent.getEventCount());
- doc.setField("event_dur_ms", auditEvent.getEventDurationMS());
- doc.setField("tags", auditEvent.getTags());
- doc.setField("cluster", auditEvent.getClusterName());
- doc.setField("zoneName", auditEvent.getZoneName());
- doc.setField("agentHost", auditEvent.getAgentHostname());
- doc.setField("policyVersion", auditEvent.getPolicyVersion());
-
- return doc;
- }
-
- public boolean isAsync() {
- return true;
- }
-
- private void init() {
- LOG.info("==>SolrAuditDestination.init()" );
- try {
- // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
- // Acutal solrclient JAAS configs are read from the ranger-<component>-audit.xml present in components conf folder and set by InMemoryJAASConfiguration
- // Refer InMemoryJAASConfiguration doc for JAAS Configuration
- String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
- LOG.info("In solrAuditDestination.init() : JAAS Configuration set as [" + confFileName + "]");
- if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) {
- if ( MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG,false) ) {
- System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null");
- } else {
- LOG.warn("No Client JAAS config present in solr audit config. Ranger Audit to Kerberized Solr will fail...");
- }
- }
-
- LOG.info("Loading SolrClient JAAS config from Ranger audit config if present...");
-
- InMemoryJAASConfiguration conf = InMemoryJAASConfiguration.init(props);
-
- KerberosUser kerberosUser = new KerberosJAASConfigUser("Client", conf);
-
- if (kerberosUser.getPrincipal() != null) {
- this.kerberosUser = kerberosUser;
- }
- } catch (Exception e) {
- LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e);
- } finally {
- String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
- LOG.info("In solrAuditDestination.init() (finally) : JAAS Configuration set as [" + confFileName + "]");
- }
- LOG.info("<==SolrAuditDestination.init()" );
- }
-
- private KeyManager[] getKeyManagers() {
- KeyManager[] kmList = null;
- String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL);
- String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS;
- String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE);
- String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias);
- if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) {
- InputStream in = null;
-
- try {
- in = getFileInputStream(keyStoreFile);
-
- if (in != null) {
- String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE);
- keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT;
- KeyStore keyStore = KeyStore.getInstance(keyStoreType);
-
- keyStore.load(in, keyStoreFilepwd.toCharArray());
-
- KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
-
- keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
-
- kmList = keyManagerFactory.getKeyManagers();
- } else {
- LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]");
- }
- } catch (KeyStoreException e) {
- LOG.error("Unable to obtain from KeyStore :" + e.getMessage(), e);
- } catch (NoSuchAlgorithmException e) {
- LOG.error("SSL algorithm is NOT available in the environment", e);
- } catch (CertificateException e) {
- LOG.error("Unable to obtain the requested certification ", e);
- } catch (FileNotFoundException e) {
- LOG.error("Unable to find the necessary SSL Keystore Files", e);
- } catch (IOException e) {
- LOG.error("Unable to read the necessary SSL Keystore Files", e);
- } catch (UnrecoverableKeyException e) {
- LOG.error("Unable to recover the key from keystore", e);
- } finally {
- close(in, keyStoreFile);
- }
- }
-
- return kmList;
- }
-
- private TrustManager[] getTrustManagers() {
- TrustManager[] tmList = null;
- String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL);
- String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS;
- String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE);
- String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias);
- if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) {
- InputStream in = null;
-
- try {
- in = getFileInputStream(trustStoreFile);
-
- if (in != null) {
- String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE);
- trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT;
- KeyStore trustStore = KeyStore.getInstance(trustStoreType);
-
- trustStore.load(in, trustStoreFilepwd.toCharArray());
-
- TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
-
- trustManagerFactory.init(trustStore);
-
- tmList = trustManagerFactory.getTrustManagers();
- } else {
- LOG.error("Unable to obtain truststore from file [" + trustStoreFile + "]");
- }
- } catch (KeyStoreException e) {
- LOG.error("Unable to obtain from KeyStore", e);
- } catch (NoSuchAlgorithmException e) {
- LOG.error("SSL algorithm is NOT available in the environment :" + e.getMessage(), e);
- } catch (CertificateException e) {
- LOG.error("Unable to obtain the requested certification :" + e.getMessage(), e);
- } catch (FileNotFoundException e) {
- LOG.error("Unable to find the necessary SSL TrustStore File:" + trustStoreFile, e);
- } catch (IOException e) {
- LOG.error("Unable to read the necessary SSL TrustStore Files :" + trustStoreFile, e);
- } finally {
- close(in, trustStoreFile);
- }
- }
-
- return tmList;
- }
-
- private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) {
- SSLContext sslContext = null;
- try {
- sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
- if (sslContext != null) {
- sslContext.init(kmList, tmList, new SecureRandom());
- }
- } catch (NoSuchAlgorithmException e) {
- LOG.error("SSL algorithm is not available in the environment", e);
- } catch (KeyManagementException e) {
- LOG.error("Unable to initialise the SSLContext", e);
- }
- return sslContext;
- }
-
- private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection<SolrInputDocument> docs) throws Exception {
- final UpdateResponse ret;
-
- try {
- final PrivilegedExceptionAction<UpdateResponse> action = () -> solrClient.add(docs);
-
- if (kerberosUser != null) {
- // execute the privileged action as the given keytab user
- final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG);
-
- ret = (UpdateResponse) kerberosAction.execute();
- } else {
- ret = action.run();
- }
- } catch (Exception e) {
- throw e;
- }
-
- return ret;
- }
-
- private InputStream getFileInputStream(String fileName) throws IOException {
- InputStream in = null;
- if (StringUtils.isNotEmpty(fileName)) {
- File file = new File(fileName);
- if (file != null && file.exists()) {
- in = new FileInputStream(file);
- } else {
- in = ClassLoader.getSystemResourceAsStream(fileName);
- }
- }
- return in;
- }
-
- private void close(InputStream str, String filename) {
- if (str != null) {
- try {
- str.close();
- } catch (IOException excp) {
- LOG.error("Error while closing file: [" + filename + "]", excp);
- }
- }
- }
+ public SolrAuditDestination() {
+ }
+
+ @Override
+ public void init(Properties props, String propPrefix) {
+ LOG.info("init() called");
+
+ super.init(props, propPrefix);
+
+ init();
+ connect();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("SolrAuditDestination.stop() called..");
+
+ logStatus();
+
+ SolrClient solrClient = this.solrClient;
+
+ if (solrClient != null) {
+ try {
+ solrClient.close();
+ } catch (IOException ioe) {
+ LOG.error("Error while stopping solr!", ioe);
+ } finally {
+ this.solrClient = null;
+ }
+ }
+
+ KerberosUser kerberosUser = this.kerberosUser;
+
+ if (kerberosUser != null) {
+ try {
+ kerberosUser.logout();
+ } catch (LoginException excp) {
+ LOG.error("Error logging out keytab user", excp);
+ } finally {
+ this.kerberosUser = null;
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ boolean ret = false;
+
+ try {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ SolrClient solrClient = this.solrClient;
+
+ if (solrClient == null) {
+ connect();
+
+ solrClient = this.solrClient;
+
+ if (solrClient == null) {
+ // Solr is still not initialized. So need return error
+ addDeferredCount(events.size());
+
+ return ret;
+ }
+ }
+
+ final Collection<SolrInputDocument> docs = new ArrayList<>();
+
+ for (AuditEventBase event : events) {
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+
+ // Convert AuditEventBase to Solr document
+ SolrInputDocument document = toSolrDoc(authzEvent);
+
+ docs.add(document);
+ }
+
+ try {
+ final UpdateResponse response = addDocsToSolr(solrClient, docs);
+
+ if (response.getStatus() != 0) {
+ addFailedCount(events.size());
+
+ logFailedEvent(events, response.toString());
+ } else {
+ addSuccessCount(events.size());
+
+ ret = true;
+ }
+ } catch (SolrException ex) {
+ addFailedCount(events.size());
+ logFailedEvent(events, ex);
+ }
+ } catch (Throwable t) {
+ addDeferredCount(events.size());
+
+ logError("Error sending message to Solr", t);
+ }
+
+ return ret;
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+ synchronized void connect() {
+ SolrClient me = solrClient;
+
+ if (me == null) {
+ synchronized (SolrAuditDestination.class) {
+ me = solrClient;
+
+ if (me == null) {
+ KeyManager[] kmList = getKeyManagers();
+ TrustManager[] tmList = getTrustManagers();
+ SSLContext sslContext = getSSLContext(kmList, tmList);
+
+ if (sslContext != null) {
+ SSLContext.setDefault(sslContext);
+ }
+
+ String urls = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SOLR_URLS);
+
+ if (urls != null) {
+ urls = urls.trim();
+ }
+
+ if (urls != null && urls.equalsIgnoreCase("NONE")) {
+ urls = null;
+ }
+
+ List<String> solrURLs = MiscUtil.toArray(urls, ",");
+ String zkHosts = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SOLR_ZK);
+
+ if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) {
+ zkHosts = null;
+ }
+
+ String collectionName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SOLR_COLLECTION);
+
+ if (collectionName == null || collectionName.equalsIgnoreCase("none")) {
+ collectionName = DEFAULT_COLLECTION_NAME;
+ }
+
+ LOG.info("Solr zkHosts={}, solrURLs={}, collectionName={}", zkHosts, urls, collectionName);
+
+ if (zkHosts != null && !zkHosts.isEmpty()) {
+ LOG.info("Connecting to solr cloud using zkHosts={}", zkHosts);
+
+ try {
+ Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder();
+ SolrHttpClientBuilder kb = krbBuild.getBuilder();
+
+ HttpClientUtil.setHttpClientBuilder(kb);
+
+ final List<String> zkhosts = new ArrayList<>(Arrays.asList(zkHosts.split(",")));
+ final CloudSolrClient solrCloudClient = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<CloudSolrClient>) () -> new CloudSolrClient.Builder(zkhosts, Optional.empty()).build());
+
+ solrCloudClient.setDefaultCollection(collectionName);
+
+ me = solrCloudClient;
+ solrClient = me;
+ } catch (Throwable t) {
+ LOG.error("Can't connect to Solr server. ZooKeepers={}", zkHosts, t);
+ }
+ } else if (solrURLs != null && !solrURLs.isEmpty()) {
+ try {
+ Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder();
+ LOG.info("Connecting to Solr using URLs={}", solrURLs);
+
+ SolrHttpClientBuilder kb = krbBuild.getBuilder();
+
+ HttpClientUtil.setHttpClientBuilder(kb);
+
+ final List<String> solrUrls = solrURLs;
+ final LBHttpSolrClient lbSolrClient = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<LBHttpSolrClient>) () -> {
+ LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
+
+ builder.withBaseSolrUrl(solrUrls.get(0));
+ builder.withConnectionTimeout(1000);
+
+ return builder.build();
+ });
+
+ for (int i = 1; i < solrURLs.size(); i++) {
+ lbSolrClient.addSolrServer(solrURLs.get(i));
+ }
+
+ me = lbSolrClient;
+ solrClient = me;
+ } catch (Throwable t) {
+ LOG.error("Can't connect to Solr server. URL={}", solrURLs, t);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
+ SolrInputDocument doc = new SolrInputDocument();
+
+ doc.addField("id", auditEvent.getEventId());
+ doc.addField("access", auditEvent.getAccessType());
+ doc.addField("enforcer", auditEvent.getAclEnforcer());
+ doc.addField("agent", auditEvent.getAgentId());
+ doc.addField("repo", auditEvent.getRepositoryName());
+ doc.addField("sess", auditEvent.getSessionId());
+ doc.addField("reqUser", auditEvent.getUser());
+ doc.addField("reqData", auditEvent.getRequestData());
+ doc.addField("resource", auditEvent.getResourcePath());
+ doc.addField("cliIP", auditEvent.getClientIP());
+ doc.addField("logType", auditEvent.getLogType());
+ doc.addField("result", auditEvent.getAccessResult());
+ doc.addField("policy", auditEvent.getPolicyId());
+ doc.addField("repoType", auditEvent.getRepositoryType());
+ doc.addField("resType", auditEvent.getResourceType());
+ doc.addField("reason", auditEvent.getResultReason());
+ doc.addField("action", auditEvent.getAction());
+ doc.addField("evtTime", auditEvent.getEventTime());
+ doc.addField("seq_num", auditEvent.getSeqNum());
+ doc.setField("event_count", auditEvent.getEventCount());
+ doc.setField("event_dur_ms", auditEvent.getEventDurationMS());
+ doc.setField("tags", auditEvent.getTags());
+ doc.addField("datasets", auditEvent.getDatasets());
+ doc.addField("projects", auditEvent.getProjects());
+ doc.addField("datasetIds", auditEvent.getDatasetIds());
+ doc.setField("cluster", auditEvent.getClusterName());
+ doc.setField("zoneName", auditEvent.getZoneName());
+ doc.setField("agentHost", auditEvent.getAgentHostname());
+ doc.setField("policyVersion", auditEvent.getPolicyVersion());
+
+ return doc;
+ }
+
+ private void init() {
+ LOG.info("==>SolrAuditDestination.init()");
+
+ try {
+ // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
+ // Acutal solrclient JAAS configs are read from the ranger-<component>-audit.xml present in components conf folder and set by InMemoryJAASConfiguration
+ // Refer InMemoryJAASConfiguration doc for JAAS Configuration
+ String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
+
+ LOG.info("In solrAuditDestination.init() : JAAS Configuration set as [{}]", confFileName);
+
+ if (System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null) {
+ if (MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG, false)) {
+ System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null");
+ } else {
+ LOG.warn("No Client JAAS config present in solr audit config. Ranger Audit to Kerberized Solr will fail...");
+ }
+ }
+
+ LOG.info("Loading SolrClient JAAS config from Ranger audit config if present...");
+
+ InMemoryJAASConfiguration conf = InMemoryJAASConfiguration.init(props);
+
+ KerberosUser kerberosUser = new KerberosJAASConfigUser("Client", conf);
+
+ if (kerberosUser.getPrincipal() != null) {
+ this.kerberosUser = kerberosUser;
+ }
+ } catch (Exception e) {
+ LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e);
+ } finally {
+ String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
+ LOG.info("In solrAuditDestination.init() (finally) : JAAS Configuration set as [{}]", confFileName);
+ }
+
+ LOG.info("<==SolrAuditDestination.init()");
+ }
+
+ private KeyManager[] getKeyManagers() {
+ KeyManager[] kmList = null;
+ String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL);
+ String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS;
+ String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE);
+ String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias);
+
+ if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(keyStoreFile);
+
+ if (in != null) {
+ String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE);
+
+ keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT;
+
+ KeyStore keyStore = KeyStore.getInstance(keyStoreType);
+
+ keyStore.load(in, keyStoreFilepwd.toCharArray());
+
+ KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
+
+ keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
+
+ kmList = keyManagerFactory.getKeyManagers();
+ } else {
+ LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore :{}", e.getMessage(), e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification ", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL Keystore Files", e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL Keystore Files", e);
+ } catch (UnrecoverableKeyException e) {
+ LOG.error("Unable to recover the key from keystore", e);
+ } finally {
+ close(in, keyStoreFile);
+ }
+ }
+
+ return kmList;
+ }
+
+ private TrustManager[] getTrustManagers() {
+ TrustManager[] tmList = null;
+ String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL);
+ String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS;
+ String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE);
+ String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias);
+
+ if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(trustStoreFile);
+
+ if (in != null) {
+ String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE);
+
+ trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT;
+
+ KeyStore trustStore = KeyStore.getInstance(trustStoreType);
+
+ trustStore.load(in, trustStoreFilepwd.toCharArray());
+
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
+
+ trustManagerFactory.init(trustStore);
+
+ tmList = trustManagerFactory.getTrustManagers();
+ } else {
+ LOG.error("Unable to obtain truststore from file [{}]", trustStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment :{}", e.getMessage(), e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification :{}", e.getMessage(), e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL TrustStore File:{}", trustStoreFile, e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL TrustStore Files :{}", trustStoreFile, e);
+ } finally {
+ close(in, trustStoreFile);
+ }
+ }
+
+ return tmList;
+ }
+
+ private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) {
+ SSLContext sslContext = null;
+
+ try {
+ sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
+
+ if (sslContext != null) {
+ sslContext.init(kmList, tmList, new SecureRandom());
+ }
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is not available in the environment", e);
+ } catch (KeyManagementException e) {
+ LOG.error("Unable to initialise the SSLContext", e);
+ }
+
+ return sslContext;
+ }
+
+ private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection<SolrInputDocument> docs) throws Exception {
+ final UpdateResponse ret;
+
+ try {
+ final PrivilegedExceptionAction<UpdateResponse> action = () -> solrClient.add(docs);
+
+ if (kerberosUser != null) {
+ // execute the privileged action as the given keytab user
+ final KerberosAction<UpdateResponse> kerberosAction = new KerberosAction<>(kerberosUser, action, LOG);
+
+ ret = kerberosAction.execute();
+ } else {
+ ret = action.run();
+ }
+ } catch (Exception e) {
+ throw e;
+ }
+
+ return ret;
+ }
+
+ private InputStream getFileInputStream(String fileName) throws IOException {
+ InputStream in = null;
+
+ if (StringUtils.isNotEmpty(fileName)) {
+ File file = new File(fileName);
+
+ if (file.exists()) {
+ in = new FileInputStream(file);
+ } else {
+ in = ClassLoader.getSystemResourceAsStream(fileName);
+ }
+ }
+
+ return in;
+ }
+
+ private void close(InputStream str, String filename) {
+ if (str != null) {
+ try {
+ str.close();
+ } catch (IOException excp) {
+ LOG.error("Error while closing file: [{}]", filename, excp);
+ }
+ }
+ }
}
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index 08ad0c512..5ca02ca31 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -36,6 +36,7 @@
<module>dest-hdfs</module>
<module>dest-kafka</module>
<module>dest-log4j</module>
+ <module>dest-os</module>
<module>dest-solr</module>
<module>orc-util</module>
</modules>
diff --git a/agents-audit/pom.xml b/audit-server/audit-common/pom.xml
similarity index 50%
copy from agents-audit/pom.xml
copy to audit-server/audit-common/pom.xml
index 08ad0c512..d9a5cd0b4 100644
--- a/agents-audit/pom.xml
+++ b/audit-server/audit-common/pom.xml
@@ -17,26 +17,44 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+
<parent>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger</artifactId>
<version>2.10.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
+ <relativePath>../..</relativePath>
</parent>
- <artifactId>ranger-plugins-audit</artifactId>
- <packaging>pom</packaging>
- <name>Ranger Plugins Audit</name>
- <description>Ranger Plugins Audit</description>
- <!-- Sub modules -->
- <modules>
- <module>core</module>
- <module>dest-cloudwatch</module>
- <module>dest-es</module>
- <module>dest-hdfs</module>
- <module>dest-kafka</module>
- <module>dest-log4j</module>
- <module>dest-solr</module>
- <module>orc-util</module>
- </modules>
-</project>
+ <artifactId>ranger-audit-server-common</artifactId>
+ <packaging>jar</packaging>
+ <name>Ranger Audit Server Common</name>
+ <description>Shared classes between audit ingestor and dispatcher</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>ranger-audit-server-common-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java
new file mode 100644
index 000000000..a6c122d92
--- /dev/null
+++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditConfig.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.server;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Properties;
+
+/**
+ * Base configuration class for Ranger Audit Server services.
+ * Can be extended by specific services to load their custom configuration files.
+ */
+public class AuditConfig extends Configuration {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditConfig.class);
+
+ public AuditConfig() {
+ super();
+ }
+
+ public Properties getProperties() {
+ return this.getProps();
+ }
+
+ /**
+ * Add a resource file to the configuration.
+ * Subclasses can override to load their specific config files.
+ *
+ * @param resourcePath Path to the resource file (e.g., "conf/ranger-audit-ingestor-site.xml")
+ * @param required Whether this resource is required
+ * @return true if resource was loaded successfully or is optional, false otherwise
+ */
+ public boolean addAuditResource(String resourcePath, boolean required) {
+ LOG.debug("==> addAuditResource(path={}, required={})", resourcePath, required);
+
+ boolean success = addResourceIfReadable(resourcePath);
+
+ if (success) {
+ LOG.info("Successfully loaded configuration: {}", resourcePath);
+ } else if (required) {
+ LOG.error("Failed to load required configuration: {}", resourcePath);
+ } else {
+ LOG.warn("Failed to load optional configuration: {}", resourcePath);
+ }
+
+ LOG.debug("<== addAuditResource(path={}, required={}), result={}", resourcePath, required, success);
+
+ return success || !required;
+ }
+
+ public boolean addResourceIfReadable(String aResourceName) {
+ LOG.debug("==> addResourceIfReadable({})", aResourceName);
+
+ boolean ret = false;
+ URL fUrl = getFileLocation(aResourceName);
+
+ if (fUrl != null) {
+ LOG.debug("addResourceIfReadable({}): resource file is {}", aResourceName, fUrl);
+
+ try {
+ addResource(fUrl);
+
+ ret = true;
+ } catch (Exception e) {
+ LOG.error("Unable to load the resource name [{}]. Ignoring the resource:{}", aResourceName, fUrl);
+
+ LOG.debug("Resource loading failed for {}", fUrl, e);
+ }
+ } else {
+ LOG.debug("addResourceIfReadable({}): couldn't find resource file location", aResourceName);
+ }
+
+ LOG.debug("<== addResourceIfReadable({}), result={}", aResourceName, ret);
+
+ return ret;
+ }
+
+ private URL getFileLocation(String fileName) {
+ URL lurl = null;
+
+ if (!StringUtils.isEmpty(fileName)) {
+ lurl = AuditConfig.class.getClassLoader().getResource(fileName);
+
+ if (lurl == null) {
+ lurl = AuditConfig.class.getClassLoader().getResource("/" + fileName);
+ }
+
+ if (lurl == null) {
+ File f = new File(fileName);
+ if (f.exists()) {
+ try {
+ lurl = f.toURI().toURL();
+ } catch (MalformedURLException e) {
+ LOG.error("Unable to load the resource name [{}]. Ignoring the resource:{}", fileName, f.getPath());
+ }
+ } else {
+ LOG.debug("Conf file path {} does not exists", fileName);
+ }
+ }
+ }
+
+ return lurl;
+ }
+}
diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java
new file mode 100644
index 000000000..58b136b85
--- /dev/null
+++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.server;
+
+public class AuditServerConstants {
+ private AuditServerConstants() {}
+
+ public static final String AUDIT_SERVER_APPNAME = "ranger-audit";
+ public static final String PROP_PREFIX_AUDIT_SERVER = "ranger.audit.ingestor.";
+ public static final String PROP_PREFIX_AUDIT_SERVER_SERVICE = PROP_PREFIX_AUDIT_SERVER + "service.";
+ public static final String PROP_AUTH_TO_LOCAL = PROP_PREFIX_AUDIT_SERVER + "auth.to.local";
+ public static final String PROP_SUFFIX_ALLOWED_USERS = ".allowed.users";
+ public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "service.kerberos.principal";
+ public static final String PROP_AUDIT_SERVICE_KEYTAB = "service.kerberos.keytab";
+
+ /**************************************
+ AUDIT-SERVER INGESTOR Configuration
+ **************************************/
+ // kafka configuration for audit ingestor
+ public static final String JAAS_KRB5_MODULE = "com.sun.security.auth.module.Krb5LoginModule required";
+ public static final String JAAS_USE_KEYTAB = "useKeyTab=true";
+ public static final String JAAS_KEYTAB = "keyTab=\"";
+ public static final String JAAS_STOKE_KEY = "storeKey=true";
+ public static final String JAAS_SERVICE_NAME = "serviceName=kafka";
+ public static final String JAAS_USER_TICKET_CACHE = "useTicketCache=false";
+ public static final String JAAS_PRINCIPAL = "principal=\"";
+ public static final String PROP_KAFKA_PROP_PREFIX = "xasecure.audit.destination.kafka";
+ public static final String PROP_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers";
+ public static final String PROP_TOPIC_NAME = "kafka.topic.name";
+ public static final String PROP_SECURITY_PROTOCOL = "kafka.security.protocol";
+ public static final String PROP_SASL_MECHANISM = "kafka.sasl.mechanism";
+ public static final String PROP_KAFKA_TOPIC_INIT_MAX_RETRIES = "kafka.topic.init.max.retries";
+ public static final String PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS = "kafka.topic.init.retry.delay.ms";
+ public static final String PROP_REQ_TIMEOUT_MS = "kafka.request.timeout.ms";
+ public static final String PROP_CONN_MAX_IDEAL_MS = "kafka.connections.max.idle.ms";
+ public static final String PROP_SASL_JAAS_CONFIG = "sasl.jaas.config";
+ public static final String PROP_SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name";
+
+ // kafka topic - ranger_audits configuration
+ public static final String PROP_TOPIC_PARTITIONS = "kafka.topic.partitions";
+ public static final String PROP_PARTITIONER_CLASS = "kafka.partitioner.class";
+ public static final String PROP_CONFIGURED_PLUGINS = "kafka.configured.plugins";
+ public static final String PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN = "kafka.topic.partitions.per.configured.plugin";
+ public static final String PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX = "kafka.plugin.partition.overrides.";
+ public static final String PROP_BUFFER_PARTITIONS = "kafka.topic.partitions.buffer";
+ public static final String PROP_REPLICATION_FACTOR = "kafka.replication.factor";
+
+ // Kafka producer tuning (ranger.audit.ingestor.kafka.producer.*)
+ public static final String PROP_KAFKA_PRODUCER_PREFIX = "kafka.producer.";
+ public static final String PROP_PRODUCER_BATCH_SIZE = "batch.size";
+ public static final String PROP_PRODUCER_LINGER_MS = "linger.ms";
+ public static final String PROP_PRODUCER_BUFFER_MEMORY = "buffer.memory";
+ public static final String PROP_PRODUCER_COMPRESSION_TYPE = "compression.type";
+ public static final String PROP_PRODUCER_DELIVERY_TIMEOUT_MS = "delivery.timeout.ms";
+ public static final String PROP_PRODUCER_MAX_REQUEST_SIZE = "max.request.size";
+ public static final String PROP_PRODUCER_MAX_BLOCK_MS = "max.block.ms";
+ public static final String PROP_PRODUCER_BATCH_SEND_TIMEOUT_MS = "batch.send.timeout.ms";
+
+ // Optional topic-level configs applied at topic create (Kafka Admin)
+ public static final String PROP_TOPIC_RETENTION_MS = "kafka.topic.retention.ms";
+ public static final String PROP_TOPIC_COMPRESSION_TYPE = "kafka.topic.compression.type";
+ public static final String PROP_TOPIC_MIN_INSYNC_REPLICAS = "kafka.topic.min.insync.replicas";
+
+ // Kafka producer defaults (high-volume audit JSON; idempotent + acks=all)
+ public static final int DEFAULT_PRODUCER_BATCH_SIZE = 131072; // 128 KiB
+ public static final int DEFAULT_PRODUCER_LINGER_MS = 20;
+ public static final long DEFAULT_PRODUCER_BUFFER_MEMORY = 134217728L; // 128 MiB
+ public static final String DEFAULT_PRODUCER_COMPRESSION_TYPE = "lz4";
+ public static final int DEFAULT_PRODUCER_DELIVERY_TIMEOUT_MS = 120000;
+ public static final int DEFAULT_PRODUCER_MAX_REQUEST_SIZE = 1048576; // 1 MiB
+ public static final int DEFAULT_PRODUCER_MAX_BLOCK_MS = 60000;
+ public static final int DEFAULT_PRODUCER_BATCH_SEND_TIMEOUT_MS = 30000;
+ public static final int DEFAULT_PRODUCER_REQUEST_TIMEOUT_MS = 60000;
+
+ // Optional topic defaults when properties are set (production guidance; RF must allow min ISR)
+ public static final String DEFAULT_TOPIC_RETENTION_MS = "604800000"; // 7 days
+ public static final String DEFAULT_TOPIC_COMPRESSION_TYPE = "lz4";
+ public static final String DEFAULT_TOPIC_MIN_INSYNC_REPLICAS = "2";
+
+ // Kafka Topic defaults
+ public static final String DEFAULT_TOPIC = "ranger_audits";
+ public static final String DEFAULT_SASL_MECHANISM = "PLAIN";
+ public static final String DEFAULT_SECURITY_PROTOCOL = "PLAINTEXT";
+ public static final String DEFAULT_SERVICE_NAME = "kafka";
+ public static final String PROP_SECURITY_PROTOCOL_VALUE = "SASL";
+
+ // kafka Offset commit strategies
+ public static final String PROP_OFFSET_COMMIT_STRATEGY_MANUAL = "manual";
+ public static final String PROP_OFFSET_COMMIT_STRATEGY_BATCH = "batch";
+ public static final String DEFAULT_OFFSET_COMMIT_STRATEGY = PROP_OFFSET_COMMIT_STRATEGY_BATCH;
+ public static final long DEFAULT_OFFSET_COMMIT_INTERVAL_MS = 30000; // 30 seconds
+ public static final int DEFAULT_MAX_POLL_RECORDS = 500; // Kafka default batch size
+
+ // Default configured plugins: each gets allocated partitions from the topic
+ public static final String DEFAULT_PARTITIONER_CLASS = "org.apache.ranger.audit.producer.kafka.AuditPartitioner";
+ public static final String DEFAULT_CONFIGURED_PLUGINS = "hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi";
+ public static final short DEFAULT_REPLICATION_FACTOR = 3;
+ public static final int DEFAULT_TOPIC_PARTITIONS = 10;
+ public static final int DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN = 3;
+ public static final int DEFAULT_BUFFER_PARTITIONS = 9;
+
+ // Hadoop security configuration for UGI
+ public static final String PROP_HADOOP_AUTHENTICATION_TYPE = "hadoop.security.authentication";
+ public static final String PROP_HADOOP_AUTH_TYPE_KERBEROS = "kerberos";
+ public static final String PROP_HADOOP_KERBEROS_NAME_RULES = "hadoop.security.auth_to_local";
+
+ /**************************************
+ AUDIT-SERVER DISPATCHER Configuration
+ **************************************/
+ // kafka configuration for the audit-server dispatcher
+ public static final String DEFAULT_PARTITION_ASSIGNMENT_STRATEGY = "org.apache.kafka.clients.consumer.CooperativeStickyAssignor";
+ public static final String PROP_DISPATCHER_PREFIX = "ranger.audit.dispatcher";
+ public static final String PROP_DISPATCHER_THREAD_COUNT = "thread.count";
+ public static final String PROP_DISPATCHER_OFFSET_COMMIT_STRATEGY = "offset.commit.strategy";
+ public static final String PROP_DISPATCHER_OFFSET_COMMIT_INTERVAL = "offset.commit.interval.ms";
+ public static final String PROP_DISPATCHER_MAX_POLL_RECORDS = "max.poll.records";
+ public static final String PROP_DISPATCHER_SESSION_TIMEOUT_MS = "session.timeout.ms";
+ public static final String PROP_DISPATCHER_MAX_POLL_INTERVAL_MS = "max.poll.interval.ms";
+ public static final String PROP_DISPATCHER_HEARTBEAT_INTERVAL_MS = "heartbeat.interval.ms";
+ public static final String PROP_DISPATCHER_PARTITION_ASSIGNMENT_STRATEGY = "partition.assignment.strategy";
+ public static final String PROP_DISPATCHER_TYPE = "ranger.audit.dispatcher.type";
+ public static final String PROP_DISPATCHER_CLASS = "ranger.audit.dispatcher.class";
+
+ // Kafka dispatcher rebalancing timeouts (for subscribe mode)
+ public static final int DEFAULT_SESSION_TIMEOUT_MS = 60000; // 60 seconds - failure detection
+ public static final int DEFAULT_MAX_POLL_INTERVAL_MS = 300000; // 5 minutes - max processing time
+ public static final int DEFAULT_HEARTBEAT_INTERVAL_MS = 10000; // 10 seconds - heartbeat frequency
+}
diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java
new file mode 100644
index 000000000..bfba27260
--- /dev/null
+++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.CreatePartitionsResult;
+import org.apache.kafka.clients.admin.DescribeTopicsResult;
+import org.apache.kafka.clients.admin.NewPartitions;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+public class AuditMessageQueueUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditMessageQueueUtils.class);
+
+ private AuditMessageQueueUtils() {
+ }
+
+ public static String createAuditsTopicIfNotExists(Properties props, String propPrefix) {
+ LOG.info("==> AuditMessageQueueUtils:createAuditsTopicIfNotExists(propPrefix={})", propPrefix);
+
+ String ret = null;
+ String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC);
+ String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS);
+ String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL);
+ String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM);
+ int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000);
+ int partitions = getPartitions(props, propPrefix);
+ short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR);
+ int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000);
+ int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1;
+ int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000);
+
+ Map<String, Object> kafkaProp = new HashMap<>();
+
+ kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ kafkaProp.put("sasl.mechanism", saslMechanism);
+ kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol);
+
+ if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) {
+ kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix));
+ kafkaProp.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME);
+ }
+
+ kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS);
+ kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS);
+
+ for (int currentAttempt = 1; currentAttempt <= maxAttempts && ret == null; currentAttempt++) {
+ try (AdminClient admin = AdminClient.create(kafkaProp)) {
+ LOG.info("Attempting to connect to Kafka (attempt {}/{})", currentAttempt, maxAttempts);
+
+ Set<String> names = admin.listTopics().names().get();
+
+ if (!names.contains(topicName)) {
+ LOG.info("Creating topic '{}' with {} partitions and replication factor {}", topicName, partitions, replicationFactor);
+
+ NewTopic topic = new NewTopic(topicName, partitions, replicationFactor);
+ Map<String, String> topicConfigs = buildTopicConfigs(props, propPrefix);
+
+ if (topicConfigs != null && !topicConfigs.isEmpty()) {
+ topic.configs(topicConfigs);
+ LOG.info("Topic '{}' configs: {}", topicName, topicConfigs);
+ }
+
+ admin.createTopics(Collections.singletonList(topic)).all().get();
+
+ ret = topic.name();
+
+ // Wait for metadata to propagate across the cluster
+ boolean isTopicReady = AuditServerUtils.waitUntilTopicReady(admin, topicName, Duration.ofSeconds(60));
+
+ if (isTopicReady) {
+ try {
+ DescribeTopicsResult result = admin.describeTopics(Collections.singletonList(topicName));
+ TopicDescription topicDescription = result.values().get(topicName).get();
+
+ ret = topicDescription.name();
+
+ int partitionCount = topicDescription.partitions().size();
+
+ LOG.info("Topic: {} successfully created with {} partitions", ret, partitionCount);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to fetch metadata for topic:" + topicName, e);
+ }
+ }
+ } else {
+ /***
+ * Topic already existing. Check and update number of partitions for audit server. This is for upgrade
+ * from existing audit mechanism to audit server
+ */
+ ret = updateExistingTopicPartitions(admin, topicName, partitions);
+ }
+ } catch (Exception ex) {
+ if (currentAttempt < maxAttempts) {
+ LOG.warn("AuditMessageQueueUtils:createAuditsTopicIfNotExists(): Failed to connect to Kafka on attempt {}/{}. Retrying in {} ms. Error: {}",
+ currentAttempt, maxAttempts, retryDelayMs, ex);
+
+ try {
+ Thread.sleep(retryDelayMs);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.error("Interrupted while waiting to retry Kafka connection", ie);
+ break;
+ }
+ } else {
+ LOG.error("AuditMessageQueueUtils:createAuditsTopicIfNotExists(): Error creating topic after {} attempts", currentAttempt, ex);
+ }
+ }
+ }
+
+ LOG.info("<== AuditMessageQueueUtils:createAuditsTopicIfNotExists(propPrefix={}) ret: {}", propPrefix, ret);
+
+ return ret;
+ }
+
+ public static String getJAASConfig(Properties props, String propPrefix) {
+ // Use ranger service principal and keytab for Kafka authentication
+ // This ensures consistent identity across all Ranger services and destination writes
+ String hostName = props.getProperty(propPrefix + "." + "host");
+ String principal = props.getProperty(propPrefix + "." + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL);
+ String keytab = props.getProperty(propPrefix + "." + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB);
+
+ AuditServerLogFormatter.builder("Kerberos Configuration")
+ .add("Principal (raw)", principal)
+ .add("Hostname", hostName)
+ .add("Keytab path", keytab)
+ .logInfo(LOG);
+
+ // Validate keytab file exists and is readable
+ if (keytab != null) {
+ File keytabFile = new File(keytab);
+
+ if (!keytabFile.exists()) {
+ LOG.error("ERROR: Keytab file does not exist: {}", keytab);
+
+ throw new IllegalStateException("Keytab file not found: " + keytab);
+ } else if (!keytabFile.canRead()) {
+ LOG.error("ERROR: Keytab file is not readable: {}", keytab);
+
+ throw new IllegalStateException("Keytab file not readable: " + keytab);
+ }
+
+ AuditServerLogFormatter.builder("Keytab File Validation")
+ .add("Exists", keytabFile.exists())
+ .add("Readable", keytabFile.canRead())
+ .add("Size (bytes)", keytabFile.length())
+ .logInfo(LOG);
+ }
+
+ try {
+ principal = SecureClientLogin.getPrincipal(principal, hostName);
+
+ LOG.info("Principal (resolved): {}", principal);
+ } catch (Exception e) {
+ principal = null;
+
+ LOG.error("ERROR: Failed to resolve principal from _HOST pattern!", e);
+ }
+
+ if (keytab == null || principal == null) {
+ AuditServerLogFormatter.builder("Please configure the following properties in ranger-audit-ingestor-site.xml:")
+ .add(propPrefix + "." + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL, "ranger/_HOST@YOUR-REALM")
+ .add(propPrefix + "." + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB, "/path/to/ranger.keytab")
+ .logError(LOG);
+
+ throw new IllegalStateException("Ranger service principal and keytab must be configured for Kafka authentication. ");
+ }
+
+ String jaasConfig = new StringBuilder()
+ .append(AuditServerConstants.JAAS_KRB5_MODULE).append(" ")
+ .append(AuditServerConstants.JAAS_USE_KEYTAB).append(" ")
+ .append(AuditServerConstants.JAAS_KEYTAB).append(keytab).append("\"").append(" ")
+ .append(AuditServerConstants.JAAS_STOKE_KEY).append(" ")
+ .append(AuditServerConstants.JAAS_USER_TICKET_CACHE).append(" ")
+ .append(AuditServerConstants.JAAS_SERVICE_NAME).append(" ")
+ .append(AuditServerConstants.JAAS_PRINCIPAL).append(principal).append("\";")
+ .toString();
+
+ AuditServerLogFormatter.builder("JAAS Configuration Generated")
+ .add("Principal", principal)
+ .add("Keytab", keytab)
+ .add("Full JAAS Config", jaasConfig)
+ .logInfo(LOG);
+
+ return jaasConfig;
+ }
+
+ private static String updateExistingTopicPartitions(AdminClient admin, String topicName, int partitions) {
+ LOG.info("==> AuditMessageQueueUtils:updateExistingTopicPartitions() topic: {}, desired partitions: {}", topicName, partitions);
+
+ String ret;
+ int maxAttempts = 3;
+ int retryDelayMs = 1000; // Start with 1 second
+
+ try {
+ // Describe the existing topic to get current partition count
+ DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singletonList(topicName));
+ TopicDescription topicDescription = describeTopicsResult.values().get(topicName).get();
+ int currentPartitions = topicDescription.partitions().size();
+
+ ret = topicDescription.name();
+
+ LOG.info("Topic '{}' already exists with {} partitions", ret, currentPartitions);
+
+ // Check if we need to increase partitions
+ if (partitions > currentPartitions) {
+ LOG.info("Upgrading topic '{}' from {} to {} partitions for audit server mechanism", topicName, currentPartitions, partitions);
+
+ boolean updateSuccess = false;
+ Exception lastException = null;
+
+ // Retry logic while updating partitions
+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+ try {
+ LOG.info("Partition update attempt {}/{} for topic '{}'", attempt, maxAttempts, topicName);
+
+ // Create partition increase request
+ Map<String, NewPartitions> newPartitionsMap = new HashMap<>();
+
+ newPartitionsMap.put(topicName, NewPartitions.increaseTo(partitions));
+
+ // Execute partition increase
+ CreatePartitionsResult createPartitionsResult = admin.createPartitions(newPartitionsMap);
+
+ createPartitionsResult.all().get(); // Wait for operation to complete
+
+ LOG.info("Successfully initiated partition increase for topic '{}' on attempt {}", topicName, attempt);
+
+ // Wait for metadata to propagate across the cluster
+ boolean isTopicReady = AuditServerUtils.waitUntilTopicReady(admin, topicName, Duration.ofSeconds(60));
+
+ if (isTopicReady) {
+ // Verify the partition count after update
+ DescribeTopicsResult verifyResult = admin.describeTopics(Collections.singletonList(topicName));
+ TopicDescription updatedDescription = verifyResult.values().get(topicName).get();
+ int updatedPartitions = updatedDescription.partitions().size();
+
+ if (updatedPartitions >= partitions) {
+ LOG.info("Topic '{}' successfully upgraded from {} to {} partitions on attempt {}", topicName, currentPartitions, updatedPartitions, attempt);
+
+ updateSuccess = true;
+ break;
+ } else {
+ LOG.warn("Topic '{}' partition update completed but verification shows only {} partitions (expected {})", topicName, updatedPartitions, partitions);
+
+ throw new IllegalStateException("Partition count verification failed");
+ }
+ } else {
+ LOG.warn("Topic '{}' partition update completed but topic not ready within timeout on attempt {}", topicName, attempt);
+
+ throw new IllegalStateException("Topic not ready after partition update");
+ }
+ } catch (Exception e) {
+ lastException = e;
+
+ LOG.warn("Partition update attempt {}/{} failed for topic '{}': {}", attempt, maxAttempts, topicName, e.getMessage());
+
+ if (attempt < maxAttempts) {
+ int currentDelay = retryDelayMs * (1 << (attempt - 1));
+
+ LOG.info("Retrying partition update in {} ms...", currentDelay);
+
+ Thread.sleep(currentDelay);
+ }
+ }
+ }
+
+ if (!updateSuccess) {
+ String errorMsg = String.format("Failed to update partitions for topic '%s' after %d attempts. " +
+ "Required: %d partitions, Current: %d partitions. Cannot proceed without sufficient partitions.",
+ topicName, maxAttempts, partitions, currentPartitions);
+
+ LOG.error(errorMsg, lastException);
+
+ throw new RuntimeException(errorMsg, lastException);
+ }
+ } else if (partitions < currentPartitions) {
+ LOG.warn("Topic '{}' has {} partitions, which is more than the configured {} partitions. " +
+ "Kafka does not support reducing partition count. Using existing partition count.", topicName, currentPartitions, partitions);
+ } else {
+ LOG.info("Topic '{}' already has the correct number of partitions: {}", topicName, currentPartitions);
+ }
+ } catch (Exception e) {
+ String errorMsg = String.format("Error updating partitions for topic '%s'", topicName);
+
+ LOG.error(errorMsg, e);
+
+ throw new RuntimeException(errorMsg, e);
+ }
+
+ LOG.info("<== AuditMessageQueueUtils:updateExistingTopicPartitions() ret: {}", ret);
+
+ return ret;
+ }
+
+ /**
+ * Get the number of partitions for the Kafka topic.
+ *
+ * If configured.plugins is NOT set
+ * topic.partitions property (default: 10)
+ * Else
+ * sum(per-plugin partitions: default 3) + buffer partitions
+ *
+ * @return Number of partitions for the topic
+ */
+ private static int getPartitions(Properties prop, String propPrefix) {
+ // Check if configured.plugins is set (use empty string as default to detect when not configured)
+ int totalPartitions = 0;
+ String configuredPlugins = MiscUtil.getStringProperty(prop, propPrefix + "." + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS);
+
+ if (configuredPlugins == null || configuredPlugins.trim().isEmpty()) {
+ totalPartitions = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS);
+
+ LOG.info("No configured plugins - using hash-based partitioning with {} partitions", totalPartitions);
+ } else {
+ // Auto-calculate based on plugin configuration
+ int defaultPartitionsPerPlugin = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN);
+
+ // Calculate total partitions needed
+ AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("Kafka Topic Partition Allocation (Plugin-based)");
+
+ for (String plugin : configuredPlugins.split(",")) {
+ String overrideKey = propPrefix + "." + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + plugin.trim();
+ int partitionCount = MiscUtil.getIntProperty(prop, overrideKey, defaultPartitionsPerPlugin);
+
+ totalPartitions += partitionCount;
+
+ logBuilder.add("Plugin '" + plugin + "'", partitionCount + " partitions");
+ }
+
+ // Add buffer partitions for unconfigured plugins
+ int bufferPartitions = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_BUFFER_PARTITIONS, AuditServerConstants.DEFAULT_BUFFER_PARTITIONS);
+
+ totalPartitions += bufferPartitions;
+
+ logBuilder.add("Buffer partitions", bufferPartitions + " partitions");
+ logBuilder.add("Total topic partitions (calculated)", totalPartitions);
+ logBuilder.logInfo(LOG);
+ }
+
+ return totalPartitions;
+ }
+
+ /**
+ * Optional topic-level configs for {@code ranger_audits} at create time.
+ * Properties are applied only when set and non-blank in site XML.
+ */
+ static Map<String, String> buildTopicConfigs(Properties props, String propPrefix) {
+ Map<String, String> configs = new HashMap<>();
+
+ putTopicConfigIfSet(configs, props, propPrefix, AuditServerConstants.PROP_TOPIC_RETENTION_MS, "retention.ms");
+ putTopicConfigIfSet(configs, props, propPrefix, AuditServerConstants.PROP_TOPIC_COMPRESSION_TYPE, "compression.type");
+ putTopicConfigIfSet(configs, props, propPrefix, AuditServerConstants.PROP_TOPIC_MIN_INSYNC_REPLICAS, "min.insync.replicas");
+
+ return configs;
+ }
+
+ private static void putTopicConfigIfSet(Map<String, String> configs, Properties props, String propPrefix,
+ String rangerPropSuffix, String kafkaConfigKey) {
+ String value = MiscUtil.getStringProperty(props, propPrefix + "." + rangerPropSuffix);
+
+ if (value != null && !value.trim().isEmpty()) {
+ configs.put(kafkaConfigKey, value.trim());
+ }
+ }
+}
diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java
new file mode 100644
index 000000000..2f03d70db
--- /dev/null
+++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerLogFormatter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.slf4j.Logger;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Utility for logging in the ranger-audit-server
+ */
+public class AuditServerLogFormatter {
+ private AuditServerLogFormatter() {}
+
+ /**
+ * Create a builder for constructing LogDetails maps to log
+ * @param title The title for this log section
+ * @return A new LogBuilder
+ */
+ public static LogBuilder builder(String title) {
+ return new LogBuilder(title);
+ }
+
+ /**
+ * Builder class for constructing structured log messages
+ */
+ public static class LogBuilder {
+ private final String title;
+ private final Map<String, Object> logDetails = new LinkedHashMap<>();
+
+ private LogBuilder(String title) {
+ this.title = title;
+ }
+
+ /**
+ * Add a log key value pair
+ * @param key
+ * @param value
+ * @return This builder for chaining
+ */
+ public LogBuilder add(String key, Object value) {
+ logDetails.put(key, value);
+ return this;
+ }
+
+ /**
+ * Add a log only if the value is not null
+ * @param key
+ * @param value
+ * @return This builder for chaining
+ */
+ public LogBuilder addIfNotNull(String key, Object value) {
+ if (value != null) {
+ logDetails.put(key, value);
+ }
+ return this;
+ }
+
+ /**
+ * Log the accumulated LogDetails at INFO level
+ * @param logger The logger to use
+ */
+ public void logInfo(Logger logger) {
+ logger.info("{} {}", title, logDetails);
+ }
+
+ /**
+ * Log the accumulated LogDetails at DEBUG level
+ * @param logger The logger to use
+ */
+ public void logDebug(Logger logger) {
+ logger.debug("{} {}", title, logDetails);
+ }
+
+ /**
+ * Log the accumulated LogDetails at DEBUG level
+ * @param logger The logger to use
+ */
+ public void logError(Logger logger) {
+ logger.error("{} {}", title, logDetails);
+ }
+ }
+}
diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java
new file mode 100644
index 000000000..1430a6e0d
--- /dev/null
+++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditServerUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.DescribeTopicsResult;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.concurrent.ThreadLocalRandom;
+
+public class AuditServerUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditServerUtils.class);
+
+ private AuditServerUtils() {
+ }
+
+ public static boolean waitUntilTopicReady(Admin admin, String topic, Duration totalWait) throws Exception {
+ long endTime = System.nanoTime() + totalWait.toNanos();
+ long baseSleepMs = 100L;
+ long maxSleepMs = 2000L;
+
+ while (System.nanoTime() < endTime) {
+ try {
+ DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singleton(topic));
+ TopicDescription topicDescription = describeTopicsResult.values().get(topic).get();
+ boolean allHaveLeader = topicDescription.partitions().stream().allMatch(partitionInfo -> partitionInfo.leader() != null);
+ boolean allHaveISR = topicDescription.partitions().stream().allMatch(partitionInfo -> !partitionInfo.isr().isEmpty());
+
+ if (allHaveLeader && allHaveISR) {
+ return true;
+ }
+ } catch (Exception e) {
+ // If topic hasn't propagated yet, you'll see UnknownTopicOrPartitionException
+ // continue to wait for topic availability
+ if (!(rootCause(e) instanceof UnknownTopicOrPartitionException)) {
+ throw e;
+ }
+ }
+
+ // Sleep until the created topic is available for metadata fetch
+ baseSleepMs = Math.min(maxSleepMs, baseSleepMs * 2);
+
+ long sleep = baseSleepMs + ThreadLocalRandom.current().nextLong(0, baseSleepMs / 2 + 1);
+
+ LOG.info("topic {} is not ready yet. will retry after {}ms", topic, sleep);
+
+ Thread.sleep(sleep);
+ }
+
+ return false;
+ }
+
+ private static Throwable rootCause(Throwable t) {
+ Throwable throwable = t;
+
+ while (throwable.getCause() != null) {
+ throwable = throwable.getCause();
+ }
+
+ return throwable;
+ }
+}
diff --git a/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java b/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java
new file mode 100644
index 000000000..e20a69647
--- /dev/null
+++ b/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.Properties;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class AuditMessageQueueUtilsTest {
+ private static final String PROP_PREFIX = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER + "kafka";
+
+ @Test
+ public void testBuildTopicConfigsEmptyWhenUnset() {
+ Properties props = new Properties();
+
+ Map<String, String> configs = AuditMessageQueueUtils.buildTopicConfigs(props, PROP_PREFIX);
+
+ assertTrue(configs.isEmpty());
+ }
+
+ @Test
+ public void testBuildTopicConfigsSkipsBlankValues() {
+ Properties props = new Properties();
+ props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_RETENTION_MS, " ");
+ props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_COMPRESSION_TYPE, "lz4");
+ props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_MIN_INSYNC_REPLICAS, "");
+
+ Map<String, String> configs = AuditMessageQueueUtils.buildTopicConfigs(props, PROP_PREFIX);
+
+ assertEquals(1, configs.size());
+ assertEquals("lz4", configs.get("compression.type"));
+ }
+
+ @Test
+ public void testBuildTopicConfigsMapsAllSetProperties() {
+ Properties props = new Properties();
+ props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_RETENTION_MS, "604800000");
+ props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_COMPRESSION_TYPE, " lz4 ");
+ props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_MIN_INSYNC_REPLICAS, "2");
+
+ Map<String, String> configs = AuditMessageQueueUtils.buildTopicConfigs(props, PROP_PREFIX);
+
+ assertEquals(3, configs.size());
+ assertEquals("604800000", configs.get("retention.ms"));
+ assertEquals("lz4", configs.get("compression.type"));
+ assertEquals("2", configs.get("min.insync.replicas"));
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-app/pom.xml b/audit-server/audit-dispatcher/dispatcher-app/pom.xml
new file mode 100644
index 000000000..8cc489ca3
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-app/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>2.10.0-SNAPSHOT</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-dispatcher-app</artifactId>
+ <packaging>war</packaging>
+ <name>Ranger Audit Dispatcher App</name>
+ <description>Unified Ranger Kafka service for writing audits to HDFS/S3/Azure/Solr</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <version>${jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>${jaxb-impl.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>${jaxb.api.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dispatcher-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>audit-dispatcher-hdfs</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>audit-dispatcher-opensearch</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>audit-dispatcher-solr</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>ranger-audit-dispatcher-${project.version}</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/audit-server/audit-dispatcher/dispatcher-app/src/main/java/org/apache/ranger/audit/dispatcher/AuditDispatcherApplication.java b/audit-server/audit-dispatcher/dispatcher-app/src/main/java/org/apache/ranger/audit/dispatcher/AuditDispatcherApplication.java
new file mode 100644
index 000000000..63defeb57
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-app/src/main/java/org/apache/ranger/audit/dispatcher/AuditDispatcherApplication.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher;
+
+import org.apache.ranger.audit.server.AuditConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Properties;
+
+public class AuditDispatcherApplication {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditDispatcherApplication.class);
+ private static final String APP_NAME = "audit-dispatcher";
+ private static final String CONFIG_PREFIX = "ranger.audit.dispatcher.";
+
+ private AuditDispatcherApplication() {
+ }
+
+ public static void main(String[] args) {
+ String dispatcherType = System.getProperty(CONFIG_PREFIX + "type");
+ if (dispatcherType == null) {
+ LOG.error("Dispatcher initialization failed.");
+ LOG.error("No dispatcher type specified. Please set [ranger.audit.dispatcher.type] system property.");
+ System.exit(1);
+ }
+
+ AuditConfig config = new AuditConfig();
+ String specificConfig = getDispatcherConfigPath(dispatcherType);
+ if (!config.addResourceIfReadable(specificConfig)) {
+ LOG.error("Failed to load dispatcher configuration: {}", specificConfig);
+ System.exit(1);
+ }
+ LOG.info("Loaded dispatcher-specific configuration: {}", specificConfig);
+
+ LOG.info("==========================================================================");
+ LOG.info("==> Starting Ranger Audit Dispatcher Service (Type: {})", dispatcherType);
+ LOG.info("==========================================================================");
+
+ // Initialization dispatcher manager based on dispatcher type before starting EmbeddedServer
+ boolean initSuccess = false;
+ try {
+ String dispatcherMgrClass = config.get(CONFIG_PREFIX + dispatcherType + ".class");
+ if (dispatcherMgrClass != null && !dispatcherMgrClass.trim().isEmpty()) {
+ initSuccess = initializeDispatcherManager(dispatcherMgrClass, config.getProperties());
+ } else {
+ LOG.error("Unknown dispatcher type: {}. Cannot initialize dispatcher manager.", dispatcherType);
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to initialize DispatcherManager", e);
+ }
+
+ if (!initSuccess) {
+ LOG.error("Dispatcher initialization failed.");
+ System.exit(1);
+ }
+
+ try {
+ EmbeddedServer server = new EmbeddedServer(config, APP_NAME, CONFIG_PREFIX);
+ server.start();
+
+ LOG.info("<== Ranger Audit Dispatcher Service Started Successfully");
+ } catch (Exception e) {
+ LOG.error("<== Failed to start Ranger Audit Dispatcher Service", e);
+ System.exit(1);
+ }
+ }
+
+ private static boolean initializeDispatcherManager(String dispatcherMgrClass, Properties props) throws Exception {
+ Object manager = Class.forName(dispatcherMgrClass, true, Thread.currentThread().getContextClassLoader()).newInstance();
+ manager.getClass().getMethod("init", Properties.class).invoke(manager, props);
+ LOG.info("{} initialized successfully", dispatcherMgrClass);
+ return true;
+ }
+
+ private static String getDispatcherConfigPath(String dispatcherType) {
+ String confDir = System.getenv("AUDIT_DISPATCHER_CONF_DIR");
+ if (confDir == null || confDir.trim().isEmpty()) {
+ confDir = System.getProperty("ranger.audit.dispatcher.conf.dir");
+ }
+
+ String fileName = "ranger-audit-dispatcher-" + dispatcherType + "-site.xml";
+ if (confDir != null && !confDir.trim().isEmpty()) {
+ return confDir + File.separator + fileName;
+ }
+
+ return fileName;
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-app/src/main/java/org/apache/ranger/audit/rest/AuditDispatcherHealthREST.java b/audit-server/audit-dispatcher/dispatcher-app/src/main/java/org/apache/ranger/audit/rest/AuditDispatcherHealthREST.java
new file mode 100644
index 000000000..beabd29e4
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-app/src/main/java/org/apache/ranger/audit/rest/AuditDispatcherHealthREST.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.rest;
+
+import org.apache.ranger.audit.dispatcher.kafka.AuditDispatcherTracker;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Health check REST endpoint for Audit Dispatcher Service
+ */
+@Path("/health")
+@Component
+@Scope("request")
+public class AuditDispatcherHealthREST {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditDispatcherHealthREST.class);
+
+ /**
+ * Simple ping endpoint to verify the service is running
+ */
+ @GET
+ @Path("/ping")
+ @Produces("application/json")
+ public Response ping() {
+ Map<String, Object> resp = new HashMap<>();
+ resp.put("status", "UP");
+ resp.put("service", "audit-dispatcher");
+
+ try {
+ Response ret = Response.status(Response.Status.OK)
+ .entity(buildResponse(resp))
+ .build();
+ return ret;
+ } catch (Exception e) {
+ LOG.error("Error creating ping response", e);
+ return Response.serverError().build();
+ }
+ }
+
+ /**
+ * Detailed health check endpoint that verifies internal components
+ */
+ @GET
+ @Path("/status")
+ @Produces("application/json")
+ public Response status() {
+ Map<String, Object> resp = new HashMap<>();
+ try {
+ String dispatcherType = System.getProperty(AuditServerConstants.PROP_DISPATCHER_TYPE);
+ resp.put("service", "audit-dispatcher-" + (dispatcherType != null ? dispatcherType : "unknown"));
+
+ Response.Status status = null;
+ if (dispatcherType != null && !dispatcherType.trim().isEmpty()) {
+ boolean isActive = AuditDispatcherTracker.getInstance().getActiveDispatcherTypes().contains(dispatcherType.toLowerCase());
+ if (isActive) {
+ status = Response.Status.OK;
+ resp.put("status", "UP");
+ } else {
+ status = Response.Status.SERVICE_UNAVAILABLE;
+ resp.put("status", "DOWN");
+ resp.put("reason", dispatcherType + " Dispatcher is not active");
+ }
+ } else {
+ status = Response.Status.SERVICE_UNAVAILABLE;
+ resp.put("status", "DOWN");
+ resp.put("reason", "Dispatcher type not provided: ");
+ }
+
+ Response ret = Response.status(status)
+ .entity(buildResponse(resp))
+ .build();
+ return ret;
+ } catch (Exception e) {
+ LOG.error("Error checking status", e);
+ resp.put("status", "ERROR");
+ resp.put("error", e.getMessage());
+ return Response.serverError().entity(resp).build();
+ }
+ }
+
+ private String buildResponse(Map<String, Object> respMap) {
+ try {
+ return MiscUtil.getMapper().writeValueAsString(respMap);
+ } catch (Exception e) {
+ return "Error: " + e.getMessage();
+ }
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-app/src/main/webapp/WEB-INF/applicationContext.xml b/audit-server/audit-dispatcher/dispatcher-app/src/main/webapp/WEB-INF/applicationContext.xml
new file mode 100644
index 000000000..b580d710f
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-app/src/main/webapp/WEB-INF/applicationContext.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-4.3.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-4.3.xsd">
+
+ <context:component-scan base-package="org.apache.ranger.audit" />
+
+ <bean class="org.springframework.beans.factory.config.CustomScopeConfigurer">
+ <property name="scopes">
+ <map>
+ <entry key="request">
+ <bean class="org.springframework.web.context.request.RequestScope" />
+ </entry>
+ </map>
+ </property>
+ </bean>
+</beans>
diff --git a/audit-server/audit-dispatcher/dispatcher-app/src/main/webapp/WEB-INF/web.xml b/audit-server/audit-dispatcher/dispatcher-app/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 000000000..bc37c4df9
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-app/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
+ <display-name>Apache Ranger - Audit Dispatcher Service</display-name>
+ <description>Apache Ranger - Audit Dispatcher Service</description>
+
+ <context-param>
+ <param-name>contextConfigLocation</param-name>
+ <param-value>
+ WEB-INF/applicationContext.xml
+ </param-value>
+ </context-param>
+
+ <listener>
+ <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
+ </listener>
+
+ <listener>
+ <listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>Health Check Service</servlet-name>
+ <servlet-class>com.sun.jersey.spi.spring.container.servlet.SpringServlet</servlet-class>
+
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.ranger.audit.rest</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.config.feature.DisableWADL</param-name>
+ <param-value>true</param-value>
+ </init-param>
+
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>Health Check Service</servlet-name>
+ <url-pattern>/api/*</url-pattern>
+ </servlet-mapping>
+
+</web-app>
diff --git a/audit-server/audit-dispatcher/dispatcher-common/pom.xml b/audit-server/audit-dispatcher/dispatcher-common/pom.xml
new file mode 100644
index 000000000..b342fc8b6
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/pom.xml
@@ -0,0 +1,146 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>2.10.0-SNAPSHOT</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+
+ <artifactId>ranger-audit-dispatcher-common</artifactId>
+ <packaging>jar</packaging>
+ <name>Ranger Audit Dispatcher Common</name>
+ <description>Shared dispatcher framework for Kafka-based audit dispatchers</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- Ranger Dependencies -->
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-server-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Tomcat (for EmbeddedServer) -->
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-annotations-api</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-jasper</artifactId>
+ <version>${tomcat.embed.version}</version>
+ </dependency>
+
+ <!-- SLF4J -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <!-- Spring -->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>ranger-audit-dispatcher-common-${project.version}</finalName>
+ </build>
+</project>
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/AuditDispatcherLauncher.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/AuditDispatcherLauncher.java
new file mode 100644
index 000000000..273e497ea
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/AuditDispatcherLauncher.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher;
+
+import org.apache.ranger.audit.server.AuditConfig;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AuditDispatcherLauncher {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditDispatcherLauncher.class);
+
+ private static final String CONFIG_DISPATCHER_TYPE = AuditServerConstants.PROP_DISPATCHER_TYPE;
+
+ private AuditDispatcherLauncher() {
+ }
+
+ public static void main(String[] args) {
+ try {
+ String dispatcherType = System.getProperty(CONFIG_DISPATCHER_TYPE);
+ if (dispatcherType == null) {
+ LOG.error("Dispatcher type is not specified. Cannot determine which dispatcher jars to load.");
+ System.exit(1);
+ }
+
+ String specificConfig = getDispatcherConfigPath(dispatcherType);
+ AuditConfig config = new AuditConfig();
+ config.addResourceIfReadable(specificConfig);
+
+ LOG.info("Initializing classloader for dispatcher type: {}", dispatcherType);
+
+ // Build dynamic classpath
+ String homeDir = System.getenv("AUDIT_DISPATCHER_HOME_DIR");
+ if (homeDir == null) {
+ homeDir = System.getProperty("user.dir");
+ }
+
+ File dispatcherLibDir = new File(homeDir, "lib/dispatchers/" + dispatcherType);
+ List<URL> urls = new ArrayList<>();
+
+ if (dispatcherLibDir.exists() && dispatcherLibDir.isDirectory()) {
+ File[] jars = dispatcherLibDir.listFiles((dir, name) -> name.endsWith(".jar"));
+ if (jars != null) {
+ for (File jar : jars) {
+ urls.add(jar.toURI().toURL());
+ }
+ }
+ } else {
+ LOG.warn("Dispatcher lib directory does not exist: {}", dispatcherLibDir.getAbsolutePath());
+ }
+
+ ClassLoader parentClassLoader = AuditDispatcherLauncher.class.getClassLoader();
+ URLClassLoader dispatcherClassLoader = new URLClassLoader(urls.toArray(new URL[0]), parentClassLoader);
+
+ Thread.currentThread().setContextClassLoader(dispatcherClassLoader);
+
+ String mainClassName = config.get("ranger.audit.dispatcher.main.class", "org.apache.ranger.audit.dispatcher.AuditDispatcherApplication");
+
+ LOG.info("Launching main class: {}", mainClassName);
+
+ Class<?> mainClass = Class.forName(mainClassName, true, dispatcherClassLoader);
+ Method mainMethod = mainClass.getMethod("main", String[].class);
+ mainMethod.invoke(null, (Object) args);
+ } catch (Exception e) {
+ LOG.error("Failed to launch audit dispatcher", e);
+ System.exit(1);
+ }
+ }
+
+ private static String getDispatcherConfigPath(String dispatcherType) {
+ String confDir = System.getenv("AUDIT_DISPATCHER_CONF_DIR");
+ if (confDir == null || confDir.trim().isEmpty()) {
+ confDir = System.getProperty("ranger.audit.dispatcher.conf.dir");
+ }
+
+ String fileName = "ranger-audit-dispatcher-" + dispatcherType + "-site.xml";
+ if (confDir != null && !confDir.trim().isEmpty()) {
+ return confDir + File.separator + fileName;
+ }
+
+ return fileName;
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/EmbeddedServer.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/EmbeddedServer.java
new file mode 100644
index 000000000..e169381ab
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/EmbeddedServer.java
@@ -0,0 +1,784 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.connector.Connector;
+import org.apache.catalina.core.StandardContext;
+import org.apache.catalina.startup.Tomcat;
+import org.apache.catalina.valves.AccessLogValve;
+import org.apache.catalina.webresources.StandardRoot;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.BouncyCastleFipsKeyStoreProvider;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditConfig;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class EmbeddedServer {
+ private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class);
+
+ private static final String ACCESS_LOG_PREFIX = "accesslog.prefix";
+ private static final String ACCESS_LOG_DATE_FORMAT = "accesslog.dateformat";
+ private static final String ACCESS_LOG_PATTERN = "accesslog.pattern";
+ private static final String ACCESS_LOG_ROTATE_MAX_DAYS = "accesslog.rotate.max.days";
+ private static final String DEFAULT_LOG_DIR = "/tmp";
+ private static final int DEFAULT_HTTP_PORT = 6081;
+ private static final int DEFAULT_HTTPS_PORT = -1;
+ private static final int DEFAULT_SHUTDOWN_PORT = 6185;
+ private static final String DEFAULT_SHUTDOWN_COMMAND = "SHUTDOWN";
+ private static final String DEFAULT_WEBAPPS_ROOT_FOLDER = "webapps";
+ private static final String DEFAULT_ENABLED_PROTOCOLS = "TLSv1.2";
+ public static final String DEFAULT_NAME_RULE = "DEFAULT";
+ private static final String RANGER_KEYSTORE_FILE_TYPE_DEFAULT = "jks";
+ private static final String RANGER_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks";
+ private static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2";
+ private static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm();
+ private static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm();
+ public static final String KEYSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType();
+ public static final String TRUSTSTORE_FILE_TYPE_DEFAULT = KeyStore.getDefaultType();
+
+ private final Configuration configuration;
+ private final String appName;
+ private final String configPrefix;
+ private volatile Tomcat server;
+ private volatile Context webappContext;
+
+ public EmbeddedServer(Configuration configuration, String appName, String configPrefix) {
+ LOG.info("==> EmbeddedServer(appName={}, configPrefix={})", appName, configPrefix);
+
+ this.configuration = new Configuration(configuration);
+ this.appName = appName;
+ this.configPrefix = configPrefix;
+
+ LOG.info("<== EmbeddedServer(appName={}, configPrefix={}", appName, configPrefix);
+ }
+
+ public static void main(String[] args) {
+ Configuration config = new AuditConfig();
+ String appName = AuditServerConstants.AUDIT_SERVER_APPNAME;
+ String configPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER;
+
+ new EmbeddedServer(config, appName, configPrefix).start();
+ }
+
+ @SuppressWarnings("deprecation")
+ public void start() {
+ LOG.info("==> EmbeddedServer.start(appName={})", appName);
+
+ SSLContext sslContext = getSSLContext();
+
+ if (sslContext != null) {
+ SSLContext.setDefault(sslContext);
+ }
+
+ this.server = new Tomcat();
+
+ // Add shutdown hook for graceful cleanup
+ addShutdownHook();
+
+ String logDir = getConfig("log.dir", DEFAULT_LOG_DIR);
+ String hostName = getConfig("host");
+ int serverPort = getIntConfig("http.port", DEFAULT_HTTP_PORT);
+ int sslPort = getIntConfig("https.port", DEFAULT_HTTPS_PORT);
+ int shutdownPort = getIntConfig("shutdown.port", DEFAULT_SHUTDOWN_PORT);
+ String shutdownCommand = getConfig("shutdown.command", DEFAULT_SHUTDOWN_COMMAND);
+ boolean isHttpsEnabled = getBooleanConfig("https.attrib.ssl.enabled", false);
+ boolean ajpEnabled = getBooleanConfig("ajp.enabled", false);
+
+ server.setHostname(hostName);
+ server.setPort(serverPort);
+ server.getServer().setPort(shutdownPort);
+ server.getServer().setShutdown(shutdownCommand);
+
+ if (ajpEnabled) {
+ Connector ajpConnector = new Connector("org.apache.coyote.ajp.AjpNioProtocol");
+
+ ajpConnector.setPort(serverPort);
+ ajpConnector.setProperty("protocol", "AJP/1.3");
+
+ server.getService().addConnector(ajpConnector);
+
+ // Making this as a default connector
+ server.setConnector(ajpConnector);
+
+ LOG.info("Created AJP Connector");
+ } else if (isHttpsEnabled && sslPort > 0) {
+ String clientAuth = getConfig("https.attrib.clientAuth", "false");
+ String providerPath = getConfig("credential.provider.path");
+ String keyAlias = getConfig("https.attrib.keystore.credential.alias", "keyStoreCredentialAlias");
+ String keystorePass = null;
+ String enabledProtocols = getConfig("https.attrib.ssl.enabled.protocols", DEFAULT_ENABLED_PROTOCOLS);
+ String ciphers = getConfig("tomcat.ciphers");
+
+ if (StringUtils.equalsIgnoreCase(clientAuth, "false")) {
+ clientAuth = getConfig("https.attrib.client.auth", "want");
+ }
+
+ if (providerPath != null && keyAlias != null) {
+ keystorePass = getDecryptedString(providerPath.trim(), keyAlias.trim(), getConfig("keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT));
+
+ if (StringUtils.isBlank(keystorePass) || StringUtils.equalsIgnoreCase(keystorePass.trim(), "none")) {
+ keystorePass = getConfig("https.attrib.keystore.pass");
+ }
+ }
+
+ Connector ssl = new Connector();
+ String sslKeystoreKeyAlias = getConfig("https.attrib.keystore.keyalias", "");
+ ssl.setPort(sslPort);
+ ssl.setSecure(true);
+ ssl.setScheme("https");
+ ssl.setAttribute("SSLEnabled", "true");
+ ssl.setAttribute("sslProtocol", getConfig("https.attrib.ssl.protocol", "TLSv1.2"));
+ ssl.setAttribute("clientAuth", clientAuth);
+ if (StringUtils.isNotBlank(sslKeystoreKeyAlias)) {
+ ssl.setAttribute("keyAlias", sslKeystoreKeyAlias);
+ }
+ ssl.setAttribute("keystorePass", keystorePass);
+ ssl.setAttribute("keystoreFile", getKeystoreFile());
+ ssl.setAttribute("sslEnabledProtocols", enabledProtocols);
+ ssl.setAttribute("keystoreType", getConfig("audit.keystore.file.type", KEYSTORE_FILE_TYPE_DEFAULT));
+ ssl.setAttribute("truststoreType", getConfig("audit.truststore.file.type", TRUSTSTORE_FILE_TYPE_DEFAULT));
+
+ if (StringUtils.isNotBlank(ciphers)) {
+ ssl.setAttribute("ciphers", ciphers);
+ }
+
+ server.getService().addConnector(ssl);
+
+ //
+ // Making this as a default connector
+ //
+ server.setConnector(ssl);
+ }
+
+ updateHttpConnectorAttribConfig(server);
+
+ File logDirectory = new File(logDir);
+
+ if (!logDirectory.exists()) {
+ boolean created = logDirectory.mkdirs();
+ if (!created) {
+ LOG.error("Failed to create log directory: {}", logDir);
+ throw new RuntimeException("Failed to create log directory: " + logDir);
+ }
+ }
+
+ String logPattern = getConfig(ACCESS_LOG_PATTERN, "%h %l %u %t \"%r\" %s %b");
+
+ AccessLogValve valve = new AccessLogValve();
+
+ valve.setRotatable(true);
+ valve.setAsyncSupported(true);
+ valve.setBuffered(false);
+ valve.setEnabled(true);
+ valve.setPrefix(getConfig(ACCESS_LOG_PREFIX, "access_log-" + hostName + "-"));
+ valve.setFileDateFormat(getConfig(ACCESS_LOG_DATE_FORMAT, "yyyy-MM-dd.HH"));
+ valve.setDirectory(logDirectory.getAbsolutePath());
+ valve.setSuffix(".log");
+ valve.setPattern(logPattern);
+ valve.setMaxDays(getIntConfig(ACCESS_LOG_ROTATE_MAX_DAYS, 15));
+
+ server.getHost().getPipeline().addValve(valve);
+
+ try {
+ String webappDir = getConfig("webapp.dir");
+
+ if (StringUtils.isBlank(webappDir)) {
+ LOG.error("Tomcat Server failed to start: {}.webapp.dir is not set", configPrefix);
+
+ System.exit(1);
+ }
+
+ String webContextName = getConfig("contextName", "");
+
+ if (StringUtils.isBlank(webContextName)) {
+ webContextName = "";
+ } else if (!webContextName.startsWith("/")) {
+ LOG.info("Context Name [{}] is being loaded as [ /{}]", webContextName, webContextName);
+
+ webContextName = "/" + webContextName;
+ }
+
+ File wad = new File(webappDir);
+
+ if (wad.isDirectory()) {
+ LOG.info("Webapp dir={}, webAppName={}", webappDir, webContextName);
+ } else if (wad.isFile()) {
+ File webAppDir = new File(DEFAULT_WEBAPPS_ROOT_FOLDER);
+
+ if (!webAppDir.exists()) {
+ boolean created = webAppDir.mkdirs();
+ if (!created) {
+ LOG.error("Failed to create webapp directory: {}", DEFAULT_WEBAPPS_ROOT_FOLDER);
+ throw new RuntimeException("Failed to create webapp directory: " + DEFAULT_WEBAPPS_ROOT_FOLDER);
+ }
+ }
+
+ LOG.info("Webapp file={}, webAppName={}", webappDir, webContextName);
+ }
+
+ LOG.info("Adding webapp [{}] = path [{}] .....", webContextName, webappDir);
+
+ this.webappContext = server.addWebapp(webContextName, new File(webappDir).getAbsolutePath());
+
+ // Set the parent classloader to the current thread context classloader
+ // This is crucial for isolated classloading so Tomcat can find the dispatcher classes
+ this.webappContext.setParentClassLoader(Thread.currentThread().getContextClassLoader());
+
+ if (webappContext instanceof StandardContext) {
+ boolean allowLinking = getBooleanConfig("allow.linking", true);
+ StandardContext standardContext = (StandardContext) webappContext;
+ standardContext.setDelegate(true);
+ String workDirPath = getConfig("tomcat.work.dir", "");
+ if (!workDirPath.isEmpty() && new File(workDirPath).exists()) {
+ standardContext.setWorkDir(workDirPath);
+ } else {
+ LOG.debug("Skipping to set tomcat server work directory {} as it is blank or directory does not exist.", workDirPath);
+ }
+ WebResourceRoot resRoot = new StandardRoot(standardContext);
+ webappContext.setResources(resRoot);
+ webappContext.getResources().setAllowLinking(allowLinking);
+ LOG.debug("Tomcat Configuration - allowLinking=[{}]", allowLinking);
+ } else {
+ LOG.error("Tomcat Context [{}] is either NULL OR it's NOT instanceof StandardContext", webappContext);
+ }
+
+ webappContext.init();
+
+ LOG.info("Finished init of webapp [{}] = path [{}].", webContextName, webappDir);
+ } catch (LifecycleException lce) {
+ LOG.error("Tomcat Server failed to start webapp", lce);
+ }
+
+ String authenticationType = getConfig(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE);
+
+ if (StringUtils.equalsIgnoreCase(authenticationType, AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS)) {
+ String keyTab = getConfig(AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB);
+ String principal = null;
+ String nameRules = getConfig(AuditServerConstants.PROP_HADOOP_KERBEROS_NAME_RULES, DEFAULT_NAME_RULE);
+
+ try {
+ principal = SecureClientLogin.getPrincipal(getConfig(AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL), hostName);
+ } catch (IOException excp) {
+ LOG.warn("Failed to get principal" + excp);
+ }
+
+ if (SecureClientLogin.isKerberosCredentialExists(principal, keyTab)) {
+ try {
+ AuditServerLogFormatter.builder("Kerberos Login Attempt")
+ .add("Principal", principal)
+ .add("Keytab", keyTab)
+ .add("Name rules", nameRules)
+ .logInfo(LOG);
+
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE, AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS);
+ hadoopConf.set(AuditServerConstants.PROP_HADOOP_KERBEROS_NAME_RULES, nameRules);
+ UserGroupInformation.setConfiguration(hadoopConf);
+ UserGroupInformation.loginUserFromKeytab(principal, keyTab);
+ UserGroupInformation currentUGI = UserGroupInformation.getLoginUser();
+
+ AuditServerLogFormatter.builder("Kerberos Login Successful")
+ .add("UGI Username", currentUGI.getUserName())
+ .add("UGI Real User", currentUGI.getUserName())
+ .add("Authentication Method", currentUGI.getAuthenticationMethod().toString())
+ .add("Has Kerberos Credentials", currentUGI.hasKerberosCredentials())
+ .logInfo(LOG);
+
+ MiscUtil.setUGILoginUser(currentUGI, null);
+
+ LOG.info("Starting Server using kerberos credential");
+ startServer(server);
+ } catch (Exception excp) {
+ LOG.error("Tomcat Server failed to start", excp);
+ }
+ } else {
+ LOG.warn("Kerberos principal={} not found in keytab={}. Starting server in non-kerberos mode", principal, keyTab);
+
+ startServer(server);
+ }
+ } else {
+ LOG.info("Starting server in non-kerberos mode");
+
+ startServer(server);
+ }
+
+ LOG.info("<== EmbeddedServer.start(appName={})", appName);
+ }
+
+ public String getDecryptedString(String credentialProviderPath, String alias, String storeType) {
+ String ret = null;
+
+ if (StringUtils.isNotBlank(credentialProviderPath) && StringUtils.isNotBlank(alias)) {
+ try {
+ Configuration conf = new Configuration();
+ String prefixJceks = JavaKeyStoreProvider.SCHEME_NAME + "://file";
+ String prefixLocalJceks = "localjceks://file";
+ String prefixBcfks = BouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file";
+ String prefixLocalBcfks = LocalBouncyCastleFipsKeyStoreProvider.SCHEME_NAME + "://file";
+
+ String lowerPath = credentialProviderPath.toLowerCase();
+ if (lowerPath.startsWith(prefixJceks.toLowerCase()) || lowerPath.startsWith(prefixLocalJceks.toLowerCase()) || lowerPath.startsWith(prefixBcfks.toLowerCase()) || lowerPath.startsWith(prefixLocalBcfks.toLowerCase())) {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath);
+ } else {
+ if (credentialProviderPath.startsWith("/")) {
+ if ("bcfks".equalsIgnoreCase(storeType)) {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath);
+ } else {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + credentialProviderPath);
+ }
+ } else {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, prefixJceks + "/" + credentialProviderPath);
+ }
+ }
+
+ List<CredentialProvider> providers = CredentialProviderFactory.getProviders(conf);
+
+ LOG.debug("CredentialProviderPath={} alias={} storeType={}", credentialProviderPath, alias, storeType);
+ LOG.debug("List of CredentialProvider = {}", providers.toString());
+
+ for (CredentialProvider provider : providers) {
+ List<String> aliasesList = provider.getAliases();
+
+ if (CollectionUtils.isNotEmpty(aliasesList) && aliasesList.contains(alias)) {
+ CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias);
+ char[] pass = credEntry.getCredential();
+
+ if (pass != null && pass.length > 0) {
+ ret = String.valueOf(pass);
+
+ break;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("CredentialReader failed while decrypting provided string. Reason: {}", ex.toString());
+
+ ret = null;
+ }
+ }
+
+ LOG.debug("getDecryptedString() : ret = {}", ret);
+
+ return ret;
+ }
+
+ protected long getLongConfig(String key, long defaultValue) {
+ long ret = defaultValue;
+ String retStr = getConfig(key);
+
+ try {
+ if (retStr != null) {
+ ret = Long.parseLong(retStr);
+ }
+ } catch (Exception err) {
+ LOG.warn(retStr + " can't be parsed to long. Reason: {}", err.toString());
+ }
+
+ return ret;
+ }
+
+ private void startServer(final Tomcat server) {
+ LOG.info("==> EmbeddedServer.startServer(appName={})", appName);
+
+ try {
+ server.start();
+
+ server.getServer().await();
+
+ shutdownServer();
+ } catch (LifecycleException e) {
+ LOG.error("Tomcat Server failed to start", e);
+
+ e.printStackTrace();
+ } catch (Exception e) {
+ LOG.error("Tomcat Server failed to start", e);
+
+ e.printStackTrace();
+ }
+
+ LOG.info("<== EmbeddedServer.startServer(appName={})", appName);
+ }
+
+ private void addShutdownHook() {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ LOG.info("==> EmbeddedServer shutdown hook triggered");
+ try {
+ gracefulShutdown();
+ } catch (Exception e) {
+ LOG.error("Error during graceful shutdown", e);
+ }
+ LOG.info("<== EmbeddedServer shutdown hook completed");
+ }, "EmbeddedServer-ShutdownHook"));
+ }
+
+ private void gracefulShutdown() {
+ LOG.info("==> EmbeddedServer.gracefulShutdown()");
+
+ // Instead of trying to manually shutdown Spring context, let's rely on Tomcat's
+ // normal webapp lifecycle which will trigger ContextLoaderListener.contextDestroyed()
+ if (server != null) {
+ try {
+ LOG.info("Initiating Tomcat server stop to trigger webapp lifecycle shutdown");
+ server.stop();
+ LOG.info("Tomcat server stop completed");
+ } catch (Exception e) {
+ LOG.error("Error stopping Tomcat server during graceful shutdown", e);
+
+ // Fallback: try to stop the webapp context directly
+ if (webappContext != null) {
+ try {
+ LOG.info("Fallback: stopping webapp context directly");
+ webappContext.stop();
+ LOG.info("Webapp context stopped");
+ } catch (Exception contextE) {
+ LOG.error("Error stopping webapp context", contextE);
+ }
+ }
+ }
+ }
+
+ // Give some time for Spring components to shutdown through normal lifecycle
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for Spring shutdown", e);
+ Thread.currentThread().interrupt();
+ }
+
+ LOG.info("<== EmbeddedServer.gracefulShutdown()");
+ }
+
+ private void shutdownServer() {
+ LOG.info("==> EmbeddedServer.shutdownServer(appName={})", appName);
+
+ int timeWaitForShutdownInSeconds = getIntConfig("waitTimeForForceShutdownInSeconds", 0);
+
+ if (timeWaitForShutdownInSeconds > 0) {
+ long endTime = System.currentTimeMillis() + (timeWaitForShutdownInSeconds * 1000L);
+
+ LOG.info("Will wait for all threads to shutdown gracefully. Final shutdown Time: {}", new Date(endTime));
+
+ while (System.currentTimeMillis() < endTime) {
+ int activeCount = Thread.activeCount();
+
+ if (activeCount == 0) {
+ LOG.info("Number of active threads = {}", activeCount);
+
+ break;
+ } else {
+ LOG.info("Number of active threads = {}. Waiting for all threads to shutdown ...", activeCount);
+
+ try {
+ Thread.sleep(5000L);
+ } catch (InterruptedException e) {
+ LOG.warn("shutdownServer process is interrupted with exception", e);
+
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.info("Shutting down the Server.");
+
+ LOG.info("<== EmbeddedServer.shutdownServer(appName={})", appName);
+
+ System.exit(0);
+ }
+
+ @SuppressWarnings("deprecation")
+ private void updateHttpConnectorAttribConfig(Tomcat server) {
+ server.getConnector().setAllowTrace(getBooleanConfig("http.connector.attrib.allowTrace", false));
+ server.getConnector().setAsyncTimeout(getLongConfig("http.connector.attrib.asyncTimeout", 10000));
+ server.getConnector().setEnableLookups(getBooleanConfig("http.connector.attrib.enableLookups", false));
+ server.getConnector().setMaxParameterCount(getIntConfig("http.connector.attrib.maxParameterCount", 10000));
+ server.getConnector().setMaxPostSize(getIntConfig("http.connector.attrib.maxPostSize", 2097152));
+ server.getConnector().setMaxSavePostSize(getIntConfig("http.connector.attrib.maxSavePostSize", 4096));
+ server.getConnector().setParseBodyMethods(getConfig("http.connector.attrib.methods", "POST"));
+ server.getConnector().setURIEncoding(getConfig("http.connector.attrib.URIEncoding", "UTF-8"));
+ server.getConnector().setProperty("acceptCount", getConfig("http.connector.attrib.acceptCount", "1024"));
+ server.getConnector().setXpoweredBy(false);
+ server.getConnector().setAttribute("server", getConfig("servername", "Audit Server"));
+
+ int maxThreads = getIntConfig("http.connector.attrib.maxThreads", 2000);
+ int maxKeepAliveRequests = getIntConfig("http.connector.attrib.maxKeepAliveRequests", 1000);
+ int minSpareThreads = getIntConfig("http.connector.attrib.minSpareThreads", (int) (maxThreads * 0.8)); // (default 80% of maxThreads)
+ boolean prestartminSpareThreads = getBooleanConfig("http.connector.attrib.prestartminSpareThreads", true);
+
+ server.getConnector().setAttribute("maxThreads", maxThreads);
+ server.getConnector().setAttribute("maxKeepAliveRequests", maxKeepAliveRequests);
+ server.getConnector().setAttribute("minSpareThreads", minSpareThreads);
+ server.getConnector().setAttribute("prestartminSpareThreads", prestartminSpareThreads);
+ server.getConnector().setProperty("sendReasonPhrase", getConfig(configPrefix + "http.connector.property.sendReasonPhrase", "true"));
+
+ for (Map.Entry<String, String> entry : configuration) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (key != null && key.startsWith(configPrefix + "http.connector.property.")) {
+ String property = key.replace(configPrefix + "http.connector.property.", "");
+
+ server.getConnector().setProperty(property, value);
+
+ LOG.info("{}:{}", property, server.getConnector().getProperty(property));
+ }
+ }
+ }
+
+ private SSLContext getSSLContext() {
+ SSLContext ret = null;
+ KeyManager[] kmList = getKeyManagers();
+ TrustManager[] tmList = getTrustManagers();
+
+ if (tmList != null) {
+ try {
+ ret = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
+
+ ret.init(kmList, tmList, new SecureRandom());
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is not available in the environment", e);
+ } catch (KeyManagementException e) {
+ LOG.error("Unable to initials the SSLContext", e);
+ }
+ }
+
+ return ret;
+ }
+
+ private KeyManager[] getKeyManagers() {
+ KeyManager[] ret = null;
+ String keyStoreFile = getConfig("keystore.file");
+ String keyStoreAlias = getConfig("keystore.alias");
+ String credentialProviderPath = getConfig("credential.provider.path");
+ String keyStoreFilepwd = getCredential(credentialProviderPath, keyStoreAlias);
+
+ if (StringUtils.isNotBlank(keyStoreFile) && StringUtils.isNotBlank(keyStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(keyStoreFile);
+
+ if (in != null) {
+ KeyStore keyStore = KeyStore.getInstance(RANGER_KEYSTORE_FILE_TYPE_DEFAULT);
+
+ keyStore.load(in, keyStoreFilepwd.toCharArray());
+
+ KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
+
+ keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
+
+ ret = keyManagerFactory.getKeyManagers();
+ } else {
+ LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL Keystore Files", e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL Keystore Files", e);
+ } catch (UnrecoverableKeyException e) {
+ LOG.error("Unable to recover the key from keystore", e);
+ } finally {
+ close(in, keyStoreFile);
+ }
+ }
+
+ return ret;
+ }
+
+ private TrustManager[] getTrustManagers() {
+ TrustManager[] ret = null;
+ String truststoreFile = getConfig("truststore.file");
+ String truststoreAlias = getConfig("truststore.alias");
+ String credentialProviderPath = getConfig("credential.provider.path");
+ String trustStoreFilepwd = getCredential(credentialProviderPath, truststoreAlias);
+
+ if (StringUtils.isNotBlank(truststoreFile) && StringUtils.isNotBlank(trustStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(truststoreFile);
+
+ if (in != null) {
+ KeyStore trustStore = KeyStore.getInstance(RANGER_TRUSTSTORE_FILE_TYPE_DEFAULT);
+
+ trustStore.load(in, trustStoreFilepwd.toCharArray());
+
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
+
+ trustManagerFactory.init(trustStore);
+
+ ret = trustManagerFactory.getTrustManagers();
+ } else {
+ LOG.error("Unable to obtain truststore from file [{}]", truststoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL TrustStore File:{}", truststoreFile, e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL TrustStore Files:{}", truststoreFile, e);
+ } finally {
+ close(in, truststoreFile);
+ }
+ }
+
+ return ret;
+ }
+
+ private String getKeystoreFile() {
+ String ret = getConfig("https.attrib.keystore.file");
+
+ if (StringUtils.isBlank(ret)) {
+ // new property not configured, lets use the old property
+ ret = getConfig("https.attrib.keystore.file");
+ }
+
+ return ret;
+ }
+
+ private String getCredential(String url, String alias) {
+ return RangerCredentialProvider.getInstance().getCredentialString(url, alias);
+ }
+
+ private String getConfig(String key) {
+ String propertyWithPrefix = configPrefix + key;
+ String value = configuration.get(propertyWithPrefix);
+ if (value == null) { // config-with-prefix not found; look at System properties
+ value = System.getProperty(propertyWithPrefix);
+ if (value == null) { // config-with-prefix not found in System properties; look for config-without-prefix
+ value = configuration.get(key);
+ if (value == null) { // config-without-prefix not found; look at System properties
+ value = System.getProperty(key);
+ }
+ }
+ }
+
+ return value;
+ }
+
+ private String getConfig(String key, String defaultValue) {
+ String ret = getConfig(key);
+
+ if (ret == null) {
+ ret = defaultValue;
+ }
+
+ return ret;
+ }
+
+ private int getIntConfig(String key, int defaultValue) {
+ int ret = defaultValue;
+ String strValue = getConfig(key);
+
+ try {
+ if (strValue != null) {
+ ret = Integer.parseInt(strValue);
+ }
+ } catch (Exception err) {
+ LOG.warn(strValue + " can't be parsed to int. Reason: {}", err.toString());
+ }
+
+ return ret;
+ }
+
+ private boolean getBooleanConfig(String key, boolean defaultValue) {
+ boolean ret = defaultValue;
+ String strValue = getConfig(key);
+
+ if (StringUtils.isNotBlank(strValue)) {
+ ret = Boolean.valueOf(strValue);
+ }
+
+ return ret;
+ }
+
+ private InputStream getFileInputStream(String fileName) throws IOException {
+ InputStream ret = null;
+
+ if (StringUtils.isNotEmpty(fileName)) {
+ File f = new File(fileName);
+
+ if (f.exists()) {
+ ret = new FileInputStream(f);
+ } else {
+ ret = ClassLoader.getSystemResourceAsStream(fileName);
+ }
+ }
+
+ return ret;
+ }
+
+ private void close(InputStream str, String filename) {
+ if (str != null) {
+ try {
+ str.close();
+ } catch (IOException excp) {
+ LOG.error("Error while closing file: [{}]", filename, excp);
+ }
+ }
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcher.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcher.java
new file mode 100644
index 000000000..26c3bcb75
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcher.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+
+/**
+ * Interface for Ranger Kafka dispatchers that consume audit events from Kafka
+ * and forward them to various destinations like Solr, HDFS, etc.
+ */
+public interface AuditDispatcher extends Runnable {
+ /**
+ * Start consuming messages from Kafka topic and process them.
+ * This method should implement the main consumption loop.
+ * Implementation should handle subscription to topics and polling for messages.
+ */
+ @Override
+ void run();
+
+ /**
+ * Shutdown the dispatcher gracefully.
+ * This method should clean up resources and close connections.
+ */
+ void shutdown();
+
+ /**
+ * Get the underlying Kafka dispatcher instance.
+ *
+ * @return KafkaConsumer instance used by this dispatcher
+ */
+ KafkaConsumer<String, String> getDispatcher();
+
+ /**
+ * Get the topic name this dispatcher is subscribed to.
+ *
+ * @return Kafka topic name
+ */
+ String getTopicName();
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherBase.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherBase.java
new file mode 100644
index 000000000..ae9327129
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherBase.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.utils.AuditMessageQueueUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public abstract class AuditDispatcherBase implements AuditDispatcher {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditDispatcherBase.class);
+
+ public final Properties dispatcherProps = new Properties();
+ public final KafkaConsumer<String, String> dispatcher;
+ public final String topicName;
+ public final String dispatcherGroupId;
+
+ protected final AtomicBoolean running = new AtomicBoolean(false);
+ protected final Map<String, DispatcherWorker> dispatcherWorkers = new ConcurrentHashMap<>();
+ protected ExecutorService dispatcherThreadPool;
+ protected int dispatcherThreadCount = 1;
+ protected String offsetCommitStrategy = AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY;
+ protected long offsetCommitInterval = AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS;
+
+ public AuditDispatcherBase(Properties props, String propPrefix, String dispatcherGroupId) throws Exception {
+ this.dispatcherGroupId = getDispatcherGroupId(props, propPrefix, dispatcherGroupId);
+
+ dispatcherProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS));
+ dispatcherProps.put(ConsumerConfig.GROUP_ID_CONFIG, this.dispatcherGroupId);
+ dispatcherProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringDeserializer"));
+ dispatcherProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Class.forName("org.apache.kafka.common.serialization.StringDeserializer"));
+
+ String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL);
+ dispatcherProps.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol);
+
+ dispatcherProps.put("sasl.mechanism", MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM));
+ dispatcherProps.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME);
+
+ if (securityProtocol.toUpperCase().contains(AuditServerConstants.PROP_SECURITY_PROTOCOL_VALUE)) {
+ dispatcherProps.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, AuditMessageQueueUtils.getJAASConfig(props, propPrefix));
+ }
+
+ dispatcherProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_MAX_POLL_RECORDS, AuditServerConstants.DEFAULT_MAX_POLL_RECORDS));
+ dispatcherProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+
+ // Configure re-balancing parameters for subscribe mode
+ // These ensure stable dispatcher group behavior during horizontal scaling
+ int sessionTimeoutMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_SESSION_TIMEOUT_MS, AuditServerConstants.DEFAULT_SESSION_TIMEOUT_MS);
+ int maxPollIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_MAX_POLL_INTERVAL_MS, AuditServerConstants.DEFAULT_MAX_POLL_INTERVAL_MS);
+ int heartbeatIntervalMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_HEARTBEAT_INTERVAL_MS, AuditServerConstants.DEFAULT_HEARTBEAT_INTERVAL_MS);
+
+ dispatcherProps.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeoutMs);
+ dispatcherProps.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs);
+ dispatcherProps.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, heartbeatIntervalMs);
+
+ // Configure partition assignment strategy
+ String partitionAssignmentStrategy = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_PARTITION_ASSIGNMENT_STRATEGY, AuditServerConstants.DEFAULT_PARTITION_ASSIGNMENT_STRATEGY);
+ dispatcherProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partitionAssignmentStrategy);
+
+ LOG.info("Dispatcher '{}' configured for subscription-based partition assignment with re-balancing support", this.dispatcherGroupId);
+ LOG.info("Re-balancing config - session.timeout.ms: {}, max.poll.interval.ms: {}, heartbeat.interval.ms: {}", sessionTimeoutMs, maxPollIntervalMs, heartbeatIntervalMs);
+ LOG.info("Partition assignment strategy: {}", partitionAssignmentStrategy);
+
+ dispatcher = new KafkaConsumer<>(dispatcherProps);
+ topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC);
+ }
+
+ AuditDispatcherBase(String dispatcherGroupId, KafkaConsumer<String, String> dispatcher, String topicName) {
+ this.dispatcherGroupId = dispatcherGroupId;
+ this.dispatcher = dispatcher;
+ this.topicName = topicName;
+ }
+
+ @Override
+ public KafkaConsumer<String, String> getDispatcher() {
+ return dispatcher;
+ }
+
+ @Override
+ public String getTopicName() {
+ return topicName;
+ }
+
+ public String getDispatcherGroupId() {
+ return dispatcherGroupId;
+ }
+
+ private String getDispatcherGroupId(Properties props, String propPrefix, String defaultDispatcherGroupId) {
+ String configuredGroupId = MiscUtil.getStringProperty(props, propPrefix + ".kafka.group.id");
+ if (configuredGroupId != null && !configuredGroupId.trim().isEmpty()) {
+ return configuredGroupId.trim();
+ }
+ return defaultDispatcherGroupId;
+ }
+
+ protected abstract String getDispatcherName();
+
+ protected abstract DispatcherWorker createDispatcherWorker(String workerId, List<Integer> assignedPartitions);
+
+ protected void shutdownDestination() {
+ // To be overridden by subclasses if needed
+ }
+
+ @Override
+ public void run() {
+ try {
+ LOG.info("Starting {} dispatcher", getClass().getSimpleName());
+ if (running.compareAndSet(false, true)) {
+ startDispatcherWorkers();
+ }
+
+ // Keep main thread alive while dispatcher threads are running
+ while (running.get()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ LOG.info("{} dispatcher main thread interrupted", getDispatcherName());
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Error in {}", getClass().getSimpleName(), e);
+ } finally {
+ shutdown();
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ LOG.info("==> {} shutdown()", getClass().getSimpleName());
+
+ running.set(false);
+
+ if (dispatcherThreadPool != null) {
+ dispatcherThreadPool.shutdownNow();
+ try {
+ if (!dispatcherThreadPool.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS)) {
+ LOG.warn("{} dispatcher thread pool did not terminate within 30 seconds", getDispatcherName());
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for {} dispatcher thread pool to terminate", getDispatcherName(), e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ dispatcherWorkers.clear();
+
+ if (dispatcher != null) {
+ try {
+ dispatcher.close();
+ } catch (Exception e) {
+ LOG.error("Error closing main dispatcher", e);
+ }
+ }
+
+ shutdownDestination();
+
+ LOG.info("<== {} shutdown() complete", getClass().getSimpleName());
+ }
+
+ protected void startDispatcherWorkers() {
+ LOG.info("==> AuditDispatcherBase.startDispatcherWorkers(): Creating {} workers for horizontal scaling", dispatcherThreadCount);
+ LOG.info("Each worker will subscribe to topic '{}' and process partitions assigned by Kafka", topicName);
+
+ dispatcherThreadPool = Executors.newFixedThreadPool(dispatcherThreadCount);
+ LOG.info("Created thread pool with {} threads for scalable {} consumption", dispatcherThreadCount, getDispatcherName());
+
+ for (int i = 0; i < dispatcherThreadCount; i++) {
+ String workerId = getDispatcherName().toLowerCase() + "-worker-" + i;
+ DispatcherWorker worker = createDispatcherWorker(workerId, new ArrayList<>());
+ dispatcherWorkers.put(workerId, worker);
+ dispatcherThreadPool.submit(worker);
+
+ LOG.info("Started {} dispatcher worker '{}' - will process ANY appId assigned by Kafka", getDispatcherName(), workerId);
+ }
+
+ LOG.info("<== AuditDispatcherBase.startDispatcherWorkers(): All {} workers started in SUBSCRIBE mode", dispatcherThreadCount);
+ }
+
+ protected abstract class DispatcherWorker implements Runnable {
+ protected final String workerId;
+ protected final List<Integer> assignedPartitions;
+ protected KafkaConsumer<String, String> workerDispatcher;
+
+ // Offset management
+ protected final Map<TopicPartition, OffsetAndMetadata> pendingOffsets = new HashMap<>();
+ protected final AtomicLong lastCommitTime = new AtomicLong(System.currentTimeMillis());
+ protected final AtomicInteger messagesProcessedSinceLastCommit = new AtomicInteger(0);
+
+ public DispatcherWorker(String workerId, List<Integer> assignedPartitions) {
+ this.workerId = workerId;
+ this.assignedPartitions = assignedPartitions;
+ }
+
+ protected abstract void processRecordBatch(ConsumerRecords<String, String> records);
+
+ @Override
+ public void run() {
+ try {
+ // Create dispatcher for this worker with offset management configuration
+ Properties workerDispatcherProps = new Properties();
+ workerDispatcherProps.putAll(dispatcherProps);
+
+ // Configure offset management based on strategy
+ configureOffsetManagement(workerDispatcherProps);
+
+ workerDispatcher = new KafkaConsumer<>(workerDispatcherProps);
+
+ // Create re-balance listener
+ AuditDispatcherRebalanceListener rebalanceListener = new AuditDispatcherRebalanceListener(
+ workerId,
+ getDispatcherName(),
+ topicName,
+ offsetCommitStrategy,
+ dispatcherGroupId,
+ workerDispatcher,
+ pendingOffsets,
+ messagesProcessedSinceLastCommit,
+ lastCommitTime,
+ assignedPartitions);
+
+ // Subscribe to topic with re-balance listener
+ workerDispatcher.subscribe(Collections.singletonList(topicName), rebalanceListener);
+
+ LOG.info("[{}-DISPATCHER] Worker '{}' subscribed successfully, waiting for partition assignment from Kafka", getDispatcherName(), workerId);
+ long threadId = Thread.currentThread().getId();
+ String threadName = Thread.currentThread().getName();
+ LOG.info("[{}-DISPATCHER-STARTUP] Worker '{}' [Thread-ID: {}, Thread-Name: '{}'] started | Topic: '{}' | Dispatcher-Group: {} | Mode: SUBSCRIBE",
+ getDispatcherName(), workerId, threadId, threadName, topicName, dispatcherGroupId);
+
+ // Consume messages
+ while (running.get()) {
+ ConsumerRecords<String, String> records = workerDispatcher.poll(Duration.ofMillis(100));
+
+ if (!records.isEmpty()) {
+ processRecordBatch(records);
+ // Handle offset committing based on strategy
+ handleOffsetCommitting();
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Error in {} dispatcher worker '{}'", getDispatcherName(), workerId, e);
+ } finally {
+ // Final offset commit before shutdown
+ commitPendingOffsets(true);
+
+ if (workerDispatcher != null) {
+ try {
+ LOG.info("{} Worker '{}': Unsubscribing from topic", getDispatcherName(), workerId);
+ workerDispatcher.unsubscribe();
+ } catch (Exception e) {
+ LOG.warn("{} Worker '{}': Error during unsubscribe", getDispatcherName(), workerId, e);
+ }
+
+ try {
+ LOG.info("{} Worker '{}': Closing dispatcher", getDispatcherName(), workerId);
+ workerDispatcher.close();
+ } catch (Exception e) {
+ LOG.error("Error closing dispatcher for {} worker '{}'", getDispatcherName(), workerId, e);
+ }
+ }
+ LOG.info("{} dispatcher worker '{}' stopped", getDispatcherName(), workerId);
+ }
+ }
+
+ private void configureOffsetManagement(Properties dispatcherProps) {
+ // Always disable auto commit - only batch or manual strategies supported
+ dispatcherProps.put("enable.auto.commit", "false");
+ LOG.debug("{} worker '{}' configured for manual offset commit with strategy: {}", getDispatcherName(), workerId, offsetCommitStrategy);
+ }
+
+ private void handleOffsetCommitting() {
+ boolean shouldCommit = false;
+ long currentTime = System.currentTimeMillis();
+
+ if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_BATCH.equals(offsetCommitStrategy)) {
+ // Commit after processing each batch
+ shouldCommit = !pendingOffsets.isEmpty();
+ } else if (AuditServerConstants.PROP_OFFSET_COMMIT_STRATEGY_MANUAL.equals(offsetCommitStrategy)) {
+ // Commit based on time interval
+ shouldCommit = (currentTime - lastCommitTime.get()) >= offsetCommitInterval && !pendingOffsets.isEmpty();
+ }
+
+ if (shouldCommit) {
+ commitPendingOffsets(false);
+ }
+ }
+
+ private void commitPendingOffsets(boolean isShutdown) {
+ if (pendingOffsets.isEmpty()) {
+ return;
+ }
+
+ try {
+ workerDispatcher.commitSync(pendingOffsets);
+
+ LOG.debug("{} worker '{}' committed {} offsets, processed {} messages",
+ getDispatcherName(), workerId, pendingOffsets.size(), messagesProcessedSinceLastCommit.get());
+
+ // Clear committed offsets
+ pendingOffsets.clear();
+ lastCommitTime.set(System.currentTimeMillis());
+ messagesProcessedSinceLastCommit.set(0);
+ } catch (Exception e) {
+ LOG.error("Error committing offsets in {} worker '{}': {}", getDispatcherName(), workerId, pendingOffsets, e);
+
+ if (isShutdown) {
+ // During shutdown, retry to avoid loss of any offsets
+ try {
+ Thread.sleep(1000);
+ workerDispatcher.commitSync(pendingOffsets);
+ LOG.info("Successfully committed offsets on retry during shutdown for {} worker '{}'", getDispatcherName(), workerId);
+ } catch (Exception retryException) {
+ LOG.error("Failed to commit offsets even on retry during shutdown for {} worker '{}'", getDispatcherName(), workerId, retryException);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherFactory.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherFactory.java
new file mode 100644
index 000000000..d330da0ff
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import java.util.Properties;
+
+/**
+ * Factory interface for creating audit dispatcher instances.
+ */
+@FunctionalInterface
+public interface AuditDispatcherFactory {
+ /**
+ * Create a dispatcher instance with the given configuration.
+ *
+ * @param props Configuration properties
+ * @param propPrefix Property prefix for dispatcher configuration
+ * @return Initialized dispatcher instance ready to run
+ * @throws Exception if dispatcher creation or initialization fails
+ */
+ AuditDispatcher createDispatcher(Properties props, String propPrefix) throws Exception;
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherRebalanceListener.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherRebalanceListener.java
new file mode 100644
index 000000000..b412541e8
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherRebalanceListener.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Reusable ConsumerRebalanceListener for Kafka dispatcher group re-balancing.
+ *
+ * This listener handles graceful partition re-balancing by:
+ * - Committing pending offsets before partitions are revoked
+ * - Updating partition assignments when partitions are assigned
+ * - Logging re-balancing events with customizable log prefixes
+ *
+ * Used by both AuditSolrDispatcher, AuditHDFSDispatcher and any new dispatchers added to the audit serve, ensuring
+ * no message duplication during scaling operations.
+ */
+public class AuditDispatcherRebalanceListener implements ConsumerRebalanceListener {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditDispatcherRebalanceListener.class);
+
+ private final String workerId;
+ private final String destinationType;
+ private final String topicName;
+ private final String offsetCommitStrategy;
+ private final String dispatcherGroupId;
+ private final KafkaConsumer<String, String> workerDispatcher;
+ private final Map<TopicPartition, OffsetAndMetadata> pendingOffsets;
+ private final AtomicInteger messagesProcessedSinceLastCommit;
+ private final AtomicLong lastCommitTime;
+ private final List<Integer> assignedPartitions;
+
+ public AuditDispatcherRebalanceListener(
+ String workerId,
+ String destinationType,
+ String topicName,
+ String offsetCommitStrategy,
+ String dispatcherGroupId,
+ KafkaConsumer<String, String> workerDispatcher,
+ Map<TopicPartition, OffsetAndMetadata> pendingOffsets,
+ AtomicInteger messagesProcessedSinceLastCommit,
+ AtomicLong lastCommitTime,
+ List<Integer> assignedPartitions) {
+ this.workerId = workerId;
+ this.destinationType = destinationType;
+ this.topicName = topicName;
+ this.offsetCommitStrategy = offsetCommitStrategy;
+ this.dispatcherGroupId = dispatcherGroupId;
+ this.workerDispatcher = workerDispatcher;
+ this.pendingOffsets = pendingOffsets;
+ this.messagesProcessedSinceLastCommit = messagesProcessedSinceLastCommit;
+ this.lastCommitTime = lastCommitTime;
+ this.assignedPartitions = assignedPartitions;
+ }
+
+ @Override
+ public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
+ LOG.info("[{}-REBALANCE] Worker '{}': Partitions REVOKED: {} (count: {})", destinationType, workerId, partitions, partitions.size());
+
+ // Commit pending offsets before partitions are revoked
+ if (!pendingOffsets.isEmpty()) {
+ try {
+ workerDispatcher.commitSync(pendingOffsets);
+ LOG.info("[{}-REBALANCE] Worker '{}': Successfully committed {} pending offsets before rebalance",
+ destinationType, workerId, pendingOffsets.size());
+ pendingOffsets.clear();
+ } catch (Exception e) {
+ LOG.error("[{}-REBALANCE] Worker '{}': Failed to commit offsets during rebalance",
+ destinationType, workerId, e);
+ }
+ }
+
+ // Reset counters
+ messagesProcessedSinceLastCommit.set(0);
+ lastCommitTime.set(System.currentTimeMillis());
+ }
+
+ @Override
+ public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
+ LOG.info("[{}-REBALANCE] Worker '{}': Partitions ASSIGNED: {} (count: {})", destinationType, workerId, partitions, partitions.size());
+
+ // Update assigned partitions list for tracking
+ assignedPartitions.clear();
+ for (TopicPartition tp : partitions) {
+ assignedPartitions.add(tp.partition());
+ }
+
+ // Log assignment details
+ LOG.info("[{}-DISPATCHER-ASSIGNED] Worker '{}' | Topic: '{}' | Partitions: {} | Offset-Strategy: {} | Dispatcher-Group: {}",
+ destinationType, workerId, topicName, assignedPartitions, offsetCommitStrategy, dispatcherGroupId);
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherTracker.java b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherTracker.java
new file mode 100644
index 000000000..c7bb7a941
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-common/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditDispatcherTracker.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+* Tracks active audit dispatcher instances.
+*/
+public class AuditDispatcherTracker {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditDispatcherTracker.class);
+ private static final AuditDispatcherTracker tracker = new AuditDispatcherTracker();
+ private final Map<String, AuditDispatcher> activeDispatchers = new ConcurrentHashMap<>();
+
+ private AuditDispatcherTracker() {
+ LOG.debug("AuditDispatcherTracker instance created by classloader: {}", this.getClass().getClassLoader());
+ }
+
+ public static AuditDispatcherTracker getInstance() {
+ return tracker;
+ }
+
+ /**
+ * Register an active dispatcher instance for a specific destination type for healthcheck.
+ *
+ * @param destinationType The destination type identifier (e.g., "solr", "hdfs", "elasticsearch")
+ * @param dispatcher The dispatcher instance
+ */
+ public void addActiveDispatcher(String destinationType, AuditDispatcher dispatcher) {
+ if (dispatcher == null) {
+ LOG.warn("Attempted to register null dispatcher for destination type: {}", destinationType);
+ return;
+ }
+ activeDispatchers.put(destinationType, dispatcher);
+ LOG.info("Registered active dispatcher for destination type: {}", destinationType);
+ }
+
+ public Set<String> getActiveDispatcherTypes() {
+ return activeDispatchers.keySet();
+ }
+
+ /**
+ * Called during shutdown after dispatchers have been stopped.
+ */
+ public void clearActiveDispatcher(String dispatcherType) {
+ LOG.debug("Clearing {} active dispatcher references", activeDispatchers.size());
+ activeDispatchers.remove(dispatcherType);
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/pom.xml b/audit-server/audit-dispatcher/dispatcher-hdfs/pom.xml
new file mode 100644
index 000000000..851c38448
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/pom.xml
@@ -0,0 +1,452 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>2.10.0-SNAPSHOT</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+
+ <artifactId>audit-dispatcher-hdfs</artifactId>
+ <packaging>jar</packaging>
+ <name>Ranger Audit Dispatcher HDFS</name>
+ <description>Kafka dispatcher service for writing audits to HDFS/S3/Azure</description>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ <version>${jersey-bundle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-spring</artifactId>
+ <version>${jersey-spring.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax.servlet.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop.thirdparty</groupId>
+ <artifactId>hadoop-shaded-guava</artifactId>
+ <version>${hadoop-shaded-guava.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${hive.storage-api.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ <version>${orc.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-shims</artifactId>
+ <version>${orc.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dest-hdfs</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${google.guava.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax.servlet.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-dispatcher-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-audit-server-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-annotations-api</artifactId>
+ <version>${tomcat.embed.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-el</artifactId>
+ <version>${tomcat.embed.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-jasper</artifactId>
+ <version>${tomcat.embed.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.ext</groupId>
+ <artifactId>jersey-spring5</artifactId>
+ <version>${jersey-client.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>jersey-hk2</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${springframework.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${springframework.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ <version>${springframework.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>audit-dispatcher-hdfs-${project.version}</finalName>
+ <resources>
+ <resource>
+ <filtering>true</filtering>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <configuration>
+ <rulesets>
+ <ruleset>${project.parent.basedir}/dev-support/ranger-pmd-ruleset.xml</ruleset>
+ </rulesets>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/HdfsDispatcherManager.java b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/HdfsDispatcherManager.java
new file mode 100644
index 000000000..df8523f27
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/HdfsDispatcherManager.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher;
+
+import org.apache.ranger.audit.dispatcher.kafka.AuditDispatcher;
+import org.apache.ranger.audit.dispatcher.kafka.AuditDispatcherTracker;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * Spring component that manages the lifecycle of HDFS dispatcher threads.
+ * Manager that handles the lifecycle of HDFS dispatcher threads.
+ * - Initializes the dispatcher tracker
+ * - Creates HDFS dispatcher instances
+ * - Starts dispatcher threads
+ * - Handles graceful shutdown
+ */
+public class HdfsDispatcherManager {
+ private static final Logger LOG = LoggerFactory.getLogger(HdfsDispatcherManager.class);
+ private static final String CONFIG_DISPATCHER_TYPE = AuditServerConstants.PROP_DISPATCHER_TYPE;
+ private static final String DISPATCHER_TYPE_HDFS = "hdfs";
+
+ private final AuditDispatcherTracker tracker = AuditDispatcherTracker.getInstance();
+ private AuditDispatcher dispatcher;
+ private Thread dispatcherThread;
+
+ public void init(Properties props) {
+ LOG.info("==> HdfsDispatcherManager.init()");
+
+ String dispatcherType = System.getProperty(CONFIG_DISPATCHER_TYPE);
+ if (dispatcherType != null && !dispatcherType.equalsIgnoreCase(DISPATCHER_TYPE_HDFS)) {
+ LOG.info("Skipping HdfsDispatcherManager initialization since dispatcher type is {}", dispatcherType);
+ return;
+ }
+
+ try {
+ if (props == null) {
+ LOG.error("Configuration properties are null");
+ throw new RuntimeException("Failed to load configuration");
+ }
+
+ boolean isEnabled = MiscUtil.getBooleanProperty(props, "xasecure.audit.destination.hdfs", false);
+ if (!isEnabled) {
+ String clsName = MiscUtil.getStringProperty(props, AuditServerConstants.PROP_DISPATCHER_CLASS);
+ if (clsName != null && clsName.contains("org.apache.ranger.audit.dispatcher.kafka.AuditHDFSDispatcher")) {
+ isEnabled = true;
+ props.setProperty("xasecure.audit.destination.hdfs", "true");
+ }
+ }
+
+ if (!isEnabled) {
+ LOG.warn("HDFS destination is disabled (xasecure.audit.destination.hdfs=false). No dispatchers will be created.");
+ return;
+ }
+
+ // Initialize and register HDFS Dispatcher
+ initializeDispatcher(props, AuditServerConstants.PROP_DISPATCHER_PREFIX);
+
+ if (dispatcher == null) {
+ LOG.warn("No dispatcher was created! Verify that xasecure.audit.destination.hdfs=true and classes are configured correctly.");
+ } else {
+ LOG.info("Created HDFS dispatcher");
+
+ // Register shutdown hook
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ LOG.info("JVM shutdown detected, stopping HdfsDispatcherManager...");
+ shutdown();
+ }, "HdfsDispatcherManager-ShutdownHook"));
+
+ // Start dispatcher thread
+ startDispatcher();
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to initialize HdfsDispatcherManager", e);
+ throw new RuntimeException("Failed to initialize HdfsDispatcherManager", e);
+ }
+ LOG.info("<== HdfsDispatcherManager.init()");
+ }
+
+ private void initializeDispatcher(Properties props, String propPrefix) {
+ LOG.info("==> HdfsDispatcherManager.initializeDispatcher()");
+
+ String clsStr = MiscUtil.getStringProperty(props, AuditServerConstants.PROP_DISPATCHER_CLASS, "org.apache.ranger.audit.dispatcher.kafka.AuditHDFSDispatcher");
+ String[] hdfsDispatcherClasses = clsStr.split(",");
+
+ LOG.info("Initializing {} dispatcher class(es)", hdfsDispatcherClasses.length);
+
+ String hdfsDispatcherClassName = clsStr.split(",")[0].trim();
+ if (hdfsDispatcherClassName.isEmpty()) {
+ LOG.error("Dispatcher class name is empty");
+ return;
+ }
+
+ try {
+ Class<?> dispatcherClass = Class.forName(hdfsDispatcherClassName);
+ dispatcher = (AuditDispatcher) dispatcherClass
+ .getConstructor(Properties.class, String.class)
+ .newInstance(props, propPrefix);
+ tracker.addActiveDispatcher(DISPATCHER_TYPE_HDFS, dispatcher);
+ LOG.info("Successfully initialized dispatcher class: {}", dispatcherClass.getName());
+ } catch (ClassNotFoundException e) {
+ LOG.error("Dispatcher class not found: {}. Ensure the class is on the classpath.", hdfsDispatcherClassName, e);
+ } catch (Exception e) {
+ LOG.error("Error initializing dispatcher class: {}", hdfsDispatcherClassName, e);
+ }
+
+ LOG.info("<== HdfsDispatcherManager.initializeDispatcher()");
+ }
+
+ /**
+ * Start dispatcher thread
+ */
+ private void startDispatcher() {
+ LOG.info("==> HdfsDispatcherManager.startDispatcher()");
+
+ logDispatcherStartup();
+ try {
+ String dispatcherName = dispatcher.getClass().getSimpleName();
+ Thread dispatcherThread = new Thread(dispatcher, dispatcherName);
+ dispatcherThread.setDaemon(true);
+ dispatcherThread.start();
+ LOG.info("Started {} thread [Thread-ID: {}, Thread-Name: '{}']", dispatcherName, dispatcherThread.getId(), dispatcherThread.getName());
+ } catch (Exception e) {
+ LOG.error("Error starting dispatcher: {}", dispatcher.getClass().getSimpleName(), e);
+ }
+
+ LOG.info("<== HdfsDispatcherManager.startDispatcher()");
+ }
+
+ private void logDispatcherStartup() {
+ LOG.info("################## HDFS DISPATCHER SERVICE STARTUP ######################");
+
+ if (dispatcher == null) {
+ LOG.warn("WARNING: No HDFS dispatchers are enabled!");
+ LOG.warn("Verify: xasecure.audit.destination.hdfs=true in configuration");
+ } else {
+ AuditServerLogFormatter.LogBuilder builder = AuditServerLogFormatter.builder("HDFS Dispatcher Status");
+ String dispatcherType = dispatcher.getClass().getSimpleName();
+ builder.add(dispatcherType, "ENABLED");
+ builder.add("Topic", dispatcher.getTopicName());
+ builder.logInfo(LOG);
+ LOG.info("Starting HDFS dispatcher thread...");
+ }
+
+ LOG.info("########################################################################");
+ }
+
+ public void shutdown() {
+ LOG.info("==> HdfsDispatcherManager.shutdown()");
+
+ // Shutdown dispatcher
+ if (dispatcher != null) {
+ try {
+ LOG.info("Shutting down dispatcher: {}", dispatcher.getClass().getSimpleName());
+ dispatcher.shutdown();
+ LOG.info("Dispatcher shutdown completed: {}", dispatcher.getClass().getSimpleName());
+ } catch (Exception e) {
+ LOG.error("Error shutting down dispatcher: {}", dispatcher.getClass().getSimpleName(), e);
+ }
+ }
+
+ // Wait for thread to terminate
+ if (dispatcherThread != null && dispatcherThread.isAlive()) {
+ try {
+ LOG.info("Waiting for thread to terminate: {}", dispatcherThread.getName());
+ dispatcherThread.join(10000); // Wait up to 10 seconds
+ if (dispatcherThread.isAlive()) {
+ LOG.warn("Thread did not terminate within 10 seconds: {}", dispatcherThread.getName());
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for thread to terminate: {}", dispatcherThread.getName(), e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ dispatcher = null;
+ dispatcherThread = null;
+ tracker.clearActiveDispatcher(DISPATCHER_TYPE_HDFS);
+
+ LOG.info("<== HdfsDispatcherManager.shutdown() - HDFS dispatcher stopped");
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditHDFSDispatcher.java b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditHDFSDispatcher.java
new file mode 100644
index 000000000..441838dfa
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditHDFSDispatcher.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.ranger.audit.provider.AuditProviderFactory;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.server.AuditServerConstants;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+public class AuditHDFSDispatcher extends AuditDispatcherBase {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditHDFSDispatcher.class);
+
+ private static final String CONFIG_CORE_SITE = "core-site.xml";
+ private static final String CONFIG_HDFS_SITE = "hdfs-site.xml";
+ private static final String DEFAULT_RANGER_AUDIT_HDFS_DISPATCHER_GROUP = "ranger_audit_hdfs_dispatcher_group";
+ private static final String PROP_HDFS_DEST_PREFIX = "hdfs";
+
+ private final AuditRouterHDFS auditRouterHDFS;
+
+ public AuditHDFSDispatcher(Properties props, String propPrefix) throws Exception {
+ super(props, propPrefix, DEFAULT_RANGER_AUDIT_HDFS_DISPATCHER_GROUP);
+
+ auditRouterHDFS = new AuditRouterHDFS();
+
+ init(props, propPrefix);
+ }
+
+ @Override
+ protected String getDispatcherName() {
+ return PROP_HDFS_DEST_PREFIX.toUpperCase();
+ }
+
+ @Override
+ protected DispatcherWorker createDispatcherWorker(String workerId, List<Integer> assignedPartitions) {
+ return new HDFSDispatcherWorker(workerId, assignedPartitions);
+ }
+
+ @Override
+ protected void shutdownDestination() {
+ if (auditRouterHDFS != null) {
+ try {
+ auditRouterHDFS.shutdown();
+ } catch (Exception e) {
+ LOG.error("Error shutting down HDFS destination handler", e);
+ }
+ }
+ }
+
+ private void init(Properties props, String propPrefix) throws Exception {
+ LOG.info("==> AuditHDFSDispatcher.init(): AuditHDFSDispatcher initializing with appId-based threading and offset management");
+
+ // Initialize Ranger UGI for HDFS operations if Kerberos is enabled
+ initializeRangerUGI(props, propPrefix);
+
+ // Add Hadoop configuration properties from HdfsDispatcherConfig to props
+ addHadoopConfigToProps(props);
+
+ // Initialize dispatcher configuration
+ initDispatcherConfig(props, propPrefix);
+
+ // Initialize destination handler for message processing
+ auditRouterHDFS.init(props, AuditProviderFactory.AUDIT_DEST_BASE + "." + PROP_HDFS_DEST_PREFIX);
+
+ LOG.info("<== AuditHDFSDispatcher.init(): AuditHDFSDispatcher initialized successfully");
+ }
+
+ private void processMessage(String message, String partitionKey) throws Exception {
+ auditRouterHDFS.routeAuditMessage(message, partitionKey);
+ }
+
+ private void initializeRangerUGI(Properties props, String propPrefix) throws Exception {
+ LOG.info("==> AuditHDFSDispatcher.initializeRangerUGI()");
+
+ try {
+ Configuration coreSite = getCoreSiteConfiguration();
+ String authType = coreSite.get(AuditServerConstants.PROP_HADOOP_AUTHENTICATION_TYPE, "simple");
+
+ if (!AuditServerConstants.PROP_HADOOP_AUTH_TYPE_KERBEROS.equalsIgnoreCase(authType)) {
+ LOG.info("Hadoop authentication is not Kerberos ({}), skipping Ranger UGI initialization", authType);
+ return;
+ }
+
+ String principal = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_AUDIT_SERVICE_PRINCIPAL);
+ String keytab = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_AUDIT_SERVICE_KEYTAB);
+ String hostName = MiscUtil.getStringProperty(props, propPrefix + "." + "host");
+
+ if (principal == null || keytab == null) {
+ LOG.warn("Kerberos is enabled but principal or keytab is null! principal={}, keytab={}", principal, keytab);
+ String msg = String.format("Kerberos is enabled but principal or keytab is null! principal=%s, keytab=%s", principal, keytab);
+ throw new Exception(msg);
+ }
+
+ if (principal.contains("_HOST")) {
+ try {
+ principal = SecureClientLogin.getPrincipal(principal, hostName);
+ LOG.info("Resolved principal from [{}] using hostname [{}]", principal, hostName);
+ } catch (IOException e) {
+ LOG.error("Failed to resolve principal pattern [{}] with hostname [{}]", principal, hostName, e);
+ throw e;
+ }
+ }
+
+ // Set Hadoop security configuration from core-site.xml
+ UserGroupInformation.setConfiguration(coreSite);
+
+ LOG.info("Initializing Ranger AuditServer UGI for HDFS writes: principal={}, keytab={}", principal, keytab);
+
+ UserGroupInformation rangerAuditServerUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab);
+
+ MiscUtil.setUGILoginUser(rangerAuditServerUGI, null);
+
+ LOG.info("<== AuditHDFSDispatcher.initializeRangerUGI(): Ranger UGI initialized successfully: user={}, auth={}, hasKerberos={}",
+ rangerAuditServerUGI.getUserName(), rangerAuditServerUGI.getAuthenticationMethod(), rangerAuditServerUGI.hasKerberosCredentials());
+ } catch (IOException e) {
+ LOG.error("Failed to initialize Ranger AuditServer UGI for HDFS writes", e);
+ throw e;
+ }
+ }
+
+ /**
+ * Add Hadoop configuration properties from core-site.xml and hdfs-site.xml to props.
+ */
+ private void addHadoopConfigToProps(Properties props) {
+ LOG.info("==> AuditHDFSDispatcher.addHadoopConfigToProps()");
+
+ try {
+ String configPrefix = "xasecure.audit.destination.hdfs.config.";
+ Properties hadoopProps = getHadoopPropertiesWithPrefix(configPrefix);
+ props.putAll(hadoopProps);
+
+ LOG.info("<== AuditHDFSDispatcher.addHadoopConfigToProps(): Added {} Hadoop configuration properties", hadoopProps.size());
+ } catch (Exception e) {
+ LOG.error("Failed to add Hadoop configuration properties to props", e);
+ }
+ }
+
+ private void initDispatcherConfig(Properties props, String propPrefix) {
+ LOG.info("==> AuditHDFSDispatcher.initDispatcherConfig()");
+
+ // Get dispatcher thread count
+ this.dispatcherThreadCount = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_THREAD_COUNT, 1);
+ LOG.info("HDFS dispatcher thread count: {}", dispatcherThreadCount);
+
+ // Initialize offset management configuration
+ initializeOffsetManagement(props, propPrefix);
+
+ LOG.info("<== AuditHDFSDispatcher.initDispatcherConfig()");
+ }
+
+ /**
+ * Get Hadoop configuration properties (from core-site.xml and hdfs-site.xml) with a specific prefix.
+ * This is need for the HdfsAuditDestination and its parent classes for routing the audits to hdfs location.
+ * @param prefix The prefix to add to each property name ("xasecure.audit.destination.hdfs.config.")
+ * @return Properties from core-site.xml and hdfs-site.xml with the specified prefix
+ */
+ private Properties getHadoopPropertiesWithPrefix(String prefix) {
+ LOG.debug("==> AuditHDFSDispatcher.getHadoopPropertiesWithPrefix(prefix={})", prefix);
+
+ Properties prefixedProps = new java.util.Properties();
+ int propsAdded = 0;
+
+ try {
+ // Load core-site.xml separately to get pure Hadoop security properties
+ Configuration coreSite = new Configuration(false);
+ coreSite.addResource(CONFIG_CORE_SITE);
+
+ for (java.util.Map.Entry<String, String> entry : coreSite) {
+ String propName = entry.getKey();
+ String propValue = entry.getValue();
+
+ if (propValue != null && !propValue.trim().isEmpty()) {
+ prefixedProps.setProperty(prefix + propName, propValue);
+ LOG.trace("Added from core-site.xml: {} = {}", propName, propValue);
+ propsAdded++;
+ }
+ }
+
+ // Load hdfs-site.xml separately to get pure HDFS client properties
+ Configuration hdfsSite = new Configuration(false);
+ hdfsSite.addResource(CONFIG_HDFS_SITE);
+
+ for (java.util.Map.Entry<String, String> entry : hdfsSite) {
+ String propName = entry.getKey();
+ String propValue = entry.getValue();
+
+ if (propValue != null && !propValue.trim().isEmpty()) {
+ prefixedProps.setProperty(prefix + propName, propValue);
+ LOG.trace("Added from hdfs-site.xml: {} = {}", propName, propValue);
+ propsAdded++;
+ }
+ }
+
+ LOG.debug("<== AuditHDFSDispatcher.getHadoopPropertiesWithPrefix(): Added {} Hadoop properties with prefix '{}'", propsAdded, prefix);
+ } catch (Exception e) {
+ LOG.error("Failed to load Hadoop properties from core-site.xml and hdfs-site.xml", e);
+ }
+
+ return prefixedProps;
+ }
+
+ /**
+ * Get core-site.xml Configuration for UGI initialization.
+ * @return Configuration loaded from core-site.xml
+ */
+ private Configuration getCoreSiteConfiguration() {
+ LOG.debug("==> AuditHDFSDispatcher.getCoreSiteConfiguration()");
+
+ Configuration coreSite = new Configuration(false);
+ coreSite.addResource(CONFIG_CORE_SITE);
+
+ LOG.debug("<== AuditHDFSDispatcher.getCoreSiteConfiguration(): authentication={}", coreSite.get("hadoop.security.authentication"));
+
+ return coreSite;
+ }
+
+ private void initializeOffsetManagement(Properties props, String propPrefix) {
+ LOG.info("==> AuditHDFSDispatcher.initializeOffsetManagement()");
+
+ // Get offset commit strategy
+ this.offsetCommitStrategy = MiscUtil.getStringProperty(props,
+ propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_OFFSET_COMMIT_STRATEGY,
+ AuditServerConstants.DEFAULT_OFFSET_COMMIT_STRATEGY);
+
+ // Get offset commit interval (only used for manual strategy)
+ this.offsetCommitInterval = MiscUtil.getLongProperty(props,
+ propPrefix + "." + AuditServerConstants.PROP_DISPATCHER_OFFSET_COMMIT_INTERVAL,
+ AuditServerConstants.DEFAULT_OFFSET_COMMIT_INTERVAL_MS);
+
+ AuditServerLogFormatter.builder("HDFS Dispatcher Offset Management Configuration")
+ .add("Commit Strategy", offsetCommitStrategy)
+ .add("Commit Interval (ms)", offsetCommitInterval + " (used in manual mode only)")
+ .logInfo(LOG);
+
+ LOG.info("<== AuditHDFSDispatcher.initializeOffsetManagement()");
+ }
+
+ private class HDFSDispatcherWorker extends DispatcherWorker {
+ public HDFSDispatcherWorker(String workerId, List<Integer> assignedPartitions) {
+ super(workerId, assignedPartitions);
+ }
+
+ @Override
+ protected void processRecordBatch(ConsumerRecords<String, String> records) {
+ Set<TopicPartition> failedPartitions = new HashSet<>();
+
+ for (ConsumerRecord<String, String> record : records) {
+ TopicPartition partition = new TopicPartition(record.topic(), record.partition());
+
+ if (failedPartitions.contains(partition)) {
+ continue; // Skip remaining records for this partition in the current batch
+ }
+
+ try {
+ LOG.debug("HDFS worker '{}' consumed: partition={}, key={}, offset={}",
+ workerId, record.partition(), record.key(), record.offset());
+
+ // Process the message using the destination handler
+ // The partition key (record.key()) contains the appId for HDFS path routing
+ processMessage(record.value(), record.key());
+
+ // Track offset for manual commit strategies
+ pendingOffsets.put(partition, new OffsetAndMetadata(record.offset() + 1));
+ messagesProcessedSinceLastCommit.incrementAndGet();
+ } catch (Exception e) {
+ LOG.error("Error processing message in HDFS worker '{}': partition={}, key={}, offset={}",
+ workerId, record.partition(), record.key(), record.offset(), e);
+
+ // On error, track the offset of the failed message so we retry it
+ pendingOffsets.put(partition, new OffsetAndMetadata(record.offset()));
+
+ // Seek the consumer back to the failed offset so it is re-fetched in the next poll
+ try {
+ workerDispatcher.seek(partition, record.offset());
+ } catch (Exception seekEx) {
+ LOG.error("Failed to seek to offset {} for partition {} after processing error", record.offset(), partition, seekEx);
+ }
+
+ // Add sleep to prevent frequent polling when HDFS is completely down
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+
+ // Mark this partition as failed so we don't process subsequent records for it in this batch
+ failedPartitions.add(partition);
+ }
+ }
+ }
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditRouterHDFS.java b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditRouterHDFS.java
new file mode 100644
index 000000000..22728393d
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/java/org/apache/ranger/audit/dispatcher/kafka/AuditRouterHDFS.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.dispatcher.kafka;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ranger.audit.destination.HDFSAuditDestination;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.utils.AuditServerLogFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Router class that routes audit messages to different HDFSAuditDestination instances
+ * based on the app_id.
+ * Each app_id gets its own HDFSAuditDestination instance for separate path configuration.
+ * Writes are synchronous to ensure Kafka offset commits only happen after successful HDFS writes,
+ * enabling automatic recovery via Kafka redelivery on failures.
+ *
+ * This router writes audits to HDFS as the rangerauditserver user. The audit folder in HDFS
+ * should be configured with appropriate permissions to allow rangerauditserver to write audits
+ * either by Ranger policy or by HDFS acl.
+ */
+public class AuditRouterHDFS {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditRouterHDFS.class);
+
+ private final Map<String, HDFSAuditDestination> destinationMap = new ConcurrentHashMap<>();
+ private final ObjectMapper jsonMapper = new ObjectMapper();
+ private Properties props;
+ private String hdfsPropPrefix;
+
+ public AuditRouterHDFS() {
+ }
+
+ public void init(Properties props, String hdfsPropPrefix) {
+ LOG.info("==> AuditRouterHDFS.init()");
+
+ this.props = new Properties();
+ this.props.putAll(props);
+ this.hdfsPropPrefix = hdfsPropPrefix;
+
+ LOG.info("<== AuditRouterHDFS.init()");
+ }
+
+ /**
+ * Routes audit message to appropriate HDFSAuditDestination based on app_id
+ * @param message JSON audit message
+ * @param partitionKey The partition key from Kafka (used as app_id)
+ */
+ public void routeAuditMessage(String message, String partitionKey) throws Exception {
+ LOG.debug("==> AuditRouterHDFS:routeAuditMessage(): Message => {}, partitionKey => {}", message, partitionKey);
+
+ try {
+ String appId = extractAppId(message, partitionKey);
+ String serviceType = extractServiceType(message);
+ String agentHostname = extractAgentHostname(message);
+
+ LOG.debug("Routing audit message for app_id: {}, serviceType: {}, agentHostname: {}", appId, serviceType, agentHostname);
+
+ // Get or create HDFSAuditDestination for the appId
+ if (appId != null) {
+ HDFSAuditDestination hdfsAuditDestination = getHdfsAuditDestination(appId, serviceType, agentHostname);
+
+ boolean success = hdfsAuditDestination.logJSON(Collections.singletonList(message));
+
+ if (!success) {
+ throw new Exception("Failed to write audit to HDFS for app_id: " + appId);
+ }
+
+ LOG.debug("Successfully wrote audit for app_id: {}", appId);
+ } else {
+ LOG.warn("Unable to extract app_id from message, skipping audit write");
+ }
+ } catch (Exception e) {
+ String errorMessage = "Error routing audit message to HDFS";
+ LOG.error("AuditRouterHDFS:routeAuditMessage(): Error routing audit message: {}", message, e);
+ throw new Exception(errorMessage, e);
+ }
+
+ LOG.debug("<== AuditRouterHDFS:routeAuditMessage()");
+ }
+
+ /**
+ * Extract app_id from audit message. First tries partition key, then falls back to parsing JSON.
+ */
+ private String extractAppId(String message, String partitionKey) {
+ // First try to use partition key as app_id if available
+ if (partitionKey != null && !partitionKey.trim().isEmpty()) {
+ return partitionKey;
+ }
+ // Fall back to extracting from JSON message
+ try {
+ JsonNode rootNode = jsonMapper.readTree(message);
+ JsonNode agentIdNode = rootNode.get("agent");
+ if (agentIdNode != null) {
+ return agentIdNode.asText();
+ }
+ } catch (Exception e) {
+ LOG.debug("AuditRouterHDFS:extractAppId(): Failed to parse JSON message for app_id extraction: {}", e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Extract serviceType from audit message additional_info
+ */
+ private String extractServiceType(String message) {
+ try {
+ JsonNode rootNode = jsonMapper.readTree(message);
+ JsonNode additionalInfoNode = rootNode.get("additional_info");
+ JsonNode additionalInfoJson = jsonMapper.readTree(additionalInfoNode.asText());
+ JsonNode serviceTypeNode = additionalInfoJson.get("serviceType");
+ if (serviceTypeNode != null) {
+ return serviceTypeNode.asText();
+ }
+ } catch (Exception e) {
+ LOG.debug("AuditRouterHDFS:extractServiceType(): Failed to parse JSON message for serviceType extraction: {}", e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Extract agentHostname from audit message
+ */
+ private String extractAgentHostname(String message) {
+ try {
+ JsonNode rootNode = jsonMapper.readTree(message);
+ JsonNode agentHostNode = rootNode.get("agentHost");
+ if (agentHostNode != null) {
+ return agentHostNode.asText();
+ }
+ } catch (Exception e) {
+ LOG.debug("AuditRouterHDFS:extractAgentHostname(): Failed to parse JSON message for agentHostname extraction: {}", e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Get or create HDFSAuditDestination for the given app_id
+ */
+ private synchronized HDFSAuditDestination getHdfsAuditDestination(String appId, String serviceType, String agentHostname) {
+ LOG.debug("==> AuditRouterHDFS:getHdfsAuditDestination() for app_id: {}, serviceType: {} and agentHostname: {}", appId, serviceType, agentHostname);
+ HDFSAuditDestination destination = destinationMap.get(appId);
+ if (destination == null) {
+ try {
+ destination = createHDFSDestination(appId, serviceType, agentHostname);
+ destinationMap.put(appId, destination);
+ } catch (Exception e) {
+ LOG.error("Failed to create HDFSAuditDestination for app_id: {}", appId, e);
+ throw new RuntimeException("AuditRouterHDFS:getOrCreateDestination() : Failed to create destination for app_id: " + appId, e);
+ }
+ }
+
+ LOG.debug("<== AuditRouterHDFS:getHdfsAuditDestination()..got HDFSAuditDestination for app_id: {}", appId);
+
+ return destination;
+ }
+
+ /**
+ * Create and initialize HDFSAuditDestination with app_id specific configuration
+ */
+ private HDFSAuditDestination createHDFSDestination(String appId, String serviceType, String agentHostname) throws Exception {
+ LOG.debug("==> AuditRouterHDFS:createHDFSDestination(): Creating new HDFSAuditDestination for app_id: {}, serviceType: {}, agentHostname: {}", appId, serviceType, agentHostname);
+
+ HDFSAuditDestination destination = new HDFSAuditDestination();
+
+ // Create app_id specific properties
+ Properties appSpecificProps = new Properties();
+ appSpecificProps.putAll(props);
+
+ // Determine file type and writer implementation
+ String fileType = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".batch.filequeue.filetype", "json");
+ String writerImpl = getWriterImplementation(fileType);
+
+ // Configure directory properties
+ String baseDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".dir", "/ranger/audit");
+
+ // Default: {serviceType}/{appId}/%time:yyyyMMdd%/
+ String defaultSubDir = (serviceType != null && !serviceType.isEmpty()) ? serviceType + "/" + appId + "/%time:yyyyMMdd%/" : appId + "/%time:yyyyMMdd%/";
+ String subDir = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".subdir", defaultSubDir);
+
+ // Set file extension based on file type
+ String fileExtension = getFileExtension(fileType);
+ // Get unique instance identifier for filename uniqueness across scaled audit services
+ String instanceId = getUniqueInstanceIdentifier();
+
+ // Use agentHostname from audit message if available, otherwise fall back to local hostname token
+ String hostnameValue = (agentHostname != null && !agentHostname.isEmpty()) ? agentHostname : "%hostname%";
+
+ // Build default filename with agent hostname to properly identify the source system
+ String defaultFileName = appId + "_ranger_audit_" + hostnameValue + "_" + instanceId + fileExtension;
+ String fileNameFormat = MiscUtil.getStringProperty(props, hdfsPropPrefix + ".filename.format", defaultFileName);
+
+ // If filename format contains %hostname% and we have agentHostname, replace it
+ if (agentHostname != null && !agentHostname.isEmpty() && fileNameFormat.contains("%hostname%")) {
+ fileNameFormat = fileNameFormat.replace("%hostname%", agentHostname);
+ }
+
+ // Set the enhanced properties
+ appSpecificProps.setProperty(hdfsPropPrefix + ".dir", baseDir);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".subdir", subDir);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".filename.format", fileNameFormat);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".filewriter.impl", writerImpl);
+ appSpecificProps.setProperty(hdfsPropPrefix + ".batch.filequeue.filetype", fileType);
+ // Preserve other properties
+ preserveFileSystemProperties(appSpecificProps, hdfsPropPrefix);
+
+ // Log configuration
+ AuditServerLogFormatter.builder("Initializing HDFSAuditDestination for app_id: " + appId)
+ .add("Base directory", baseDir)
+ .add("Subdirectory pattern", subDir)
+ .add("Filename format", fileNameFormat)
+ .add("Agent hostname", agentHostname)
+ .add("Instance identifier", instanceId)
+ .add("File type", fileType)
+ .add("Writer implementation", writerImpl)
+ .logInfo(LOG);
+
+ destination.init(appSpecificProps, hdfsPropPrefix);
+ destination.start();
+
+ LOG.debug("<== AuditRouterHDFS:createHDFSDestination(): Created new HDFSDestination {}", destination.getName());
+
+ return destination;
+ }
+
+ private String getWriterImplementation(String fileType) {
+ switch (fileType.toLowerCase()) {
+ case "orc":
+ return "org.apache.ranger.audit.utils.RangerORCAuditWriter";
+ case "json":
+ default:
+ return "org.apache.ranger.audit.utils.RangerJSONAuditWriter";
+ }
+ }
+
+ private String getFileExtension(String fileType) {
+ switch (fileType.toLowerCase()) {
+ case "orc":
+ return ".orc";
+ case "json":
+ default:
+ return ".log";
+ }
+ }
+
+ private void preserveFileSystemProperties(Properties appSpecificProps, String baseDestPrefix) {
+ String fileRolloverSec = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.sec");
+ if (fileRolloverSec != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.sec", fileRolloverSec);
+ }
+
+ String rolloverPeriod = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.period");
+ if (rolloverPeriod != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.period", rolloverPeriod);
+ }
+
+ String appendEnabled = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.append.enabled");
+ if (appendEnabled != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.append.enabled", appendEnabled);
+ }
+
+ String periodicRolloverEnabled = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.enable.periodic.rollover");
+ if (periodicRolloverEnabled != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.enable.periodic.rollover", periodicRolloverEnabled);
+ }
+
+ String periodicRolloverCheckTime = MiscUtil.getStringProperty(props, baseDestPrefix + ".file.rollover.periodic.rollover.check.sec");
+ if (periodicRolloverCheckTime != null) {
+ appSpecificProps.setProperty(baseDestPrefix + ".file.rollover.periodic.rollover.check.sec", periodicRolloverCheckTime);
+ }
+ }
+
+ private String getUniqueInstanceIdentifier() {
+ String jvmInstanceId = MiscUtil.getJvmInstanceId();
+ LOG.info("Using JVM instance ID as unique identifier: {}", jvmInstanceId);
+ return jvmInstanceId;
+ }
+
+ public void shutdown() {
+ LOG.info("==> AuditRouterHDFS.shutdown()");
+
+ // Stop all destinations
+ for (Map.Entry<String, HDFSAuditDestination> entry : destinationMap.entrySet()) {
+ String appId = entry.getKey();
+ HDFSAuditDestination destination = entry.getValue();
+
+ LOG.info("Stopping HDFSAuditDestination for app_id: {}", appId);
+ try {
+ destination.stop();
+ } catch (Exception e) {
+ LOG.error("Error stopping HDFSAuditDestination for app_id: {}", appId, e);
+ }
+ }
+
+ destinationMap.clear();
+
+ LOG.info("<== AuditRouterHDFS.shutdown()");
+ }
+}
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/core-site.xml b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/core-site.xml
new file mode 100644
index 000000000..30a1f5e83
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/core-site.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+ <!--
+ Hadoop security settings for Ranger Audit HDFS Dispatcher Service
+ This service authenticates as the rangerauditserver principal
+ and writes all audits to HDFS as that user.
+ -->
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ <description>Authentication method for Hadoop services: simple or kerberos</description>
+ </property>
+
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[1:$1@$0](rangerauditserver@EXAMPLE\.COM)s/.*/rangerauditserver/
+ RULE:[1:$1/$2@$0](rangerauditserver/ranger-audit-dispatcher-hdfs.rangernw@EXAMPLE\.COM)s/.*/rangerauditserver/
+ DEFAULT
+ </value>
+ <description>
+ Kerberos principal to local user name mapping rules for Ranger Audit HDFS Dispatcher.
+ Note: rangerauditserver/hostname@REALM and rangerauditserver@REALM both map to 'ranger' OS user
+ </description>
+ </property>
+</configuration>
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/hdfs-site.xml b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/hdfs-site.xml
new file mode 100644
index 000000000..da3031812
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/hdfs-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+ <!--
+ HDFS Configuration for Ranger Audit HDFS Dispatcher Service
+ These settings are required for Kerberos-authenticated HDFS writes.
+ -->
+
+ <!-- NameNode Kerberos Principal -->
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>nn/_HOST@EXAMPLE.COM</value>
+ <description>
+ HDFS NameNode Kerberos principal. Required for Kerberos RPC authentication to NameNode.
+ Common formats:
+ - hdfs/_HOST@REALM
+ - nn/_HOST@REALM
+ - hdfs/namenode.fqdn@REALM (fully qualified, for specific NameNode)
+ </description>
+ </property>
+
+ <!-- Optional: Additional HDFS Client Settings -->
+ <property>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>false</value>
+ <description>
+ Whether clients should use the datanode hostname when connecting.
+ Set to true if using hostname-based Kerberos principals.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.failover.proxy.provider</name>
+ <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+ <description>
+ Failover proxy provider for HA NameNode clusters.
+ Adjust if your cluster uses a different HA configuration.
+ </description>
+ </property>
+</configuration>
diff --git a/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/logback.xml b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/logback.xml
new file mode 100644
index 000000000..5f55b2754
--- /dev/null
+++ b/audit-server/audit-dispatcher/dispatcher-hdfs/src/main/resources/conf/logback.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <!-- System properties for log directory and file -->
+ <property name="LOG_DIR" value="${audit.dispatcher.hdfs.log.dir:-./logs}" />
+ <property name="LOG_FILE" value="${audit.dispatcher.hdfs.log.file:-dispatcher-hdfs.log}" />
+
+ <!-- Console Appender -->
+ <appender name="LogToConsole" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Rolling File Appender -->
+ <appender name="LogToRollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${LOG_DIR}/${LOG_FILE}</file>
+ <encoder>
+ <pattern>%d %p %c{1} [%t] %m%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${LOG_DIR}/${LOG_FILE}-%d{MM-dd-yyyy}-%i.log.gz</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>200MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <maxHistory>10</maxHistory>
+ </rollingPolicy>
+ </appender>
+
+ <!-- Ranger Audit Logger -->
+ <logger name="org.apache.ranger.audit" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Spring Logger -->
+ <logger name="org.springframework" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Kafka Logger (reduce verbosity) -->
+ <logger name="org.apache.kafka" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- Hadoop Logger (reduce verbosity) -->
+ <logger name="org.apache.hadoop" level="INFO" additivity="false">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </logger>
+
+ <!-- WADL generator (suppress noisy output) -->
+ <logger name="com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator" level="OFF" />
+
+ <!-- Root Logger -->
+ <root level="INFO">
+ <appender-ref ref="LogToRollingFile" />
+ <appender-ref ref="LogToConsole" />
+ </root>
... 10459 lines suppressed ...