You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2023/11/21 18:52:59 UTC
(ranger) 01/02: RANGER-4282: updated audit logs to capture datasets and projects
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch RANGER-3923
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 9e05af16b7a06cb7970ce42b215fe3ad2cd0ba05
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Mon Nov 20 13:34:22 2023 -0800
RANGER-4282: updated audit logs to capture datasets and projects
---
.../destination/ElasticSearchAuditDestination.java | 2 ++
.../audit/destination/SolrAuditDestination.java | 2 ++
.../apache/ranger/audit/model/AuthzAuditEvent.java | 37 +++++++++++++++++-----
.../audit/provider/solr/SolrAuditProvider.java | 2 ++
.../plugin/audit/RangerDefaultAuditHandler.java | 17 ++++++++++
.../config/solr-ranger_audits/managed-schema | 4 ++-
.../conf/ranger_es_schema.json | 8 ++++-
.../solr_for_audit_setup/conf/managed-schema | 4 ++-
.../cloudwatch/CloudWatchAccessAuditsService.java | 10 ++++++
.../ElasticSearchAccessAuditsService.java | 8 +++++
.../ranger/solr/SolrAccessAuditsService.java | 8 +++++
.../java/org/apache/ranger/view/VXAccessAudit.java | 27 ++++++++++++++--
12 files changed, 116 insertions(+), 13 deletions(-)
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
index 1b17a934b..c06002d35 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
@@ -334,6 +334,8 @@ public class ElasticSearchAuditDestination extends AuditDestination {
doc.put("event_count", auditEvent.getEventCount());
doc.put("event_dur_ms", auditEvent.getEventDurationMS());
doc.put("tags", auditEvent.getTags());
+ doc.put("datasets", auditEvent.getDatasets());
+ doc.put("projects", auditEvent.getProjects());
doc.put("cluster", auditEvent.getClusterName());
doc.put("zoneName", auditEvent.getZoneName());
doc.put("agentHost", auditEvent.getAgentHostname());
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index f2e96bf9b..6fe92498a 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -294,6 +294,8 @@ public class SolrAuditDestination extends AuditDestination {
doc.setField("event_count", auditEvent.getEventCount());
doc.setField("event_dur_ms", auditEvent.getEventDurationMS());
doc.setField("tags", auditEvent.getTags());
+ doc.addField("datasets", auditEvent.getDatasets());
+ doc.addField("projects", auditEvent.getProjects());
doc.setField("cluster", auditEvent.getClusterName());
doc.setField("zoneName", auditEvent.getZoneName());
doc.setField("agentHost", auditEvent.getAgentHostname());
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
index e20d1a786..2e32fb579 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
@@ -110,6 +110,12 @@ public class AuthzAuditEvent extends AuditEventBase {
@SerializedName("tags")
protected Set<String> tags = new HashSet<>();
+ @SerializedName("datasets")
+ protected Set<String> datasets = null;
+
+ @SerializedName("projects")
+ protected Set<String> projects = null;
+
@SerializedName("additional_info")
protected String additionalInfo;
@@ -472,18 +478,34 @@ public class AuthzAuditEvent extends AuditEventBase {
return eventDurationMS;
}
- public Set<String> getTags() {
- return tags;
- }
-
public void setEventDurationMS(long frequencyDurationMS) {
this.eventDurationMS = frequencyDurationMS;
}
+ public Set<String> getTags() {
+ return tags;
+ }
+
public void setTags(Set<String> tags) {
this.tags = tags;
}
+ public Set<String> getDatasets() {
+ return datasets;
+ }
+
+ public void setDatasets(Set<String> datasets) {
+ this.datasets = datasets;
+ }
+
+ public Set<String> getProjects() {
+ return projects;
+ }
+
+ public void setProjects(Set<String> projects) {
+ this.projects = projects;
+ }
+
public String getClusterName() {
return clusterName;
}
@@ -561,10 +583,9 @@ public class AuthzAuditEvent extends AuditEventBase {
.append(FIELD_SEPARATOR).append("event_count=")
.append(eventCount).append(FIELD_SEPARATOR)
.append("event_dur_ms=").append(eventDurationMS)
- .append(FIELD_SEPARATOR)
- .append("tags=").append("[")
- .append(StringUtils.join(tags, ", "))
- .append("]")
+ .append(FIELD_SEPARATOR).append("tags=").append("[").append(StringUtils.join(tags, ", ")).append("]")
+ .append(FIELD_SEPARATOR).append("datasets=").append("[").append(datasets != null ? StringUtils.join(datasets, ", ") : "").append("]")
+ .append(FIELD_SEPARATOR).append("projects=").append("[").append(projects != null ? StringUtils.join(projects, ", ") : "").append("]")
.append(FIELD_SEPARATOR).append("clusterName=").append(clusterName)
.append(FIELD_SEPARATOR).append("zoneName=").append(zoneName)
.append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion)
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
index 9cda3f8f3..691cef002 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
@@ -290,6 +290,8 @@ public class SolrAuditProvider extends AuditDestination {
doc.addField("action", auditEvent.getAction());
doc.addField("evtTime", auditEvent.getEventTime());
doc.addField("tags", auditEvent.getTags());
+ doc.addField("datasets", auditEvent.getDatasets());
+ doc.addField("projects", auditEvent.getProjects());
doc.addField("cluster", auditEvent.getClusterName());
doc.addField("zone", auditEvent.getZoneName());
doc.addField("agentHost", auditEvent.getAgentHostname());
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
index c99465d7a..8efea1fba 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
@@ -33,6 +33,7 @@ import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants;
import org.apache.ranger.authorization.utils.JsonUtils;
import org.apache.ranger.plugin.contextenricher.RangerTagForEval;
import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyengine.gds.GdsAccessResult;
import org.apache.ranger.plugin.service.RangerBasePlugin;
import org.apache.ranger.plugin.util.JsonUtilsV2;
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
@@ -132,10 +133,14 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor {
ret.setClientType(request.getClientType());
ret.setSessionId(request.getSessionId());
ret.setAclEnforcer(moduleName);
+
Set<String> tags = getTags(request);
if (tags != null) {
ret.setTags(tags);
}
+
+ ret.setDatasets(getDatasets(request));
+ ret.setProjects(getProjects(request));
ret.setAdditionalInfo(getAdditionalInfo(request));
ret.setClusterName(request.getClusterName());
ret.setZoneName(result.getZoneName());
@@ -263,6 +268,18 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor {
return ret;
}
+ public final Set<String> getDatasets(RangerAccessRequest request) {
+ GdsAccessResult gdsResult = RangerAccessRequestUtil.getGdsResultFromContext(request.getContext());
+
+ return gdsResult != null ? gdsResult.getDatasets() : null;
+ }
+
+ public final Set<String> getProjects(RangerAccessRequest request) {
+ GdsAccessResult gdsResult = RangerAccessRequestUtil.getGdsResultFromContext(request.getContext());
+
+ return gdsResult != null ? gdsResult.getProjects() : null;
+ }
+
public String getAdditionalInfo(RangerAccessRequest request) {
if (StringUtils.isBlank(request.getRemoteIPAddress()) && CollectionUtils.isEmpty(request.getForwardedAddresses())) {
return null;
diff --git a/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema b/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema
index c33f6de06..df53a05df 100644
--- a/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema
+++ b/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema
@@ -92,4 +92,6 @@
<field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>
<field name="zoneName" type="key_lower_case" multiValued="false"/>
<field name="policyVersion" type="tlong" multiValued="false"/>
-</schema>
\ No newline at end of file
+ <field name="datasets" type="key_lower_case" multiValued="true"/>
+ <field name="projects" type="key_lower_case" multiValued="true"/>
+</schema>
diff --git a/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json b/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json
index 801667bce..d90e18b7d 100644
--- a/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json
+++ b/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json
@@ -111,6 +111,12 @@
"tags_str": {
"type": "text"
},
+ "datasets": {
+ "type": "keyword"
+ },
+ "projects": {
+ "type": "keyword"
+ },
"text": {
"type": "text"
},
@@ -121,4 +127,4 @@
"type": "long"
}
}
-}
\ No newline at end of file
+}
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
index c33f6de06..df53a05df 100644
--- a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
+++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
@@ -92,4 +92,6 @@
<field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>
<field name="zoneName" type="key_lower_case" multiValued="false"/>
<field name="policyVersion" type="tlong" multiValued="false"/>
-</schema>
\ No newline at end of file
+ <field name="datasets" type="key_lower_case" multiValued="true"/>
+ <field name="projects" type="key_lower_case" multiValued="true"/>
+</schema>
diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java
index 0b36f6e90..06a21a3c2 100644
--- a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java
@@ -283,6 +283,16 @@ public class CloudWatchAccessAuditsService extends org.apache.ranger.AccessAudit
accessAudit.setTags(value.toString());
}
+ value = auditEvent.getDatasets();
+ if (value != null) {
+ accessAudit.setDatasets(value.toString());
+ }
+
+ value = auditEvent.getProjects();
+ if (value != null) {
+ accessAudit.setProjects(value.toString());
+ }
+
return accessAudit;
}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
index 4c9b049a0..4195567fa 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
@@ -275,6 +275,14 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu
if (value != null) {
accessAudit.setTags(value.toString());
}
+ value = source.get("datasets");
+ if (value != null) {
+ accessAudit.setDatasets(value.toString());
+ }
+ value = source.get("projects");
+ if (value != null) {
+ accessAudit.setProjects(value.toString());
+ }
return accessAudit;
}
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
index bb279349a..3485f4dba 100644
--- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
@@ -254,6 +254,14 @@ public class SolrAccessAuditsService extends AccessAuditsService {
if (value != null) {
accessAudit.setTags(value.toString());
}
+ value = doc.getFieldValue("datasets");
+ if (value != null) {
+ accessAudit.setDatasets(value.toString());
+ }
+ value = doc.getFieldValue("projects");
+ if (value != null) {
+ accessAudit.setProjects(value.toString());
+ }
return accessAudit;
}
diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java b/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java
index cce18fafb..9cf06f93c 100644
--- a/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java
+++ b/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java
@@ -137,7 +137,11 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable
protected long eventDuration;
protected String tags;
-
+
+ protected String datasets;
+
+ protected String projects;
+
protected String clusterName;
// Security Zone
@@ -548,7 +552,24 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable
public void setTags(String tags) {
this.tags = tags;
}
-
+
+ public String getDatasets() {
+ return datasets;
+ }
+
+ public void setDatasets(String datasets) {
+ this.datasets = datasets;
+ }
+
+ public String getProjects() {
+ return projects;
+ }
+
+ public void setProjects(String projects) {
+ this.projects = projects;
+ }
+
+
/**
* @return the clusterName
*/
@@ -645,6 +666,8 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable
str += "eventCount={" + eventCount + "}";
str += "eventDuration={" + eventDuration + "}";
str += "tags={" + tags + "}";
+ str += "datasets={" + datasets + "}";
+ str += "projects={" + projects + "}";
str += "clusterName={" + clusterName + "}";
str += "zoneName={" + zoneName + "}";
str += "agentHost={" + agentHost + "}";