You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by al...@apache.org on 2016/03/03 01:06:34 UTC
[10/50] [abbrv] incubator-ranger git commit: RANGER-615 Audit to db:
Truncate all string values of audit record so that writing of audit does not
fail
RANGER-615 Audit to db: Truncate all string values of audit record so that writing of audit does not fail
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/33f84070
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/33f84070
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/33f84070
Branch: refs/heads/HDP-2.3.2-groupid
Commit: 33f840704962d7ed5d5e26c63c8f7247d5faeaf9
Parents: eec5ac4
Author: Alok Lal <al...@apache.org>
Authored: Thu Aug 13 08:48:14 2015 -0700
Committer: Alok Lal <al...@apache.org>
Committed: Fri Aug 21 10:15:11 2015 -0700
----------------------------------------------------------------------
.../audit/destination/DBAuditDestination.java | 5 +-
.../audit/entity/AuthzAuditEventDbObj.java | 143 +++++++++++++++++--
.../ranger/audit/model/AuditEventBase.java | 14 +-
.../ranger/audit/model/AuthzAuditEvent.java | 12 +-
.../ranger/audit/provider/DbAuditProvider.java | 4 +
5 files changed, 144 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
index 3d31c06..376e724 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
@@ -33,6 +33,7 @@ import javax.persistence.EntityTransaction;
import javax.persistence.Persistence;
import org.apache.ranger.audit.dao.DaoManager;
+import org.apache.ranger.audit.entity.AuthzAuditEventDbObj;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.provider.MiscUtil;
@@ -63,9 +64,11 @@ public class DBAuditDestination extends AuditDestination {
public void init(Properties props, String propPrefix) {
logger.info("init() called");
super.init(props, propPrefix);
-
// Initial connect
connect();
+
+ // initialize the database related classes
+ AuthzAuditEventDbObj.init(props);
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java b/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
index 435393e..d52a60a 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
@@ -21,6 +21,7 @@
import java.io.Serializable;
import java.util.Date;
+import java.util.Properties;
import javax.persistence.Column;
import javax.persistence.Entity;
@@ -32,8 +33,11 @@ import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.SequenceGenerator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.model.EnumRepositoryType;
import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.MiscUtil;
/**
* Entity implementation class for Entity: AuthzAuditEventDbObj
@@ -42,8 +46,25 @@ import org.apache.ranger.audit.model.AuthzAuditEvent;
@Entity
@Table(name="xa_access_audit")
public class AuthzAuditEventDbObj implements Serializable {
+
+ private static final Log LOG = LogFactory.getLog(AuthzAuditEventDbObj.class);
+
private static final long serialVersionUID = 1L;
+ static int MaxValueLengthAccessType = 255;
+ static int MaxValueLengthAclEnforcer = 255;
+ static int MaxValueLengthAgentId = 255;
+ static int MaxValueLengthClientIp = 255;
+ static int MaxValueLengthClientType = 255;
+ static int MaxValueLengthRepoName = 255;
+ static int MaxValueLengthResultReason = 255;
+ static int MaxValueLengthSessionId = 255;
+ static int MaxValueLengthRequestUser = 255;
+ static int MaxValueLengthAction = 2000;
+ static int MaxValueLengthRequestData = 4000;
+ static int MaxValueLengthResourcePath = 4000;
+ static int MaxValueLengthResourceType = 255;
+
private long auditId;
private int repositoryType;
private String repositoryName;
@@ -63,6 +84,60 @@ public class AuthzAuditEventDbObj implements Serializable {
private String clientIP;
private String requestData;
+ public static void init(Properties props)
+ {
+ LOG.info("AuthzAuditEventDbObj.init()");
+
+ final String AUDIT_DB_MAX_COLUMN_VALUE = "xasecure.audit.destination.db.max.column.length";
+ MaxValueLengthAccessType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "access_type", MaxValueLengthAccessType);
+ logMaxColumnValue("access_type", MaxValueLengthAccessType);
+
+ MaxValueLengthAclEnforcer = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "acl_enforcer", MaxValueLengthAclEnforcer);
+ logMaxColumnValue("acl_enforcer", MaxValueLengthAclEnforcer);
+
+ MaxValueLengthAction = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "action", MaxValueLengthAction);
+ logMaxColumnValue("action", MaxValueLengthAction);
+
+ MaxValueLengthAgentId = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "agent_id", MaxValueLengthAgentId);
+ logMaxColumnValue("agent_id", MaxValueLengthAgentId);
+
+ MaxValueLengthClientIp = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "client_id", MaxValueLengthClientIp);
+ logMaxColumnValue("client_id", MaxValueLengthClientIp);
+
+ MaxValueLengthClientType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "client_type", MaxValueLengthClientType);
+ logMaxColumnValue("client_type", MaxValueLengthClientType);
+
+ MaxValueLengthRepoName = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "repo_name", MaxValueLengthRepoName);
+ logMaxColumnValue("repo_name", MaxValueLengthRepoName);
+
+ MaxValueLengthResultReason = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "result_reason", MaxValueLengthResultReason);
+ logMaxColumnValue("result_reason", MaxValueLengthResultReason);
+
+ MaxValueLengthSessionId = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "session_id", MaxValueLengthSessionId);
+ logMaxColumnValue("session_id", MaxValueLengthSessionId);
+
+ MaxValueLengthRequestUser = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "request_user", MaxValueLengthRequestUser);
+ logMaxColumnValue("request_user", MaxValueLengthRequestUser);
+
+ MaxValueLengthRequestData = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "request_data", MaxValueLengthRequestData);
+ logMaxColumnValue("request_data", MaxValueLengthRequestData);
+
+ MaxValueLengthResourcePath = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "resource_path", MaxValueLengthResourcePath);
+ logMaxColumnValue("resource_path", MaxValueLengthResourcePath);
+
+ MaxValueLengthResourceType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "resource_type", MaxValueLengthResourceType);
+ logMaxColumnValue("resource_type", MaxValueLengthResourceType);
+ }
+
+ public static void logMaxColumnValue(String columnName, int configuredMaxValueLength) {
+ LOG.info("Setting max column value for column[" + columnName + "] to [" + configuredMaxValueLength + "].");
+ if (configuredMaxValueLength == 0) {
+ LOG.info("Max length of column[" + columnName + "] was 0! Column will NOT be emitted in the audit.");
+ } else if (configuredMaxValueLength < 0) {
+ LOG.info("Max length of column[" + columnName + "] was less than 0! Column value will never be truncated.");
+ }
+ }
+
public AuthzAuditEventDbObj() {
super();
@@ -113,7 +188,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "repo_name")
public String getRepositoryName() {
- return this.repositoryName;
+ return truncate(this.repositoryName, MaxValueLengthRepoName, "repo_name");
}
public void setRepositoryName(String repositoryName) {
@@ -122,7 +197,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "request_user")
public String getUser() {
- return this.user;
+ return truncate(this.user, MaxValueLengthRequestUser, "request_user");
}
public void setUser(String user) {
@@ -141,7 +216,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "access_type")
public String getAccessType() {
- return this.accessType;
+ return truncate(this.accessType, MaxValueLengthAccessType, "access_type");
}
public void setAccessType(String accessType) {
@@ -150,7 +225,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "resource_path")
public String getResourcePath() {
- return this.resourcePath;
+ return truncate(this.resourcePath, MaxValueLengthResourcePath, "resource_path");
}
public void setResourcePath(String resourcePath) {
@@ -159,7 +234,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "resource_type")
public String getResourceType() {
- return this.resourceType;
+ return truncate(this.resourceType, MaxValueLengthResourceType, "resource_type");
}
public void setResourceType(String resourceType) {
@@ -168,7 +243,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "action")
public String getAction() {
- return this.action;
+ return truncate(this.action, MaxValueLengthAction, "action");
}
public void setAction(String action) {
@@ -186,7 +261,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "agent_id")
public String getAgentId() {
- return agentId;
+ return truncate(this.agentId, MaxValueLengthAgentId, "agent_id");
}
public void setAgentId(String agentId) {
@@ -204,7 +279,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "result_reason")
public String getResultReason() {
- return this.resultReason;
+ return truncate(this.resultReason, MaxValueLengthResultReason, "result_reason");
}
public void setResultReason(String resultReason) {
@@ -213,7 +288,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "acl_enforcer")
public String getAclEnforcer() {
- return this.aclEnforcer;
+ return truncate(this.aclEnforcer, MaxValueLengthAclEnforcer, "acl_enforcer");
}
public void setAclEnforcer(String aclEnforcer) {
@@ -222,7 +297,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "session_id")
public String getSessionId() {
- return this.sessionId;
+ return truncate(this.sessionId, MaxValueLengthSessionId, "session_id");
}
public void setSessionId(String sessionId) {
@@ -231,7 +306,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "client_type")
public String getClientType() {
- return this.clientType;
+ return truncate(this.clientType, MaxValueLengthClientType, "client_type");
}
public void setClientType(String clientType) {
@@ -240,7 +315,7 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "client_ip")
public String getClientIP() {
- return this.clientIP;
+ return truncate(this.clientIP, MaxValueLengthClientIp, "client_ip");
}
public void setClientIP(String clientIP) {
@@ -249,10 +324,52 @@ public class AuthzAuditEventDbObj implements Serializable {
@Column(name = "request_data")
public String getRequestData() {
- return this.requestData;
+ return truncate(this.requestData, MaxValueLengthRequestData, "request_data");
}
public void setRequestData(String requestData) {
this.requestData = requestData;
}
+ static final String TruncationMarker = "...";
+ static final int TruncationMarkerLength = TruncationMarker.length();
+
+ protected String truncate(String value, int limit, String columnName) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("==> getTrunctedValue(%s, %d, %s)", value, limit, columnName));
+ }
+
+ String result = value;
+ if (value != null) {
+ if (limit < 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Truncation is suppressed for column[%s]: old value [%s], new value[%s]", columnName, value, result));
+ }
+ } else if (limit == 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Column[%s] is to be excluded from audit: old value [%s], new value[%s]", columnName, value, result));
+ }
+ result = null;
+ } else {
+ if (value.length() > limit) {
+ if (limit <= TruncationMarkerLength) {
+ // NOTE: If value is to be truncated to a size that is less than of equal to the Truncation Marker then we won't put the marker in!!
+ result = value.substring(0, limit);
+ } else {
+ StringBuilder sb = new StringBuilder(value.substring(0, limit - TruncationMarkerLength));
+ sb.append(TruncationMarker);
+ result = sb.toString();
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Truncating value for column[%s] to [%d] characters: old value [%s], new value[%s]", columnName, limit, value, result));
+ }
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("<== getTrunctedValue(%s, %d, %s): %s", value, limit, columnName, result));
+ }
+ return result;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
index 2c6a87f..2a07e94 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
@@ -21,6 +21,8 @@ package org.apache.ranger.audit.model;
import java.util.Date;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.dao.DaoManager;
public abstract class AuditEventBase {
@@ -34,16 +36,4 @@ public abstract class AuditEventBase {
public abstract Date getEventTime ();
public abstract void setEventCount(long eventCount);
public abstract void setEventDurationMS(long eventDurationMS);
-
- protected String trim(String str, int len) {
- String ret = str;
- if (str != null) {
- if (str.length() > len) {
- ret = str.substring(0, len);
- }
- }
- return ret;
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
index d648de3..2a8d792 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
@@ -190,8 +190,8 @@ public class AuthzAuditEvent extends AuditEventBase {
}
/**
- * @param timeStamp
- * the timeStamp to set
+ * @param eventTime
+ * the eventTime to set
*/
public void setEventTime(Date eventTime) {
this.eventTime = eventTime;
@@ -245,9 +245,7 @@ public class AuthzAuditEvent extends AuditEventBase {
/**
* @return the action
*/
- public String getAction() {
- return trim(action, MAX_ACTION_FIELD_SIZE);
- }
+ public String getAction() { return action; }
/**
* @param action
@@ -380,9 +378,7 @@ public class AuthzAuditEvent extends AuditEventBase {
/**
* @return the requestData
*/
- public String getRequestData() {
- return trim(requestData, MAX_REQUEST_DATA_FIELD_SIZE);
- }
+ public String getRequestData() { return requestData; }
/**
* @param requestData
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
index 98da1c2..f23f17d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.dao.DaoManager;
import org.apache.ranger.audit.destination.AuditDestination;
+import org.apache.ranger.audit.entity.AuthzAuditEventDbObj;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
@@ -91,6 +92,9 @@ public class DbAuditProvider extends AuditDestination {
if(jdbcPassword != null && !jdbcPassword.isEmpty()) {
mDbProperties.put(AUDIT_JPA_JDBC_PASSWORD, jdbcPassword);
}
+
+ // initialize the database related classes
+ AuthzAuditEventDbObj.init(props);
}
@Override