You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/06/02 21:42:52 UTC
[2/6] incubator-ranger git commit: RANGER-397 Incorporate review
comments
RANGER-397 Incorporate review comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/8469e643
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/8469e643
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/8469e643
Branch: refs/heads/tag-policy
Commit: 8469e643da3a81312aff58dc74673ed48e440f95
Parents: c526417
Author: Don Bosco Durai <bo...@apache.org>
Authored: Mon Jun 1 18:02:42 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Mon Jun 1 18:02:42 2015 -0700
----------------------------------------------------------------------
.../audit/destination/DBAuditDestination.java | 3 ++-
.../audit/destination/FileAuditDestination.java | 3 ++-
.../audit/destination/HDFSAuditDestination.java | 7 ++++--
.../destination/Log4JAuditDestination.java | 15 ++++++++---
.../audit/destination/SolrAuditDestination.java | 8 +++++-
.../ranger/audit/provider/BaseAuditHandler.java | 14 +++++++++--
.../audit/provider/MultiDestAuditProvider.java | 21 +++++++++++++---
.../ranger/audit/queue/AuditBatchQueue.java | 5 ++--
.../apache/ranger/audit/queue/AuditQueue.java | 26 +++++++++++++++-----
9 files changed, 79 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
index 6825637..3d31c06 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
@@ -78,7 +78,7 @@ public class DBAuditDestination extends AuditDestination {
@Override
public boolean log(Collection<AuditEventBase> events) {
boolean retValue = false;
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
if (beginTransaction()) {
@@ -210,6 +210,7 @@ public class DBAuditDestination extends AuditDestination {
entityManagerFactory = null;
daoManager = null;
}
+ logStatus();
}
private EntityManager getEntityManager() {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
index 77c8f61..c6cd8b2 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
@@ -106,7 +106,7 @@ public class FileAuditDestination extends AuditDestination {
@Override
synchronized public boolean logJSON(Collection<String> events) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
if (isStopped) {
@@ -183,6 +183,7 @@ public class FileAuditDestination extends AuditDestination {
}
logWriter = null;
}
+ logStatus();
}
// Helper methods in this class
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
index b63fa40..e79e42d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
@@ -105,7 +105,7 @@ public class HDFSAuditDestination extends AuditDestination {
@Override
synchronized public boolean logJSON(Collection<String> events) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
if (!initDone) {
@@ -142,8 +142,9 @@ public class HDFSAuditDestination extends AuditDestination {
@Override
public boolean log(Collection<AuditEventBase> events) {
if (isStopped) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
+ addDeferredCount(events.size());
logError("log() called after stop was requested. name=" + getName());
return false;
}
@@ -153,6 +154,7 @@ public class HDFSAuditDestination extends AuditDestination {
jsonList.add(MiscUtil.stringify(event));
} catch (Throwable t) {
logger.error("Error converting to JSON. event=" + event);
+ addTotalCount(1);
addFailedCount(1);
logFailedEvent(event);
}
@@ -185,6 +187,7 @@ public class HDFSAuditDestination extends AuditDestination {
}
logWriter = null;
}
+ logStatus();
}
// Helper methods in this class
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
index b57cd86..9521a4a 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
@@ -59,10 +59,17 @@ public class Log4JAuditDestination extends AuditDestination {
+ ", loggerName=" + loggerName);
}
+
+ @Override
+ public void stop() {
+ super.stop();
+ logStatus();
+ }
+
@Override
public boolean log(AuditEventBase event) {
if (!auditLogger.isInfoEnabled()) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(1);
return true;
}
@@ -77,7 +84,7 @@ public class Log4JAuditDestination extends AuditDestination {
@Override
public boolean log(Collection<AuditEventBase> events) {
if (!auditLogger.isInfoEnabled()) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
return true;
}
@@ -90,7 +97,7 @@ public class Log4JAuditDestination extends AuditDestination {
@Override
public boolean logJSON(String event) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(1);
if (!auditLogger.isInfoEnabled()) {
return true;
@@ -106,7 +113,7 @@ public class Log4JAuditDestination extends AuditDestination {
@Override
public boolean logJSON(Collection<String> events) {
if (!auditLogger.isInfoEnabled()) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index b43e573..213e4b2 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -55,6 +55,12 @@ public class SolrAuditDestination extends AuditDestination {
super.init(props, propPrefix);
connect();
}
+
+ @Override
+ public void stop() {
+ super.stop();
+ logStatus();
+ }
synchronized void connect() {
if (solrClient == null) {
@@ -103,7 +109,7 @@ public class SolrAuditDestination extends AuditDestination {
@Override
public boolean log(Collection<AuditEventBase> events) {
try {
- logStatusIfRequired(true);
+ logStatusIfRequired();
addTotalCount(events.size());
if (solrClient == null) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index ef85743..3859a7e 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -234,9 +234,17 @@ public abstract class BaseAuditHandler implements AuditHandler {
return lastDeferredCount;
}
- public void logStatusIfRequired(boolean ifRequired) {
+ public void logStatusIfRequired() {
long currTime = System.currentTimeMillis();
- if (!ifRequired || (currTime - lastStatusLogTime) > statusLogIntervalMS) {
+ if ((currTime - lastStatusLogTime) > statusLogIntervalMS) {
+ logStatus();
+ }
+ }
+
+ public void logStatus() {
+ try {
+ long currTime = System.currentTimeMillis();
+
long diffTime = currTime - lastStatusLogTime;
lastStatusLogTime = currTime;
@@ -278,6 +286,8 @@ public abstract class BaseAuditHandler implements AuditHandler {
+ (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
: "");
LOG.info(msg);
+ } catch (Throwable t) {
+ LOG.error("Error while printing stats. auditProvider=" + getName());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
index 9a5ff13..26108ca 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
@@ -33,13 +33,16 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
protected List<AuditHandler> mProviders = new ArrayList<AuditHandler>();
static final String DEFAULT_NAME = "multi_dest";
-
+
public MultiDestAuditProvider() {
LOG.info("MultiDestAuditProvider: creating..");
setName(DEFAULT_NAME);
}
public MultiDestAuditProvider(AuditHandler provider) {
+ LOG.info("MultiDestAuditProvider(): provider="
+ + (provider == null ? null : provider.getName()));
+ setName(DEFAULT_NAME);
addAuditProvider(provider);
}
@@ -59,12 +62,22 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
}
}
-
+ @Override
+ public void setParentPath(String parentPath) {
+ super.setParentPath(parentPath);
+ for (AuditHandler provider : mProviders) {
+ if (provider instanceof BaseAuditHandler) {
+ BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
+ baseAuditHander.setParentPath(getName());
+ }
+ }
+ }
+
@Override
public void setName(String name) {
super.setName(name);
for (AuditHandler provider : mProviders) {
- if( provider instanceof BaseAuditHandler) {
+ if (provider instanceof BaseAuditHandler) {
BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
baseAuditHander.setParentPath(getName());
}
@@ -77,7 +90,7 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
+ provider.getClass().getCanonicalName() + ")");
mProviders.add(provider);
- if( provider instanceof BaseAuditHandler) {
+ if (provider instanceof BaseAuditHandler) {
BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
baseAuditHander.setParentPath(getName());
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
index 4278731..2361bbf 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
@@ -39,7 +39,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
Thread consumerThread = null;
static int threadCount = 0;
- static final String DEFAULT_NAME = "summary";
+ static final String DEFAULT_NAME = "batch";
public AuditBatchQueue(AuditHandler consumer) {
super(consumer);
@@ -209,7 +209,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
long lastDispatchTime = System.currentTimeMillis();
boolean isDestActive = true;
while (true) {
- logStatusIfRequired(true);
+ logStatusIfRequired();
// Time to next dispatch
long nextDispatchDuration = lastDispatchTime
@@ -347,6 +347,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
} catch (Throwable t) {
logger.error("Error while calling stop on consumer.", t);
}
+ logStatus();
logger.info("Exiting consumerThread.run() method. name=" + getName());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
index 50a74b5..057f192 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
@@ -33,8 +33,9 @@ public abstract class AuditQueue extends BaseAuditHandler {
public static final int AUDIT_MAX_QUEUE_SIZE_DEFAULT = 1024 * 1024;
public static final int AUDIT_BATCH_INTERVAL_DEFAULT_MS = 1000;
public static final int AUDIT_BATCH_SIZE_DEFAULT = 1000;
-
- //This is the max time the consumer thread will wait before exiting the loop
+
+ // This is the max time the consumer thread will wait before exiting the
+ // loop
public static final int AUDIT_CONSUMER_THREAD_WAIT_MS = 5000;
private int maxQueueSize = AUDIT_MAX_QUEUE_SIZE_DEFAULT;
@@ -60,14 +61,18 @@ public abstract class AuditQueue extends BaseAuditHandler {
protected int fileSpoolMaxWaitTime = 5 * 60 * 1000; // Default 5 minutes
protected int fileSpoolDrainThresholdPercent = 80;
- //This is set when the first time stop is called.
+ // This is set when the first time stop is called.
protected long stopTime = 0;
-
+
/**
* @param consumer
*/
public AuditQueue(AuditHandler consumer) {
this.consumer = consumer;
+ if (consumer instanceof BaseAuditHandler) {
+ BaseAuditHandler baseAuditHander = (BaseAuditHandler) consumer;
+ baseAuditHander.setParentPath(getName());
+ }
}
@Override
@@ -111,9 +116,18 @@ public abstract class AuditQueue extends BaseAuditHandler {
}
@Override
+ public void setParentPath(String parentPath) {
+ super.setParentPath(parentPath);
+ if (consumer != null && consumer instanceof BaseAuditHandler) {
+ BaseAuditHandler base = (BaseAuditHandler) consumer;
+ base.setParentPath(getName());
+ }
+ }
+
+ @Override
public void setName(String name) {
super.setName(name);
- if( consumer != null && consumer instanceof BaseAuditHandler) {
+ if (consumer != null && consumer instanceof BaseAuditHandler) {
BaseAuditHandler base = (BaseAuditHandler) consumer;
base.setParentPath(getName());
}
@@ -126,7 +140,7 @@ public abstract class AuditQueue extends BaseAuditHandler {
public boolean isDrainMaxTimeElapsed() {
return (stopTime - System.currentTimeMillis()) > AUDIT_CONSUMER_THREAD_WAIT_MS;
}
-
+
public boolean isDrain() {
return isDrain;
}