You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by sn...@apache.org on 2015/06/09 08:43:11 UTC

[07/17] incubator-ranger git commit: RANGER-261 Fix audit to HDFS in Kerberos environment

RANGER-261 Fix audit to HDFS in Kerberos environment


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/859f3bd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/859f3bd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/859f3bd3

Branch: refs/heads/ranger-0.5
Commit: 859f3bd3ec7a9098b27429646547d1d52f0f353c
Parents: b12e761
Author: Don Bosco Durai <bo...@apache.org>
Authored: Thu Jun 4 21:55:44 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Fri Jun 5 08:26:48 2015 -0700

----------------------------------------------------------------------
 .../audit/destination/HDFSAuditDestination.java |   6 +-
 .../audit/provider/AuditProviderFactory.java    |   3 +-
 .../apache/ranger/audit/provider/MiscUtil.java  | 132 +++++++++++++++++--
 .../ranger/audit/queue/AuditAsyncQueue.java     |  27 +++-
 .../ranger/audit/queue/AuditBatchQueue.java     |  26 +++-
 .../ranger/audit/queue/AuditFileSpool.java      |  25 +++-
 .../apache/ranger/audit/queue/AuditQueue.java   |   8 ++
 .../ranger/audit/queue/AuditSummaryQueue.java   |  28 +++-
 .../plugin/audit/RangerDefaultAuditHandler.java |   3 +-
 .../service-defs/ranger-servicedef-kafka.json   |  29 +++-
 .../service-defs/ranger-servicedef-solr.json    |   8 +-
 .../scripts/ranger-admin-services.sh            |   4 +-
 .../hadoop/RangerHdfsAuthorizer.java            |   3 +-
 kms/scripts/ranger-kms                          |   2 +-
 plugin-kafka/conf/kafka-ranger-env.sh           |   2 +-
 .../kafka/authorizer/RangerKafkaAuthorizer.java |  86 ++++++++----
 .../service/filter/RangerRESTAPIFilter.java     |   9 --
 17 files changed, 327 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
index e79e42d..4fc3a0b 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
@@ -119,6 +119,10 @@ public class HDFSAuditDestination extends AuditDestination {
 		}
 
 		try {
+			if (logger.isDebugEnabled()) {
+				logger.debug("UGI=" + MiscUtil.getUGILoginUser()
+						+ ". Will write to HDFS file=" + currentFileName);
+			}
 			PrintWriter out = getLogFileStream();
 			for (String event : events) {
 				out.println(event);
@@ -154,7 +158,7 @@ public class HDFSAuditDestination extends AuditDestination {
 				jsonList.add(MiscUtil.stringify(event));
 			} catch (Throwable t) {
 				logger.error("Error converting to JSON. event=" + event);
-				addTotalCount(1);				
+				addTotalCount(1);
 				addFailedCount(1);
 				logFailedEvent(event);
 			}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index f02ba62..1146e0b 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -125,9 +125,8 @@ public class AuditProviderFactory {
 
 		List<AuditHandler> providers = new ArrayList<AuditHandler>();
 
-		// TODO: Delete me
 		for (Object propNameObj : props.keySet()) {
-			LOG.info("DELETE ME: " + propNameObj.toString() + "="
+			LOG.info("AUDIT PROPERTY: " + propNameObj.toString() + "="
 					+ props.getProperty(propNameObj.toString()));
 		}
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
index abb0a90..907b9b8 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
@@ -17,17 +17,25 @@
 package org.apache.ranger.audit.provider;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.InetAddress;
 import java.rmi.dgc.VMID;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Hashtable;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.UUID;
 
+import javax.security.auth.Subject;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.log4j.helpers.LogLog;
 import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
 
@@ -35,6 +43,8 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
 public class MiscUtil {
+	private static final Log logger = LogFactory.getLog(MiscUtil.class);
+
 	public static final String TOKEN_START = "%";
 	public static final String TOKEN_END = "%";
 	public static final String TOKEN_HOSTNAME = "hostname";
@@ -51,6 +61,11 @@ public class MiscUtil {
 
 	private static Gson sGsonBuilder = null;
 	private static String sApplicationType = null;
+	private static UserGroupInformation ugiLoginUser = null;
+	private static Subject subjectLoginUser = null;
+
+	private static Map<String, LogHistory> logHistoryList = new Hashtable<String,LogHistory>();
+	private static int logInterval = 30000; // 30 seconds
 
 	static {
 		try {
@@ -371,26 +386,127 @@ public class MiscUtil {
 	public static List<String> toArray(String destListStr, String delim) {
 		List<String> list = new ArrayList<String>();
 		if (destListStr != null && !destListStr.isEmpty()) {
-			StringTokenizer tokenizer = new StringTokenizer(destListStr, delim.trim());
+			StringTokenizer tokenizer = new StringTokenizer(destListStr,
+					delim.trim());
 			while (tokenizer.hasMoreTokens()) {
 				list.add(tokenizer.nextToken());
 			}
 		}
 		return list;
 	}
-	
-	public static String getCredentialString(String url,String alias) {
+
+	public static String getCredentialString(String url, String alias) {
 		String ret = null;
 
-		if(url != null && alias != null) {
-			char[] cred = RangerCredentialProvider.getInstance().getCredentialString(url,alias);
+		if (url != null && alias != null) {
+			char[] cred = RangerCredentialProvider.getInstance()
+					.getCredentialString(url, alias);
 
-			if ( cred != null ) {
-				ret = new String(cred);	
+			if (cred != null) {
+				ret = new String(cred);
 			}
 		}
-		
+
 		return ret;
 	}
 
+	/**
+	 * @param ugiLoginUser
+	 */
+	public static void setUGILoginUser(UserGroupInformation newUGI, Subject newSubject) {
+		if (newUGI != null) {
+			UserGroupInformation.setLoginUser(newUGI);
+			ugiLoginUser = newUGI;
+			logger.info("Setting UGI=" + newUGI );
+		} else {
+			logger.error("UGI is null. Not setting it.");
+			ugiLoginUser = null;
+		}
+		logger.info("Setting SUBJECT");
+		subjectLoginUser = newSubject;
+	}
+
+	public static UserGroupInformation getUGILoginUser() {
+		if (ugiLoginUser == null) {
+			try {
+				ugiLoginUser = UserGroupInformation.getLoginUser();
+			} catch (IOException e) {
+				logger.error("Error getting UGI.", e);
+			}
+		}
+		return ugiLoginUser;
+	}
+
+	
+	public static Subject getSubjectLoginUser() {
+		return subjectLoginUser;
+	}
+
+	/**
+	 * @param userName
+	 * @return
+	 */
+	static public Set<String> getGroupsForRequestUser(String userName) {
+		if (userName == null) {
+			return null;
+		}
+		try {
+			UserGroupInformation ugi = UserGroupInformation
+					.createRemoteUser(userName);
+			// UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+			String groups[] = ugi.getGroupNames();
+			if (groups != null && groups.length > 0) {
+				java.util.Set<String> groupsSet = new java.util.HashSet<String>();
+				for (int i = 0; i < groups.length; i++) {
+					groupsSet.add(groups[i]);
+				}
+				return groupsSet;
+			}
+		} catch (Throwable e) {
+			logErrorMessageByInterval(
+					logger, "Error getting groups for users. userName=" + userName, e);
+		}
+		return null;
+	}
+
+	static public boolean logErrorMessageByInterval(Log useLogger, String message) {
+		return logErrorMessageByInterval(useLogger, message, null);
+	}
+
+	/**
+	 * @param string
+	 * @param e
+	 */
+	static public boolean logErrorMessageByInterval(Log useLogger, String message, Throwable e) {
+		LogHistory log = logHistoryList.get(message);
+		if (log == null) {
+			log = new LogHistory();
+			logHistoryList.put(message, log);
+		}
+		if ((System.currentTimeMillis() - log.lastLogTime) > logInterval) {
+			log.lastLogTime = System.currentTimeMillis();
+			int counter = log.counter;
+			log.counter = 0;
+			if (counter > 0) {
+				message += ". Messages suppressed before: " + counter;
+			}
+			if (e == null) {
+				useLogger.error(message);
+			} else {
+				useLogger.error(message, e);
+			}
+			
+			return true;
+		} else {
+			log.counter++;
+		}
+		return false;
+
+	}
+
+	static class LogHistory {
+		long lastLogTime = 0;
+		int counter = 0;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
index 47480da..9b4fcbd 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
@@ -19,6 +19,7 @@
 
 package org.apache.ranger.audit.queue;
 
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -27,6 +28,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.provider.AuditHandler;
+import org.apache.ranger.audit.provider.MiscUtil;
 
 /**
  * This is a non-blocking queue with no limit on capacity.
@@ -124,6 +126,26 @@ public class AuditAsyncQueue extends AuditQueue implements Runnable {
 	 */
 	@Override
 	public void run() {
+		try {
+			if (isConsumerDestination) {
+				PrivilegedAction<Void> action = new PrivilegedAction<Void>() {
+					public Void run() {
+						runDoAs();
+						return null;
+					};
+				};
+				logger.info("Running queue " + getName() + " as user "
+						+ MiscUtil.getUGILoginUser());
+				MiscUtil.getUGILoginUser().doAs(action);
+			} else {
+				runDoAs();
+			}
+		} catch (Throwable t) {
+			logger.fatal("Exited thread abnormaly. queue=" + getName(), t);
+		}
+	}
+
+	public void runDoAs() {
 		while (true) {
 			try {
 				AuditEventBase event = null;
@@ -141,9 +163,7 @@ public class AuditAsyncQueue extends AuditQueue implements Runnable {
 					consumer.log(eventList);
 				}
 			} catch (InterruptedException e) {
-				logger.info(
-						"Caught exception in consumer thread. Shutdown might be in progress",
-						e);
+				logger.info("Caught exception in consumer thread. Shutdown might be in progress");
 			} catch (Throwable t) {
 				logger.error("Caught error during processing request.", t);
 			}
@@ -172,7 +192,6 @@ public class AuditAsyncQueue extends AuditQueue implements Runnable {
 			logger.error("Error while calling stop on consumer.", t);
 		}
 		logger.info("Exiting consumerThread.run() method. name=" + getName());
-
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
index 2361bbf..dca3030 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
@@ -19,6 +19,7 @@
 
 package org.apache.ranger.audit.queue;
 
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Properties;
@@ -30,6 +31,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.provider.AuditHandler;
+import org.apache.ranger.audit.provider.MiscUtil;
 
 public class AuditBatchQueue extends AuditQueue implements Runnable {
 	private static final Log logger = LogFactory.getLog(AuditBatchQueue.class);
@@ -206,6 +208,26 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 	 */
 	@Override
 	public void run() {
+		try {
+			if (isConsumerDestination) {
+				PrivilegedAction<Void> action = new PrivilegedAction<Void>() {
+					public Void run() {
+						runDoAs();
+						return null;
+					};
+				};
+				logger.info("Running queue " + getName() + " as user "
+						+ MiscUtil.getUGILoginUser());
+				MiscUtil.getUGILoginUser().doAs(action);
+			} else {
+				runDoAs();
+			}
+		} catch (Throwable t) {
+			logger.fatal("Exited thread abnormaly. queue=" + getName(), t);
+		}
+	}
+
+	public void runDoAs() {
 		long lastDispatchTime = System.currentTimeMillis();
 		boolean isDestActive = true;
 		while (true) {
@@ -265,9 +287,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 					lastDispatchTime = System.currentTimeMillis();
 				}
 			} catch (InterruptedException e) {
-				logger.info(
-						"Caught exception in consumer thread. Shutdown might be in progress",
-						e);
+				logger.info("Caught exception in consumer thread. Shutdown might be in progress");
 				setDrain(true);
 			} catch (Throwable t) {
 				logger.error("Caught error during processing request.", t);

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
index 9c10c54..4a4e707 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
@@ -28,6 +28,7 @@ import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
@@ -108,6 +109,8 @@ public class AuditFileSpool implements Runnable {
 	boolean isDrain = false;
 	boolean isDestDown = true;
 
+	int ugiVersion = -1;
+
 	private Gson gson = null;
 
 	public AuditFileSpool(AuditQueue queueProvider,
@@ -751,9 +754,27 @@ public class AuditFileSpool implements Runnable {
 	 */
 	@Override
 	public void run() {
+		try {
+			PrivilegedAction<Void> action = new PrivilegedAction<Void>() {
+				public Void run() {
+					runDoAs();
+					return null;
+				};
+			};
+			logger.info("Running fileSpool " + consumerProvider.getName()
+					+ " as user " + MiscUtil.getUGILoginUser());
+			MiscUtil.getUGILoginUser().doAs(action);
+		} catch (Throwable t) {
+			logger.fatal("Exited thread without abnormaly. queue="
+					+ consumerProvider.getName(), t);
+		}
+	}
+
+	public void runDoAs() {
 		// boolean isResumed = false;
 		while (true) {
 			try {
+
 				// Let's pause between each iteration
 				if (currentConsumerIndexRecord == null) {
 					currentConsumerIndexRecord = indexQueue.poll(
@@ -843,9 +864,7 @@ public class AuditFileSpool implements Runnable {
 					closeFileIfNeeded();
 				}
 			} catch (InterruptedException e) {
-				logger.info(
-						"Caught exception in consumer thread. Shutdown might be in progress",
-						e);
+				logger.info("Caught exception in consumer thread. Shutdown might be in progress");
 			} catch (Throwable t) {
 				logger.error("Exception in destination writing thread.", t);
 			}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
index 057f192..000a658 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
@@ -23,6 +23,7 @@ import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.audit.destination.AuditDestination;
 import org.apache.ranger.audit.provider.AuditHandler;
 import org.apache.ranger.audit.provider.BaseAuditHandler;
 import org.apache.ranger.audit.provider.MiscUtil;
@@ -61,6 +62,7 @@ public abstract class AuditQueue extends BaseAuditHandler {
 	protected int fileSpoolMaxWaitTime = 5 * 60 * 1000; // Default 5 minutes
 	protected int fileSpoolDrainThresholdPercent = 80;
 
+	boolean isConsumerDestination = false;
 	// This is set when the first time stop is called.
 	protected long stopTime = 0;
 
@@ -73,6 +75,12 @@ public abstract class AuditQueue extends BaseAuditHandler {
 			BaseAuditHandler baseAuditHander = (BaseAuditHandler) consumer;
 			baseAuditHander.setParentPath(getName());
 		}
+
+		if (consumer != null && consumer instanceof AuditDestination) {
+			// If consumer is destination, then the thread should run as server
+			// user
+			isConsumerDestination = true;
+		}
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditSummaryQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditSummaryQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditSummaryQueue.java
index f1ce799..56ba55e 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditSummaryQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditSummaryQueue.java
@@ -19,6 +19,7 @@
 
 package org.apache.ranger.audit.queue;
 
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
@@ -145,6 +146,28 @@ public class AuditSummaryQueue extends AuditQueue implements Runnable {
 	 */
 	@Override
 	public void run() {
+		try {
+			if (isConsumerDestination) {
+				PrivilegedAction<Void> action = new PrivilegedAction<Void>() {
+					public Void run() {
+						runDoAs();
+						return null;
+					};
+				};
+				logger.info("Running queue " + getName() + " as user "
+						+ MiscUtil.getUGILoginUser());
+				MiscUtil.getUGILoginUser().doAs(action);
+			} else {
+				runDoAs();
+			}
+		} catch (Throwable t) {
+			logger.fatal("Exited thread without abnormaly. queue=" + getName(),
+					t);
+		}
+	}
+
+	public void runDoAs() {
+
 		long lastDispatchTime = System.currentTimeMillis();
 
 		while (true) {
@@ -174,9 +197,7 @@ public class AuditSummaryQueue extends AuditQueue implements Runnable {
 					lastDispatchTime = System.currentTimeMillis();
 				}
 			} catch (InterruptedException e) {
-				logger.info(
-						"Caught exception in consumer thread. Shutdown might be in progress",
-						e);
+				logger.info("Caught exception in consumer thread. Shutdown might be in progress");
 			} catch (Throwable t) {
 				logger.error("Caught error during processing request.", t);
 			}
@@ -243,7 +264,6 @@ public class AuditSummaryQueue extends AuditQueue implements Runnable {
 			logger.error("Error while calling stop on consumer.", t);
 		}
 		logger.info("Exiting consumerThread.run() method. name=" + getName());
-
 	}
 
 	class AuditSummary {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
index 844d0ac..c553618 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
@@ -28,7 +28,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.model.AuthzAuditEvent;
 import org.apache.ranger.audit.provider.AuditProviderFactory;
 import org.apache.ranger.audit.provider.MiscUtil;
-import org.apache.ranger.plugin.model.RangerServiceDef;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
 import org.apache.ranger.plugin.policyengine.RangerAccessResult;
 import org.apache.ranger.plugin.policyengine.RangerAccessResource;
@@ -83,7 +82,7 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor {
 		RangerAccessRequest request = result != null ? result.getAccessRequest() : null;
 
 		if(request != null && result != null && result.getIsAudited()) {
-			RangerServiceDef     serviceDef   = result.getServiceDef();
+			//RangerServiceDef     serviceDef   = result.getServiceDef();
 			RangerAccessResource resource     = request.getResource();
 			String               resourceType = resource == null ? null : resource.getLeafName();
 			String               resourcePath = resource == null ? null : resource.getAsString();

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
----------------------------------------------------------------------
diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
index 4752eff..bf7a4df 100644
--- a/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
+++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
@@ -31,17 +31,29 @@
 		{
 			"itemId":1,
 			"name":"publish",
-			"label":"Publish"
+			"label":"Publish",
+			"impliedGrants":[
+				"describe"
+			]
+			
 		},
 		{
 			"itemId":2,
 			"name":"consume",
-			"label":"Consume"
+			"label":"Consume",
+			"impliedGrants":[
+				"describe"
+			]
+			
 		},
 		{
 			"itemId":5,
 			"name":"configure",
-			"label":"Configure"
+			"label":"Configure",
+			"impliedGrants":[
+				"describe"
+			]
+			
 		},
 		{
 			"itemId":6,
@@ -51,7 +63,14 @@
 		{
 			"itemId":7,
 			"name":"kafka_admin",
-			"label":"Kafka Admin"
+			"label":"Kafka Admin",
+			"impliedGrants":[
+				"publish",
+				"consume",
+				"configure",
+				"describe"
+			]
+			
 		}
 		
 	],
@@ -109,4 +128,4 @@
 		}
 		
 	]
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
----------------------------------------------------------------------
diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
index 107b5d6..bc27352 100644
--- a/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
+++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
@@ -47,7 +47,11 @@
 		{
 			"itemId":900,
 			"name":"solr_admin",
-			"label":"Solr Admin"
+			"label":"Solr Admin",
+			"impliedGrants":[
+				"query,update,others"
+			]
+			
 		}
 		
 	],
@@ -117,4 +121,4 @@
 		}
 		
 	]
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/embeddedwebserver/scripts/ranger-admin-services.sh
----------------------------------------------------------------------
diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh
index 6931dbf..880ff49 100755
--- a/embeddedwebserver/scripts/ranger-admin-services.sh
+++ b/embeddedwebserver/scripts/ranger-admin-services.sh
@@ -53,12 +53,12 @@ then
 fi
 
 start() {
-	java -Dproc_rangeradmin ${JAVA_OPTS} -Dlogdir=${XAPOLICYMGR_EWS_DIR}/logs/ -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*" org.apache.ranger.server.tomcat.EmbeddedServer > logs/catalina.out 2>&1 &
+	java -Dproc_rangeradmin ${JAVA_OPTS} -Dlogdir=${XAPOLICYMGR_EWS_DIR}/logs/ -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:$CLASSPATH" org.apache.ranger.server.tomcat.EmbeddedServer > logs/catalina.out 2>&1 &
 	echo "Apache Ranger Admin has started."
 }
 
 stop(){
-	java ${JAVA_OPTS} -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}" org.apache.ranger.server.tomcat.StopEmbeddedServer > logs/catalina.out 2>&1
+	java ${JAVA_OPTS} -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:$CLASSPATH" org.apache.ranger.server.tomcat.StopEmbeddedServer > logs/catalina.out 2>&1
 	echo "Apache Ranger Admin has been stopped."
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
----------------------------------------------------------------------
diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
index f6fa8bd..fa2155c 100644
--- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
+++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
@@ -49,7 +49,6 @@ import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants;
 import org.apache.ranger.authorization.hadoop.exceptions.RangerAccessControlException;
 import org.apache.ranger.authorization.utils.StringUtil;
 import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
-import org.apache.ranger.plugin.model.RangerServiceDef;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
 import org.apache.ranger.plugin.policyengine.RangerAccessResource;
@@ -478,7 +477,7 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler {
 		}
 
 		RangerAccessRequest  request      = result.getAccessRequest();
-		RangerServiceDef     serviceDef   = result.getServiceDef();
+//		RangerServiceDef     serviceDef   = result.getServiceDef();
 		RangerAccessResource resource     = request.getResource();
 		String               resourceType = resource != null ? resource.getLeafName() : null;
 		String               resourcePath = resource != null ? resource.getAsString() : null;

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/kms/scripts/ranger-kms
----------------------------------------------------------------------
diff --git a/kms/scripts/ranger-kms b/kms/scripts/ranger-kms
index f0fa1b4..805ebcc 100755
--- a/kms/scripts/ranger-kms
+++ b/kms/scripts/ranger-kms
@@ -72,7 +72,7 @@ fi
 
 KMS_CONF_DIR=${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf
 
-JAVA_OPTS="${JAVA_OPTS} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/* "
+JAVA_OPTS="${JAVA_OPTS} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:$CLASSPATH "
 
 if [ "${action}" == "START" ]; then
 	echo "+ java -D${PROC_NAME} ${JAVA_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} "

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/plugin-kafka/conf/kafka-ranger-env.sh
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/kafka-ranger-env.sh b/plugin-kafka/conf/kafka-ranger-env.sh
index 2ec122d..219353d 100755
--- a/plugin-kafka/conf/kafka-ranger-env.sh
+++ b/plugin-kafka/conf/kafka-ranger-env.sh
@@ -17,4 +17,4 @@
 
 curr_dir=`pwd`
 cd `dirname $0`; script_dir=`pwd`; cd $curr_dir
-export CLASSPATH="$CLASSPATH:${script_dir}:/etc/kafka/conf"
+export CLASSPATH="$CLASSPATH:${script_dir}:/etc/kafka/conf:/usr/hdp/current/hadoop-hdfs-client/*:/usr/hdp/current/hadoop-hdfs-client/lib/*"

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
----------------------------------------------------------------------
diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
index 0d0cffc..f982bbf 100644
--- a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
+++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
@@ -19,8 +19,11 @@
 
 package org.apache.ranger.authorization.kafka.authorizer;
 
+import java.security.Principal;
 import java.util.Date;
 
+import javax.security.auth.Subject;
+
 import kafka.security.auth.Acl;
 import kafka.security.auth.Authorizer;
 import kafka.security.auth.KafkaPrincipal;
@@ -28,11 +31,14 @@ import kafka.security.auth.Operation;
 import kafka.security.auth.Resource;
 import kafka.security.auth.ResourceType;
 import kafka.server.KafkaConfig;
+import kafka.common.security.LoginManager;
 import kafka.network.RequestChannel.Session;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ranger.audit.provider.MiscUtil;
 import org.apache.ranger.authorization.utils.StringUtil;
 import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
@@ -64,9 +70,6 @@ public class RangerKafkaAuthorizer implements Authorizer {
 	int errorLogFreq = 30000; // Log after every 30 seconds
 
 	public RangerKafkaAuthorizer() {
-		if (rangerPlugin == null) {
-			rangerPlugin = new RangerBasePlugin("kafka", "kafka");
-		}
 	}
 
 	/*
@@ -76,23 +79,70 @@ public class RangerKafkaAuthorizer implements Authorizer {
 	 */
 	@Override
 	public void initialize(KafkaConfig kafkaConfig) {
-		rangerPlugin.init();
-		RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler();
 
-		rangerPlugin.setResultProcessor(auditHandler);
+		if (rangerPlugin == null) {
+			rangerPlugin = new RangerBasePlugin("kafka", "kafka");
+
+			try {
+				Subject subject = LoginManager.subject();
+				logger.info("SUBJECT "
+						+ (subject == null ? "not found" : "found"));
+				if (subject != null) {
+					logger.info("SUBJECT.PRINCIPALS.size()="
+							+ subject.getPrincipals().size());
+					java.util.Set<Principal> principals = subject
+							.getPrincipals();
+					for (Principal principal : principals) {
+						logger.info("SUBJECT.PRINCIPAL.NAME="
+								+ principal.getName());
+					}
+					try {
+						// Do not remove the below statement. The default
+						// getLoginUser does some initialization which is needed
+						// for getUGIFromSubject() to work.
+						logger.info("Default UGI before using Subject from Kafka:"
+								+ UserGroupInformation.getLoginUser());
+					} catch (Throwable t) {
+						logger.error(t);
+					}
+					UserGroupInformation ugi = UserGroupInformation
+							.getUGIFromSubject(subject);
+					logger.info("SUBJECT.UGI.NAME=" + ugi.getUserName()
+							+ ", ugi=" + ugi);
+					MiscUtil.setUGILoginUser(ugi, subject);
+				} else {
+					logger.info("Server username is not available");
+				}
+				logger.info("LoginUser=" + MiscUtil.getUGILoginUser());
+			} catch (Throwable t) {
+				logger.error("Error getting principal.", t);
+			}
+
+			logger.info("Calling plugin.init()");
+			rangerPlugin.init();
+
+			RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler();
+			rangerPlugin.setResultProcessor(auditHandler);
+		}
 	}
 
 	@Override
 	public boolean authorize(Session session, Operation operation,
 			Resource resource) {
 
+		if (rangerPlugin == null) {
+			MiscUtil.logErrorMessageByInterval(logger,
+					"Authorizer is still not initialized");
+			return false;
+		}
 		String userName = null;
 		if (session.principal() != null) {
 			userName = session.principal().getName();
 			userName = StringUtils.substringBefore(userName, "/");
 			userName = StringUtils.substringBefore(userName, "@");
 		}
-		java.util.Set<String> userGroups = getGroupsForUser(userName);
+		java.util.Set<String> userGroups = MiscUtil
+				.getGroupsForRequestUser(userName);
 		String ip = session.host();
 
 		Date eventTime = StringUtil.getUTCDate();
@@ -101,9 +151,8 @@ public class RangerKafkaAuthorizer implements Authorizer {
 		String validationStr = "";
 
 		if (accessType == null) {
-			if (rangerPlugin
-					.logErrorMessage("Unsupported access type. operation="
-							+ operation)) {
+			if (MiscUtil.logErrorMessageByInterval(logger,
+					"Unsupported access type. operation=" + operation)) {
 				logger.fatal("Unsupported access type. session=" + session
 						+ ", operation=" + operation + ", resource=" + resource);
 			}
@@ -138,8 +187,8 @@ public class RangerKafkaAuthorizer implements Authorizer {
 
 		boolean returnValue = true;
 		if (validationFailed) {
-			rangerPlugin.logErrorMessage(validationStr + ", request="
-					+ rangerRequest);
+			MiscUtil.logErrorMessageByInterval(logger, validationStr
+					+ ", request=" + rangerRequest);
 			returnValue = false;
 		} else {
 
@@ -229,19 +278,6 @@ public class RangerKafkaAuthorizer implements Authorizer {
 	}
 
 	/**
-	 * @param userName
-	 * @return
-	 */
-	private java.util.Set<String> getGroupsForUser(String userName) {
-		if (userName == null) {
-			return null;
-		}
-
-		// TODO: Need to implement this method
-		return null;
-	}
-
-	/**
 	 * @param operation
 	 * @return
 	 */

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/859f3bd3/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java b/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java
index 21536ac..0e5bb08 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java
@@ -93,15 +93,6 @@ public class RangerRESTAPIFilter extends LoggingFilter {
 		if (logStdOut) {
 			String path = request.getRequestUri().getPath();
 
-			// mediaType=multipart/form-data;boundary=----WebKitFormBoundaryTHan76r5AkgpAuVG
-			if (request.getMediaType() != null) {
-				// logger.info("DELETE ME: mediaType=" + request.getMediaType()
-				// + ", getType()" + request.getMediaType().getType()
-				// + ", getSubType()="
-				// + request.getMediaType().getSubtype());
-			} else {
-				logger.info("DELETE ME: mediaType is null. path=" + path);
-			}
 			if ((request.getMediaType() == null || !request.getMediaType()
 					.getType().equals("multipart"))
 					&& !path.endsWith("/service/general/logs")) {