You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2015/11/13 03:38:13 UTC
[43/45] drill git commit: DRILL-4065: Maintain a cache of created
UserGroupInformation objects so that we don't accidentally generate thousands
of FileSystem objects & threads.
DRILL-4065: Maintain a cache of created UserGroupInformation objects so that we don't accidentally generate thousands of FileSystem objects & threads.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bdd2386e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bdd2386e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bdd2386e
Branch: refs/heads/master
Commit: bdd2386e284b21727b3498624a01fd1fbf429dea
Parents: 494492f
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Nov 10 16:04:09 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 18:33:37 2015 -0800
----------------------------------------------------------------------
.../drill/exec/util/ImpersonationUtil.java | 80 ++++++++++++++++++--
1 file changed, 72 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/bdd2386e/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
index 37fdc40..38d2797 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
@@ -17,18 +17,24 @@
*/
package org.apache.drill.exec.util;
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Set;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Sets;
/**
* Utilities for impersonation purpose.
@@ -36,8 +42,66 @@ import java.util.Set;
public class ImpersonationUtil {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ImpersonationUtil.class);
+ private static final LoadingCache<Key, UserGroupInformation> CACHE = CacheBuilder.newBuilder()
+ .maximumSize(100)
+ .expireAfterAccess(60, TimeUnit.MINUTES)
+ .build(new CacheLoader<Key, UserGroupInformation>() {
+ @Override
+ public UserGroupInformation load(Key key) throws Exception {
+ return UserGroupInformation.createProxyUser(key.proxyUserName, key.loginUser);
+ }
+ });
+
private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+ private static class Key {
+ final String proxyUserName;
+ final UserGroupInformation loginUser;
+
+ public Key(String proxyUserName, UserGroupInformation loginUser) {
+ super();
+ this.proxyUserName = proxyUserName;
+ this.loginUser = loginUser;
+ }
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((loginUser == null) ? 0 : loginUser.hashCode());
+ result = prime * result + ((proxyUserName == null) ? 0 : proxyUserName.hashCode());
+ return result;
+ }
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Key other = (Key) obj;
+ if (loginUser == null) {
+ if (other.loginUser != null) {
+ return false;
+ }
+ } else if (!loginUser.equals(other.loginUser)) {
+ return false;
+ }
+ if (proxyUserName == null) {
+ if (other.proxyUserName != null) {
+ return false;
+ }
+ } else if (!proxyUserName.equals(other.proxyUserName)) {
+ return false;
+ }
+ return true;
+ }
+
+
+ }
/**
* Create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} of operator owner if operator
* owner is valid. Otherwise create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} for
@@ -83,8 +147,8 @@ public class ImpersonationUtil {
return getProcessUserUGI();
}
- return UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser());
- } catch(IOException e) {
+ return CACHE.get(new Key(proxyUserName, UserGroupInformation.getLoginUser()));
+ } catch (IOException | ExecutionException e) {
final String errMsg = "Failed to create proxy user UserGroupInformation object: " + e.getMessage();
logger.error(errMsg, e);
throw new DrillRuntimeException(errMsg, e);