You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/09/13 23:21:02 UTC
svn commit: r1384541 - in /hive/branches/branch-0.9:
metastore/src/java/org/apache/hadoop/hive/metastore/
shims/src/0.20/java/org/apache/hadoop/hive/shims/
shims/src/common-secure/java/org/apache/hadoop/hive/shims/
shims/src/common-secure/java/org/apac...
Author: hashutosh
Date: Thu Sep 13 21:21:01 2012
New Revision: 1384541
URL: http://svn.apache.org/viewvc?rev=1384541&view=rev
Log:
HIVE-3098 : Memory leak from large number of FileSystem instances in FileSystem.CACHE (Mithun Radhakrishnan via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java
hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java?rev=1384541&r1=1384540&r2=1384541&view=diff
==============================================================================
--- hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java (original)
+++ hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java Thu Sep 13 21:21:01 2012
@@ -131,6 +131,8 @@ public class TUGIBasedProcessor<I extend
throw new RuntimeException(ie); // unexpected!
} catch (IOException ioe) {
throw new RuntimeException(ioe); // unexpected!
+ } finally {
+ shim.closeAllForUGI(clientUgi);
}
}
}
Modified: hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1384541&r1=1384540&r2=1384541&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Thu Sep 13 21:21:01 2012
@@ -573,4 +573,10 @@ public class Hadoop20Shims implements Ha
public String getJobLauncherHttpAddress(Configuration conf) {
return conf.get("mapred.job.tracker.http.address");
}
+
+ @Override
+ public void closeAllForUGI(UserGroupInformation ugi) {
+ // No such functionality in ancient hadoop
+ return;
+ }
}
Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1384541&r1=1384540&r2=1384541&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Thu Sep 13 21:21:01 2012
@@ -25,6 +25,8 @@ import java.security.PrivilegedException
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -63,6 +65,9 @@ import org.apache.hadoop.util.ToolRunner
* Base implemention for shims against secure Hadoop 0.20.3/0.23.
*/
public abstract class HadoopShimsSecure implements HadoopShims {
+
+ static final Log LOG = LogFactory.getLog(HadoopShimsSecure.class);
+
public boolean usesJobShell() {
return false;
}
@@ -520,6 +525,15 @@ public abstract class HadoopShimsSecure
}
@Override
+ public void closeAllForUGI(UserGroupInformation ugi) {
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException e) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, e);
+ }
+ }
+
+ @Override
abstract public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception;
@Override
Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java?rev=1384541&r1=1384540&r2=1384541&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java (original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java Thu Sep 13 21:21:01 2012
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUti
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -510,8 +511,9 @@ import org.apache.thrift.transport.TTran
}
Socket socket = ((TSocket)(saslTrans.getUnderlyingTransport())).getSocket();
remoteAddress.set(socket.getInetAddress());
+ UserGroupInformation clientUgi = null;
try {
- UserGroupInformation clientUgi = UserGroupInformation.createProxyUser(
+ clientUgi = UserGroupInformation.createProxyUser(
endUser, UserGroupInformation.getLoginUser());
return clientUgi.doAs(new PrivilegedExceptionAction<Boolean>() {
public Boolean run() {
@@ -532,6 +534,14 @@ import org.apache.thrift.transport.TTran
} catch (IOException ioe) {
throw new RuntimeException(ioe); // unexpected!
}
+ finally {
+ if (clientUgi != null) {
+ try { FileSystem.closeAllForUGI(clientUgi); }
+ catch(IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + clientUgi, exception);
+ }
+ }
+ }
}
}
Modified: hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1384541&r1=1384540&r2=1384541&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Thu Sep 13 21:21:01 2012
@@ -163,6 +163,9 @@ public interface HadoopShims {
* In secure versions of Hadoop, this simply returns the current
* access control context's user, ignoring the configuration.
*/
+
+ public void closeAllForUGI(UserGroupInformation ugi);
+
public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException, IOException;
/**