You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2019/07/11 02:16:08 UTC
[hadoop] branch branch-2.9 updated: HDFS-12748. NameNode memory
leak when accessing webhdfs GETHOMEDIRECTORY. Contributed by Weiwei Yang.
This is an automated email from the ASF dual-hosted git repository.
wwei pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.9 by this push:
new 330e5c0 HDFS-12748. NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY. Contributed by Weiwei Yang.
330e5c0 is described below
commit 330e5c00fe1a13db8dc7fea2971129fc77a90a0d
Author: Weiwei Yang <ww...@apache.org>
AuthorDate: Thu Jul 11 09:53:28 2019 +0800
HDFS-12748. NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY. Contributed by Weiwei Yang.
(cherry picked from commit 6c8cfd135eea15e3dfd76dd33af24a9a8c38fd8d)
(cherry picked from commit 3f112e7347fa7b75eab3a417c9d6520ca8b3053e)
---
.../java/org/apache/hadoop/hdfs/DFSUtilClient.java | 21 +++++++++++++++++++++
.../apache/hadoop/hdfs/DistributedFileSystem.java | 8 +-------
.../web/resources/NamenodeWebHdfsMethods.java | 6 +++---
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index 311494e..15c9b84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -682,4 +682,25 @@ public class DFSUtilClient {
iioe.initCause(e);
return iioe;
}
+
+ /**
+ * Returns current user home directory under a home directory prefix.
+ * The home directory prefix can be defined by
+ * {@link HdfsClientConfigKeys#DFS_USER_HOME_DIR_PREFIX_KEY}.
+ * User info is obtained from given {@link UserGroupInformation}.
+ * @param conf configuration
+ * @param ugi {@link UserGroupInformation} of current user.
+ * @return the home directory of current user.
+ */
+ public static Path getHomeDirectory(Configuration conf,
+ UserGroupInformation ugi) {
+ String userHomePrefix = HdfsClientConfigKeys
+ .DFS_USER_HOME_DIR_PREFIX_DEFAULT;
+ if (conf != null) {
+ userHomePrefix = conf.get(
+ HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY,
+ HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT);
+ }
+ return new Path(userHomePrefix + "/" + ugi.getShortUserName());
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 46df7d5..d40cfff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -119,8 +119,6 @@ public class DistributedFileSystem extends FileSystem
implements KeyProviderTokenIssuer {
private Path workingDir;
private URI uri;
- private String homeDirPrefix =
- HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT;
DFSClient dfs;
private boolean verifyChecksum = true;
@@ -157,9 +155,6 @@ public class DistributedFileSystem extends FileSystem
if (host == null) {
throw new IOException("Incomplete HDFS URI, no host: "+ uri);
}
- homeDirPrefix = conf.get(
- HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY,
- HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT);
this.dfs = new DFSClient(uri, conf, statistics);
this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority());
@@ -202,8 +197,7 @@ public class DistributedFileSystem extends FileSystem
@Override
public Path getHomeDirectory() {
- return makeQualified(new Path(homeDirPrefix + "/"
- + dfs.ugi.getShortUserName()));
+ return makeQualified(DFSUtilClient.getHomeDirectory(getConf(), dfs.ugi));
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 6e8e325..416e3ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -1093,9 +1094,8 @@ public class NamenodeWebHdfsMethods {
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case GETHOMEDIRECTORY: {
- final String js = JsonUtil.toJsonString("Path",
- FileSystem.get(conf != null ? conf : new Configuration())
- .getHomeDirectory().toUri().getPath());
+ String userHome = DFSUtilClient.getHomeDirectory(conf, ugi).toString();
+ final String js = JsonUtil.toJsonString("Path", userHome);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case GETACLSTATUS: {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org