You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/09/04 04:33:44 UTC
git commit: HDFS-6959 Make the HDFS home directory location
customizable. Contributed by Yongjun Zhang (cherry picked from commit
f4caedfcbfeae7e2fe7c0e812ddbb087608a5ffd)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 69ee11dc9 -> 23c41897c
HDFS-6959 Make the HDFS home directory location customizable. Contributed by Yongjun Zhang
(cherry picked from commit f4caedfcbfeae7e2fe7c0e812ddbb087608a5ffd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23c41897
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23c41897
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23c41897
Branch: refs/heads/branch-2
Commit: 23c41897cbe4e6abafc03a7a404953bbdb58ba65
Parents: 69ee11d
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Sep 3 19:30:30 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Sep 3 19:33:30 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 ++
.../hadoop/hdfs/DistributedFileSystem.java | 11 ++++++--
.../src/main/resources/hdfs-default.xml | 8 ++++++
.../org/apache/hadoop/hdfs/TestLocalDFS.java | 29 +++++++++++++++++++-
5 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c41897/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0cace3f..125c47a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -176,6 +176,9 @@ Release 2.6.0 - UNRELEASED
HDFS-4257. The ReplaceDatanodeOnFailure policies could have a forgiving
option (szetszwo via cmccabe)
+ HDFS-6959. Make the HDFS home directory location customizable. (yzhang via
+ cmccabe)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c41897/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bf7f5ba..4c244b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -41,6 +41,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_STREAM_BUFFER_SIZE_DEFAULT = 4096;
public static final String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum";
public static final int DFS_BYTES_PER_CHECKSUM_DEFAULT = 512;
+ public static final String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix";
+ public static final String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user";
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY = "dfs.client.retry.policy.enabled";
public static final boolean DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = false;
public static final String DFS_CLIENT_RETRY_POLICY_SPEC_KEY = "dfs.client.retry.policy.spec";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c41897/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index af7d307..6c04f01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -102,6 +102,8 @@ import com.google.common.base.Preconditions;
public class DistributedFileSystem extends FileSystem {
private Path workingDir;
private URI uri;
+ private String homeDirPrefix =
+ DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT;
DFSClient dfs;
private boolean verifyChecksum = true;
@@ -136,7 +138,10 @@ public class DistributedFileSystem extends FileSystem {
if (host == null) {
throw new IOException("Incomplete HDFS URI, no host: "+ uri);
}
-
+ homeDirPrefix = conf.get(
+ DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY,
+ DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT);
+
this.dfs = new DFSClient(uri, conf, statistics);
this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority());
this.workingDir = getHomeDirectory();
@@ -167,10 +172,10 @@ public class DistributedFileSystem extends FileSystem {
workingDir = fixRelativePart(dir);
}
-
@Override
public Path getHomeDirectory() {
- return makeQualified(new Path("/user/" + dfs.ugi.getShortUserName()));
+ return makeQualified(new Path(homeDirPrefix + "/"
+ + dfs.ugi.getShortUserName()));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c41897/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 76336eb..8bcca8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2090,4 +2090,12 @@
</description>
</property>
+<property>
+ <name>dfs.user.home.dir.prefix</name>
+ <value>/user</value>
+ <description>The directory to prepend to user name to get the user's
+ home direcotry.
+ </description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c41897/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java
index fb90ad2..5385046 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java
@@ -90,7 +90,9 @@ public class TestLocalDFS {
// test home directory
Path home =
- fileSys.makeQualified(new Path("/user/" + getUserName(fileSys)));
+ fileSys.makeQualified(
+ new Path(DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT
+ + "/" + getUserName(fileSys)));
Path fsHome = fileSys.getHomeDirectory();
assertEquals(home, fsHome);
@@ -99,4 +101,29 @@ public class TestLocalDFS {
cluster.shutdown();
}
}
+
+ /**
+ * Tests get/set working directory in DFS.
+ */
+ @Test(timeout=30000)
+ public void testHomeDirectory() throws IOException {
+ final String[] homeBases = new String[] {"/home", "/home/user"};
+ Configuration conf = new HdfsConfiguration();
+ for (final String homeBase : homeBases) {
+ conf.set(DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, homeBase);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fileSys = cluster.getFileSystem();
+ try {
+ // test home directory
+ Path home =
+ fileSys.makeQualified(
+ new Path(homeBase + "/" + getUserName(fileSys)));
+ Path fsHome = fileSys.getHomeDirectory();
+ assertEquals(home, fsHome);
+ } finally {
+ fileSys.close();
+ cluster.shutdown();
+ }
+ }
+ }
}