You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/29 05:44:27 UTC
[1/3] git commit: HDFS-7156. Update fsck documentation. Contributed
by Masahiro Yamaguch.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 9cbb2c55d -> 9a53c3699
HDFS-7156. Update fsck documentation. Contributed by Masahiro Yamaguch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/400e1bb4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/400e1bb4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/400e1bb4
Branch: refs/heads/HDFS-6581
Commit: 400e1bb4eff78333005144c58d33f10d4697ff70
Parents: 7f300bc
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Sat Sep 27 21:04:23 2014 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sat Sep 27 21:04:23 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/tools/DFSck.java | 3 +-
.../src/site/apt/HDFSCommands.apt.vm | 56 +++++++++++---------
3 files changed, 36 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/400e1bb4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0524cf0..a0e176a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -512,6 +512,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6956. Allow dynamically changing the tracing level in Hadoop servers
(cmccabe)
+ HDFS-7156. Update fsck documentation. (Masahiro Yamaguch via shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/400e1bb4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index b91090d..31ab596 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -77,7 +77,8 @@ public class DFSck extends Configured implements Tool {
private static final String USAGE = "Usage: DFSck <path> "
+ "[-list-corruptfileblocks | "
+ "[-move | -delete | -openforwrite] "
- + "[-files [-blocks [-locations | -racks]]]] [-showprogress]\n"
+ + "[-files [-blocks [-locations | -racks]]]] "
+ + "[-includeSnapshots] [-showprogress]\n"
+ "\t<path>\tstart checking from this path\n"
+ "\t-move\tmove corrupted files to /lost+found\n"
+ "\t-delete\tdelete corrupted files\n"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/400e1bb4/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 170f352..121f0ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -82,32 +82,40 @@ HDFS Commands Guide
See {{{./HdfsUserGuide.html#fsck}fsck}} for more info.
Usage: <<<hdfs fsck [GENERIC_OPTIONS] <path>
+ [-list-corruptfileblocks |
[-move | -delete | -openforwrite]
[-files [-blocks [-locations | -racks]]]
- [-showprogress]>>>
-
-*------------------+---------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------+---------------------------------------------+
-| <path> | Start checking from this path.
-*------------------+---------------------------------------------+
-| -move | Move corrupted files to /lost+found
-*------------------+---------------------------------------------+
-| -delete | Delete corrupted files.
-*------------------+---------------------------------------------+
-| -openforwrite | Print out files opened for write.
-*------------------+---------------------------------------------+
-| -files | Print out files being checked.
-*------------------+---------------------------------------------+
-| -blocks | Print out block report.
-*------------------+---------------------------------------------+
-| -locations | Print out locations for every block.
-*------------------+---------------------------------------------+
-| -racks | Print out network topology for data-node locations.
-*------------------+---------------------------------------------+
-| -showprogress | Print out dots for progress in output. Default is OFF
-| | (no progress).
-*------------------+---------------------------------------------+
+ [-includeSnapshots] [-showprogress]>>>
+
+*------------------------+---------------------------------------------+
+|| COMMAND_OPTION || Description
+*------------------------+---------------------------------------------+
+| <path> | Start checking from this path.
+*------------------------+---------------------------------------------+
+| -move | Move corrupted files to /lost+found.
+*------------------------+---------------------------------------------+
+| -delete | Delete corrupted files.
+*------------------------+---------------------------------------------+
+| -files | Print out files being checked.
+*------------------------+---------------------------------------------+
+| -openforwrite | Print out files opened for write.
+*------------------------+---------------------------------------------+
+| | Include snapshot data if the given path
+| -includeSnapshots | indicates a snapshottable directory or
+| | there are snapshottable directories under it.
+*------------------------+---------------------------------------------+
+| -list-corruptfileblocks| Print out list of missing blocks and
+| | files they belong to.
+*------------------------+---------------------------------------------+
+| -blocks | Print out block report.
+*------------------------+---------------------------------------------+
+| -locations | Print out locations for every block.
+*------------------------+---------------------------------------------+
+| -racks | Print out network topology for data-node locations.
+*------------------------+---------------------------------------------+
+| -showprogress | Print out dots for progress in output. Default is OFF
+| | (no progress).
+*------------------------+---------------------------------------------+
* Administration Commands
[3/3] git commit: Merge branch 'trunk' into HDFS-6581
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-6581
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a53c369
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a53c369
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a53c369
Branch: refs/heads/HDFS-6581
Commit: 9a53c3699b52a0bbeb0f723ac409ac189aa00f7f
Parents: 9cbb2c5 b38e52b
Author: arp <ar...@apache.org>
Authored: Sun Sep 28 20:42:56 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Sun Sep 28 20:42:56 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 28 +++++++++-
.../org/apache/hadoop/hdfs/tools/DFSck.java | 3 +-
.../src/main/resources/hdfs-default.xml | 8 +++
.../src/site/apt/HDFSCommands.apt.vm | 56 +++++++++++---------
.../hadoop/hdfs/TestBlockStoragePolicy.java | 35 ++++++++++++
7 files changed, 110 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a53c369/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a53c369/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a53c369/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
[2/3] git commit: HDFS-7093. Add config key to restrict
setStoragePolicy. (Arpit Agarwal)
Posted by ar...@apache.org.
HDFS-7093. Add config key to restrict setStoragePolicy. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b38e52b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b38e52b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b38e52b5
Branch: refs/heads/HDFS-6581
Commit: b38e52b5e8f2b2ad91c2c8c4eabc232ee4753ca0
Parents: 400e1bb
Author: arp <ar...@apache.org>
Authored: Sun Sep 28 19:28:51 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Sun Sep 28 19:28:51 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 28 ++++++++++++++--
.../src/main/resources/hdfs-default.xml | 8 +++++
.../hadoop/hdfs/TestBlockStoragePolicy.java | 35 ++++++++++++++++++++
5 files changed, 74 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a0e176a..5e1d717 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -514,6 +514,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7156. Update fsck documentation. (Masahiro Yamaguch via shv)
+ HDFS-7093. Add config key to restrict setStoragePolicy. (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 3aa9acf..b016750 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -558,6 +558,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path";
public static final String DFS_DOMAIN_SOCKET_PATH_DEFAULT = "";
+ public static final String DFS_STORAGE_POLICY_ENABLED_KEY = "dfs.storage.policy.enabled";
+ public static final boolean DFS_STORAGE_POLICY_ENABLED_DEFAULT = true;
+
// HA related configuration
public static final String DFS_HA_NAMENODES_KEY_PREFIX = "dfs.ha.namenodes";
public static final String DFS_HA_NAMENODE_ID_KEY = "dfs.ha.namenode.id";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4ae47b7..9b4380c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -84,6 +84,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROU
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
import static org.apache.hadoop.util.Time.now;
@@ -420,6 +422,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
private final CacheManager cacheManager;
private final DatanodeStatistics datanodeStatistics;
+ // whether setStoragePolicy is allowed.
+ private final boolean isStoragePolicyEnabled;
+
private String nameserviceId;
private RollingUpgradeInfo rollingUpgradeInfo = null;
@@ -781,6 +786,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics();
this.blockIdGenerator = new SequentialBlockIdGenerator(this.blockManager);
+ this.isStoragePolicyEnabled =
+ conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
+ DFS_STORAGE_POLICY_ENABLED_DEFAULT);
+
this.fsOwner = UserGroupInformation.getCurrentUser();
this.fsOwnerShortUserName = fsOwner.getShortUserName();
this.supergroup = conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
@@ -2273,8 +2282,17 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
private void setStoragePolicyInt(String src, final String policyName)
- throws IOException {
- checkSuperuserPrivilege();
+ throws IOException, UnresolvedLinkException, AccessControlException {
+
+ if (!isStoragePolicyEnabled) {
+ throw new IOException("Failed to set storage policy since "
+ + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
+ }
+ FSPermissionChecker pc = null;
+ if (isPermissionEnabled) {
+ pc = getPermissionChecker();
+ }
+
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
waitForLoadingFSImage();
@@ -2283,6 +2301,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set storage policy for " + src);
+
+ if (pc != null) {
+ checkPermission(pc, src, false, null, null, FsAction.WRITE, null,
+ false, true);
+ }
+
src = FSDirectory.resolvePath(src, pathComponents, dir);
// get the corresponding policy and make sure the policy name is valid
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 2d4109a..23b25fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2132,4 +2132,12 @@
</description>
</property>
+<property>
+ <name>dfs.storage.policy.enabled</name>
+ <value>true</value>
+ <description>
+ Allow users to change the storage policy on files and directories.
+ </description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 39d1439..771b7bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySu
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.util.*;
import com.google.common.collect.Lists;
@@ -69,6 +70,40 @@ public class TestBlockStoragePolicy {
static final byte WARM = (byte) 8;
static final byte HOT = (byte) 12;
+
+ @Test (timeout=300000)
+ public void testConfigKeyEnabled() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY, true);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(1).build();
+ try {
+ cluster.waitActive();
+ cluster.getFileSystem().setStoragePolicy(new Path("/"), "COLD");
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Ensure that setStoragePolicy throws IOException when
+ * dfs.storage.policy.enabled is set to false.
+ * @throws IOException
+ */
+ @Test (timeout=300000, expected=IOException.class)
+ public void testConfigKeyDisabled() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY, false);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(1).build();
+ try {
+ cluster.waitActive();
+ cluster.getFileSystem().setStoragePolicy(new Path("/"), "COLD");
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
@Test
public void testDefaultPolicies() {
final Map<Byte, String> expectedPolicyStrings = new HashMap<Byte, String>();