You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/19 07:28:11 UTC
[09/50] git commit: HDFS-6876. Archival Storage: support set/get
storage policy in DFSAdmin. Contributed by Jing Zhao.
HDFS-6876. Archival Storage: support set/get storage policy in DFSAdmin. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/185200e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/185200e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/185200e7
Branch: refs/heads/HDFS-6581
Commit: 185200e7096d15a5c2c2d59b7c7705362820aebf
Parents: e08701e
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Sep 4 20:14:46 2014 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Sep 4 20:14:46 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirectory.java | 9 +-
.../hdfs/server/namenode/FSNamesystem.java | 8 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 77 +++++++++-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 54 +++++++
.../hadoop/hdfs/TestSnapshotCommands.java | 152 ++++++-------------
.../hadoop/hdfs/TestStoragePolicyCommands.java | 81 ++++++++++
6 files changed, 263 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/185200e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 522d2a3..91c33e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -1487,10 +1487,12 @@ public class FSDirectory implements Closeable {
* @param src The string representation of the path to the file
* @param resolveLink whether to throw UnresolvedLinkException
* @param isRawPath true if a /.reserved/raw pathname was passed by the user
+ * @param includeStoragePolicy whether to include storage policy
* @return object containing information regarding the file
* or null if file not found
*/
- HdfsFileStatus getFileInfo(String src, boolean resolveLink, boolean isRawPath)
+ HdfsFileStatus getFileInfo(String src, boolean resolveLink,
+ boolean isRawPath, boolean includeStoragePolicy)
throws IOException {
String srcs = normalizePath(src);
readLock();
@@ -1500,9 +1502,10 @@ public class FSDirectory implements Closeable {
}
final INodesInPath inodesInPath = getLastINodeInPath(srcs, resolveLink);
final INode i = inodesInPath.getINode(0);
+ byte policyId = includeStoragePolicy && i != null ?
+ i.getStoragePolicyID() : BlockStoragePolicy.ID_UNSPECIFIED;
return i == null ? null : createFileStatus(HdfsFileStatus.EMPTY_NAME, i,
- BlockStoragePolicy.ID_UNSPECIFIED, inodesInPath.getPathSnapshotId(),
- isRawPath);
+ policyId, inodesInPath.getPathSnapshotId(), isRawPath);
} finally {
readUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/185200e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 86a08f7..e0d0158 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -334,7 +334,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
private HdfsFileStatus getAuditFileInfo(String path, boolean resolveSymlink)
throws IOException {
return (isAuditEnabled() && isExternalInvocation())
- ? dir.getFileInfo(path, resolveSymlink, false) : null;
+ ? dir.getFileInfo(path, resolveSymlink, false, false) : null;
}
private void logAuditEvent(boolean succeeded, String cmd, String src)
@@ -2546,7 +2546,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
overwrite, createParent, replication, blockSize, suite, edek,
logRetryCache);
stat = dir.getFileInfo(src, false,
- FSDirectory.isReservedRawName(srcArg));
+ FSDirectory.isReservedRawName(srcArg), false);
} catch (StandbyException se) {
skipSync = true;
throw se;
@@ -3970,12 +3970,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
try {
checkOperation(OperationCategory.READ);
src = resolvePath(src, pathComponents);
+ boolean isSuperUser = true;
if (isPermissionEnabled) {
checkPermission(pc, src, false, null, null, null, null, false,
resolveLink);
+ isSuperUser = pc.isSuperUser();
}
stat = dir.getFileInfo(src, resolveLink,
- FSDirectory.isReservedRawName(srcArg));
+ FSDirectory.isReservedRawName(srcArg), isSuperUser);
} catch (AccessControlException e) {
logAuditEvent(false, "getfileinfo", srcArg);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/185200e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index ad7be18..0764fdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.tools;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
@@ -43,6 +44,7 @@ import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.hdfs.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -58,23 +60,24 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.GenericRefreshProtocol;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
-import org.apache.hadoop.ipc.GenericRefreshProtocol;
import org.apache.hadoop.ipc.RefreshResponse;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
-import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -555,6 +558,32 @@ public class DFSAdmin extends FsShell {
return inSafeMode;
}
+ public int setStoragePolicy(String[] argv) throws IOException {
+ DistributedFileSystem dfs = getDFS();
+ dfs.setStoragePolicy(new Path(argv[1]), argv[2]);
+ System.out.println("Set storage policy " + argv[2] + " on " + argv[1]);
+ return 0;
+ }
+
+ public int getStoragePolicy(String[] argv) throws IOException {
+ DistributedFileSystem dfs = getDFS();
+ HdfsFileStatus status = dfs.getClient().getFileInfo(argv[1]);
+ if (status == null) {
+ throw new FileNotFoundException("File/Directory does not exist: "
+ + argv[1]);
+ }
+ byte storagePolicyId = status.getStoragePolicy();
+ BlockStoragePolicy.Suite suite = BlockStoragePolicy
+ .readBlockStorageSuite(getConf());
+ BlockStoragePolicy policy = suite.getPolicy(storagePolicyId);
+ if (policy != null) {
+ System.out.println("The storage policy of " + argv[1] + ":\n" + policy);
+ return 0;
+ } else {
+ throw new IOException("Cannot identify the storage policy for " + argv[1]);
+ }
+ }
+
/**
* Allow snapshot on a directory.
* Usage: java DFSAdmin -allowSnapshot snapshotDir
@@ -806,6 +835,8 @@ public class DFSAdmin extends FsShell {
"\t[-disallowSnapshot <snapshotDir>]\n" +
"\t[-shutdownDatanode <datanode_host:ipc_port> [upgrade]]\n" +
"\t[-getDatanodeInfo <datanode_host:ipc_port>\n" +
+ "\t[-setStoragePolicy path policyName\n" +
+ "\t[-getStoragePolicy path\n" +
"\t[-help [cmd]]\n";
String report ="-report [-live] [-dead] [-decommissioning]:\n" +
@@ -923,7 +954,13 @@ public class DFSAdmin extends FsShell {
String getDatanodeInfo = "-getDatanodeInfo <datanode_host:ipc_port>\n"
+ "\tGet the information about the given datanode. This command can\n"
+ "\tbe used for checking if a datanode is alive.\n";
-
+
+ String setStoragePolicy = "-setStoragePolicy path policyName\n"
+ + "\tSet the storage policy for a file/directory.\n";
+
+ String getStoragePolicy = "-getStoragePolicy path\n"
+ + "\tGet the storage policy for a file/directory.\n";
+
String help = "-help [cmd]: \tDisplays help for the given command or all commands if none\n" +
"\t\tis specified.\n";
@@ -981,6 +1018,10 @@ public class DFSAdmin extends FsShell {
System.out.println(shutdownDatanode);
} else if ("getDatanodeInfo".equalsIgnoreCase(cmd)) {
System.out.println(getDatanodeInfo);
+ } else if ("setStoragePolicy".equalsIgnoreCase(cmd)) {
+ System.out.println(setStoragePolicy);
+ } else if ("getStoragePolicy".equalsIgnoreCase(cmd)) {
+ System.out.println(getStoragePolicy);
} else if ("help".equals(cmd)) {
System.out.println(help);
} else {
@@ -1012,6 +1053,8 @@ public class DFSAdmin extends FsShell {
System.out.println(disallowSnapshot);
System.out.println(shutdownDatanode);
System.out.println(getDatanodeInfo);
+ System.out.println(setStoragePolicy);
+ System.out.println(getStoragePolicy);
System.out.println(help);
System.out.println();
ToolRunner.printGenericCommandUsage(System.out);
@@ -1371,6 +1414,12 @@ public class DFSAdmin extends FsShell {
} else if ("-safemode".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-safemode enter | leave | get | wait]");
+ } else if ("-setStoragePolicy".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-setStoragePolicy path policyName]");
+ } else if ("-getStoragePolicy".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-getStoragePolicy path]");
} else if ("-allowSnapshot".equalsIgnoreCase(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-allowSnapshot <snapshotDir>]");
@@ -1476,6 +1525,8 @@ public class DFSAdmin extends FsShell {
System.err.println(" [-fetchImage <local directory>]");
System.err.println(" [-shutdownDatanode <datanode_host:ipc_port> [upgrade]]");
System.err.println(" [-getDatanodeInfo <datanode_host:ipc_port>]");
+ System.err.println(" [-setStoragePolicy path policyName]");
+ System.err.println(" [-getStoragePolicy path]");
System.err.println(" [-help [cmd]]");
System.err.println();
ToolRunner.printGenericCommandUsage(System.err);
@@ -1607,6 +1658,16 @@ public class DFSAdmin extends FsShell {
printUsage(cmd);
return exitCode;
}
+ } else if ("-setStoragePolicy".equals(cmd)) {
+ if (argv.length != 3) {
+ printUsage(cmd);
+ return exitCode;
+ }
+ } else if ("-getStoragePolicy".equals(cmd)) {
+ if (argv.length != 2) {
+ printUsage(cmd);
+ return exitCode;
+ }
}
// initialize DFSAdmin
@@ -1678,6 +1739,10 @@ public class DFSAdmin extends FsShell {
exitCode = shutdownDatanode(argv, i);
} else if ("-getDatanodeInfo".equals(cmd)) {
exitCode = getDatanodeInfo(argv, i);
+ } else if ("-setStoragePolicy".equals(cmd)) {
+ exitCode = setStoragePolicy(argv);
+ } else if ("-getStoragePolicy".equals(cmd)) {
+ exitCode = getStoragePolicy(argv);
} else if ("-help".equals(cmd)) {
if (i < argv.length) {
printHelp(argv[i]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/185200e7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 777127e..174f973 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha
.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
@@ -73,6 +74,8 @@ import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.VersionInfo;
import org.junit.Assume;
@@ -1419,4 +1422,55 @@ public class DFSTestUtil {
}
return expectedPrimary.getDatanodeDescriptor();
}
+
+ public static void toolRun(Tool tool, String cmd, int retcode, String contain)
+ throws Exception {
+ String [] cmds = StringUtils.split(cmd, ' ');
+ System.out.flush();
+ System.err.flush();
+ PrintStream origOut = System.out;
+ PrintStream origErr = System.err;
+ String output = null;
+ int ret = 0;
+ try {
+ ByteArrayOutputStream bs = new ByteArrayOutputStream(1024);
+ PrintStream out = new PrintStream(bs);
+ System.setOut(out);
+ System.setErr(out);
+ ret = tool.run(cmds);
+ System.out.flush();
+ System.err.flush();
+ out.close();
+ output = bs.toString();
+ } finally {
+ System.setOut(origOut);
+ System.setErr(origErr);
+ }
+ System.out.println("Output for command: " + cmd + " retcode: " + ret);
+ if (output != null) {
+ System.out.println(output);
+ }
+ assertEquals(retcode, ret);
+ if (contain != null) {
+ assertTrue("The real output is: " + output + ".\n It should contain: "
+ + contain, output.contains(contain));
+ }
+ }
+
+ public static void FsShellRun(String cmd, int retcode, String contain,
+ Configuration conf) throws Exception {
+ FsShell shell = new FsShell(new Configuration(conf));
+ toolRun(shell, cmd, retcode, contain);
+ }
+
+ public static void DFSAdminRun(String cmd, int retcode, String contain,
+ Configuration conf) throws Exception {
+ DFSAdmin admin = new DFSAdmin(new Configuration(conf));
+ toolRun(admin, cmd, retcode, contain);
+ }
+
+ public static void FsShellRun(String cmd, Configuration conf)
+ throws Exception {
+ FsShellRun(cmd, 0, null, conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/185200e7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
index e2db636..eec4e99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
@@ -18,22 +18,11 @@
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.*;
-
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.PrintStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.tools.DFSAdmin;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Tool;
-
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -89,136 +78,87 @@ public class TestSnapshotCommands {
}
}
- private void toolRun(Tool tool, String cmd, int retcode, String contain)
- throws Exception {
- String [] cmds = StringUtils.split(cmd, ' ');
- System.out.flush();
- System.err.flush();
- PrintStream origOut = System.out;
- PrintStream origErr = System.err;
- String output = null;
- int ret = 0;
- try {
- ByteArrayOutputStream bs = new ByteArrayOutputStream(1024);
- PrintStream out = new PrintStream(bs);
- System.setOut(out);
- System.setErr(out);
- ret = tool.run(cmds);
- System.out.flush();
- System.err.flush();
- out.close();
- output = bs.toString();
- } finally {
- System.setOut(origOut);
- System.setErr(origErr);
- }
- System.out.println("Output for command: " + cmd + " retcode: " + ret);
- if (output != null) {
- System.out.println(output);
- }
- assertEquals(retcode, ret);
- if (contain != null) {
- assertTrue(output.contains(contain));
- }
- }
-
- private void FsShellRun(String cmd, int retcode, String contain)
- throws Exception {
- FsShell shell = new FsShell(new Configuration(conf));
- toolRun(shell, cmd, retcode, contain);
- }
-
- private void DFSAdminRun(String cmd, int retcode, String contain)
- throws Exception {
- DFSAdmin admin = new DFSAdmin(new Configuration(conf));
- toolRun(admin, cmd, retcode, contain);
- }
-
- private void FsShellRun(String cmd) throws Exception {
- FsShellRun(cmd, 0, null);
- }
-
@Test
public void testAllowSnapshot() throws Exception {
// Idempotent test
- DFSAdminRun("-allowSnapshot /sub1", 0, "Allowing snaphot on /sub1 succeeded");
+ DFSTestUtil.DFSAdminRun("-allowSnapshot /sub1", 0, "Allowing snaphot on /sub1 succeeded", conf);
// allow normal dir success
- FsShellRun("-mkdir /sub2");
- DFSAdminRun("-allowSnapshot /sub2", 0, "Allowing snaphot on /sub2 succeeded");
+ DFSTestUtil.FsShellRun("-mkdir /sub2", conf);
+ DFSTestUtil.DFSAdminRun("-allowSnapshot /sub2", 0, "Allowing snaphot on /sub2 succeeded", conf);
// allow non-exists dir failed
- DFSAdminRun("-allowSnapshot /sub3", -1, null);
+ DFSTestUtil.DFSAdminRun("-allowSnapshot /sub3", -1, null, conf);
}
@Test
public void testCreateSnapshot() throws Exception {
// test createSnapshot
- FsShellRun("-createSnapshot /sub1 sn0", 0, "Created snapshot /sub1/.snapshot/sn0");
- FsShellRun("-createSnapshot /sub1 sn0", 1, "there is already a snapshot with the same name \"sn0\"");
- FsShellRun("-rmr /sub1/sub1sub2");
- FsShellRun("-mkdir /sub1/sub1sub3");
- FsShellRun("-createSnapshot /sub1 sn1", 0, "Created snapshot /sub1/.snapshot/sn1");
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn0", 0, "Created snapshot /sub1/.snapshot/sn0", conf);
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn0", 1, "there is already a snapshot with the same name \"sn0\"", conf);
+ DFSTestUtil.FsShellRun("-rmr /sub1/sub1sub2", conf);
+ DFSTestUtil.FsShellRun("-mkdir /sub1/sub1sub3", conf);
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn1", 0, "Created snapshot /sub1/.snapshot/sn1", conf);
// check snapshot contents
- FsShellRun("-ls /sub1", 0, "/sub1/sub1sub1");
- FsShellRun("-ls /sub1", 0, "/sub1/sub1sub3");
- FsShellRun("-ls /sub1/.snapshot", 0, "/sub1/.snapshot/sn0");
- FsShellRun("-ls /sub1/.snapshot", 0, "/sub1/.snapshot/sn1");
- FsShellRun("-ls /sub1/.snapshot/sn0", 0, "/sub1/.snapshot/sn0/sub1sub1");
- FsShellRun("-ls /sub1/.snapshot/sn0", 0, "/sub1/.snapshot/sn0/sub1sub2");
- FsShellRun("-ls /sub1/.snapshot/sn1", 0, "/sub1/.snapshot/sn1/sub1sub1");
- FsShellRun("-ls /sub1/.snapshot/sn1", 0, "/sub1/.snapshot/sn1/sub1sub3");
+ DFSTestUtil.FsShellRun("-ls /sub1", 0, "/sub1/sub1sub1", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1", 0, "/sub1/sub1sub3", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot", 0, "/sub1/.snapshot/sn0", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot", 0, "/sub1/.snapshot/sn1", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot/sn0", 0, "/sub1/.snapshot/sn0/sub1sub1", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot/sn0", 0, "/sub1/.snapshot/sn0/sub1sub2", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot/sn1", 0, "/sub1/.snapshot/sn1/sub1sub1", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot/sn1", 0, "/sub1/.snapshot/sn1/sub1sub3", conf);
}
@Test
public void testMkdirUsingReservedName() throws Exception {
// test can not create dir with reserved name: .snapshot
- FsShellRun("-ls /");
- FsShellRun("-mkdir /.snapshot", 1, "File exists");
- FsShellRun("-mkdir /sub1/.snapshot", 1, "File exists");
+ DFSTestUtil.FsShellRun("-ls /", conf);
+ DFSTestUtil.FsShellRun("-mkdir /.snapshot", 1, "File exists", conf);
+ DFSTestUtil.FsShellRun("-mkdir /sub1/.snapshot", 1, "File exists", conf);
// mkdir -p ignore reserved name check if dir already exists
- FsShellRun("-mkdir -p /sub1/.snapshot");
- FsShellRun("-mkdir -p /sub1/sub1sub1/.snapshot", 1, "mkdir: \".snapshot\" is a reserved name.");
+ DFSTestUtil.FsShellRun("-mkdir -p /sub1/.snapshot", conf);
+ DFSTestUtil.FsShellRun("-mkdir -p /sub1/sub1sub1/.snapshot", 1, "mkdir: \".snapshot\" is a reserved name.", conf);
}
@Test
public void testRenameSnapshot() throws Exception {
- FsShellRun("-createSnapshot /sub1 sn.orig");
- FsShellRun("-renameSnapshot /sub1 sn.orig sn.rename");
- FsShellRun("-ls /sub1/.snapshot", 0, "/sub1/.snapshot/sn.rename");
- FsShellRun("-ls /sub1/.snapshot/sn.rename", 0, "/sub1/.snapshot/sn.rename/sub1sub1");
- FsShellRun("-ls /sub1/.snapshot/sn.rename", 0, "/sub1/.snapshot/sn.rename/sub1sub2");
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn.orig", conf);
+ DFSTestUtil.FsShellRun("-renameSnapshot /sub1 sn.orig sn.rename", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot", 0, "/sub1/.snapshot/sn.rename", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot/sn.rename", 0, "/sub1/.snapshot/sn.rename/sub1sub1", conf);
+ DFSTestUtil.FsShellRun("-ls /sub1/.snapshot/sn.rename", 0, "/sub1/.snapshot/sn.rename/sub1sub2", conf);
//try renaming from a non-existing snapshot
- FsShellRun("-renameSnapshot /sub1 sn.nonexist sn.rename", 1,
- "renameSnapshot: The snapshot sn.nonexist does not exist for directory /sub1");
+ DFSTestUtil.FsShellRun("-renameSnapshot /sub1 sn.nonexist sn.rename", 1,
+ "renameSnapshot: The snapshot sn.nonexist does not exist for directory /sub1", conf);
//try renaming to existing snapshots
- FsShellRun("-createSnapshot /sub1 sn.new");
- FsShellRun("-renameSnapshot /sub1 sn.new sn.rename", 1,
- "renameSnapshot: The snapshot sn.rename already exists for directory /sub1");
- FsShellRun("-renameSnapshot /sub1 sn.rename sn.new", 1,
- "renameSnapshot: The snapshot sn.new already exists for directory /sub1");
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn.new", conf);
+ DFSTestUtil.FsShellRun("-renameSnapshot /sub1 sn.new sn.rename", 1,
+ "renameSnapshot: The snapshot sn.rename already exists for directory /sub1", conf);
+ DFSTestUtil.FsShellRun("-renameSnapshot /sub1 sn.rename sn.new", 1,
+ "renameSnapshot: The snapshot sn.new already exists for directory /sub1", conf);
}
@Test
public void testDeleteSnapshot() throws Exception {
- FsShellRun("-createSnapshot /sub1 sn1");
- FsShellRun("-deleteSnapshot /sub1 sn1");
- FsShellRun("-deleteSnapshot /sub1 sn1", 1,
- "deleteSnapshot: Cannot delete snapshot sn1 from path /sub1: the snapshot does not exist.");
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn1", conf);
+ DFSTestUtil.FsShellRun("-deleteSnapshot /sub1 sn1", conf);
+ DFSTestUtil.FsShellRun("-deleteSnapshot /sub1 sn1", 1,
+ "deleteSnapshot: Cannot delete snapshot sn1 from path /sub1: the snapshot does not exist.", conf);
}
@Test
public void testDisallowSnapshot() throws Exception {
- FsShellRun("-createSnapshot /sub1 sn1");
+ DFSTestUtil.FsShellRun("-createSnapshot /sub1 sn1", conf);
// cannot delete snapshotable dir
- FsShellRun("-rmr /sub1", 1, "The directory /sub1 cannot be deleted since /sub1 is snapshottable and already has snapshots");
- DFSAdminRun("-disallowSnapshot /sub1", -1,
- "disallowSnapshot: The directory /sub1 has snapshot(s). Please redo the operation after removing all the snapshots.");
- FsShellRun("-deleteSnapshot /sub1 sn1");
- DFSAdminRun("-disallowSnapshot /sub1", 0, "Disallowing snaphot on /sub1 succeeded");
+ DFSTestUtil.FsShellRun("-rmr /sub1", 1, "The directory /sub1 cannot be deleted since /sub1 is snapshottable and already has snapshots", conf);
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", -1,
+ "disallowSnapshot: The directory /sub1 has snapshot(s). Please redo the operation after removing all the snapshots.", conf);
+ DFSTestUtil.FsShellRun("-deleteSnapshot /sub1 sn1", conf);
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", 0, "Disallowing snaphot on /sub1 succeeded", conf);
// Idempotent test
- DFSAdminRun("-disallowSnapshot /sub1", 0, "Disallowing snaphot on /sub1 succeeded");
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", 0, "Disallowing snaphot on /sub1 succeeded", conf);
// now it can be deleted
- FsShellRun("-rmr /sub1");
+ DFSTestUtil.FsShellRun("-rmr /sub1", conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/185200e7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java
new file mode 100644
index 0000000..d6ead09
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test storage policy related DFSAdmin commands
+ */
+public class TestStoragePolicyCommands {
+ private static final short REPL = 1;
+ private static final int SIZE = 128;
+
+ private static Configuration conf;
+ private static MiniDFSCluster cluster;
+ private static DistributedFileSystem fs;
+
+ @Before
+ public void clusterSetUp() throws IOException {
+ conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void clusterShutdown() throws IOException{
+ if(fs != null){
+ fs.close();
+ }
+ if(cluster != null){
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testSetAndGetStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ DFSTestUtil.createFile(fs, bar, SIZE, REPL, 0);
+
+ DFSTestUtil.DFSAdminRun("-setStoragePolicy /foo WARM", 0,
+ "Set storage policy WARM on " + foo.toString(), conf);
+ DFSTestUtil.DFSAdminRun("-setStoragePolicy /foo/bar COLD", 0,
+ "Set storage policy COLD on " + bar.toString(), conf);
+ DFSTestUtil.DFSAdminRun("-setStoragePolicy /fooz WARM", -1,
+ "File/Directory does not exist: /fooz", conf);
+
+ final BlockStoragePolicy.Suite suite = BlockStoragePolicy
+ .readBlockStorageSuite(conf);
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ DFSTestUtil.DFSAdminRun("-getStoragePolicy /foo", 0,
+ "The storage policy of " + foo.toString() + ":\n" + warm, conf);
+ DFSTestUtil.DFSAdminRun("-getStoragePolicy /foo/bar", 0,
+ "The storage policy of " + bar.toString() + ":\n" + cold, conf);
+ DFSTestUtil.DFSAdminRun("-getStoragePolicy /fooz", -1,
+ "File/Directory does not exist: /fooz", conf);
+ }
+}