You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/07/25 16:53:26 UTC
[1/3] hadoop git commit: HDFS-10660. Expose storage policy apis via
HDFSAdmin interface. (Contributed by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 71c8bdd50 -> 00cbd3136
HDFS-10660. Expose storage policy apis via HDFSAdmin interface. (Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00cbd313
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00cbd313
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00cbd313
Branch: refs/heads/branch-2.8
Commit: 00cbd3136e057b3b5096581e3f8189f4544c73a2
Parents: 54be36a
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Jul 22 11:38:33 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jul 25 09:51:59 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 35 +++++++++
.../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 82 +++++++++++++++++++-
2 files changed, 115 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00cbd313/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index c92e99f..2ee7eea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -20,12 +20,14 @@ package org.apache.hadoop.hdfs.client;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
+import java.util.Collection;
import java.util.EnumSet;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -412,6 +414,39 @@ public class HdfsAdmin {
dfs.setStoragePolicy(src, policyName);
}
+ /**
+ * Unset the storage policy set for a given file or directory.
+ *
+ * @param src file or directory path.
+ * @throws IOException
+ */
+ public void unsetStoragePolicy(final Path src) throws IOException {
+ dfs.unsetStoragePolicy(src);
+ }
+
+ /**
+ * Query the effective storage policy ID for the given file or directory.
+ *
+ * @param src file or directory path.
+ * @return storage policy for the given file or directory.
+ * @throws IOException
+ */
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ return dfs.getStoragePolicy(src);
+ }
+
+ /**
+ * Retrieve all the storage policies supported by HDFS file system.
+ *
+ * @return all storage policies supported by HDFS file system.
+ * @throws IOException
+ */
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return dfs.getAllStoragePolicies();
+ }
+
private void provisionEZTrash(Path path) throws IOException {
// make sure the path is an EZ
EncryptionZone ez = dfs.getEZForPath(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00cbd313/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
index 0f5bdf5..717d79e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
@@ -23,24 +23,35 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.collect.Sets;
+
public class TestHdfsAdmin {
private static final Path TEST_PATH = new Path("/test");
+ private static final short REPL = 1;
+ private static final int SIZE = 128;
private final Configuration conf = new Configuration();
private MiniDFSCluster cluster;
-
+
@Before
public void setUpCluster() throws IOException {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ cluster.waitActive();
}
@After
@@ -94,4 +105,71 @@ public class TestHdfsAdmin {
public void testHdfsAdminWithBadUri() throws IOException, URISyntaxException {
new HdfsAdmin(new URI("file:///bad-scheme"), conf);
}
+
+ /**
+ * Test that we can set, get, unset storage policies via {@link HdfsAdmin}.
+ */
+ @Test
+ public void testHdfsAdminStoragePolicies() throws Exception {
+ HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ FileSystem fs = FileSystem.get(conf);
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: set storage policy
+ */
+ hdfsAdmin.setStoragePolicy(foo, warm.getName());
+ hdfsAdmin.setStoragePolicy(bar, cold.getName());
+ hdfsAdmin.setStoragePolicy(wow, hot.getName());
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(hdfsAdmin.getStoragePolicy(foo), warm);
+ assertEquals(hdfsAdmin.getStoragePolicy(bar), cold);
+ assertEquals(hdfsAdmin.getStoragePolicy(wow), hot);
+
+ /*
+ * test: unset storage policy
+ */
+ hdfsAdmin.unsetStoragePolicy(foo);
+ hdfsAdmin.unsetStoragePolicy(bar);
+ hdfsAdmin.unsetStoragePolicy(wow);
+
+ /*
+ * test: get storage policy after unset. HOT by default.
+ */
+ assertEquals(hdfsAdmin.getStoragePolicy(foo), hot);
+ assertEquals(hdfsAdmin.getStoragePolicy(bar), hot);
+ assertEquals(hdfsAdmin.getStoragePolicy(wow), hot);
+
+ /*
+ * test: get all storage policies
+ */
+ // Get policies via HdfsAdmin
+ Set<String> policyNamesSet1 = new HashSet<>();
+ for (BlockStoragePolicySpi policy : hdfsAdmin.getAllStoragePolicies()) {
+ policyNamesSet1.add(policy.getName());
+ }
+
+ // Get policies via BlockStoragePolicySuite
+ Set<String> policyNamesSet2 = new HashSet<>();
+ for (BlockStoragePolicy policy : suite.getAllPolicies()) {
+ policyNamesSet2.add(policy.getName());
+ }
+ // Ensure that we got the same set of policies in both cases.
+ Assert.assertTrue(
+ Sets.difference(policyNamesSet1, policyNamesSet2).isEmpty());
+ Assert.assertTrue(
+ Sets.difference(policyNamesSet2, policyNamesSet1).isEmpty());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HDFS-9534. Add CLI command to clear storage
policy from a path. (Contributed by Xiaobing Zhou)
Posted by ar...@apache.org.
HDFS-9534. Add CLI command to clear storage policy from a path. (Contributed by Xiaobing Zhou)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbf711c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbf711c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbf711c0
Branch: refs/heads/branch-2.8
Commit: bbf711c030cb7ce063bc0a4e35f8f996de3e4622
Parents: 71c8bdd
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Mar 2 18:35:28 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jul 25 09:51:59 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 10 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++
.../hadoop/hdfs/DistributedFileSystem.java | 25 ++
.../hadoop/hdfs/protocol/ClientProtocol.java | 13 +
.../ClientNamenodeProtocolTranslatorPB.java | 12 +
.../src/main/proto/ClientNamenodeProtocol.proto | 9 +
...tNamenodeProtocolServerSideTranslatorPB.java | 17 ++
.../hdfs/server/namenode/FSDirAttrOp.java | 74 +++--
.../hdfs/server/namenode/FSNamesystem.java | 22 ++
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 48 +++-
.../src/site/markdown/ArchivalStorage.md | 15 ++
.../hadoop/hdfs/TestApplyingStoragePolicy.java | 268 +++++++++++++++++++
.../hdfs/tools/TestStoragePolicyCommands.java | 63 +++++
16 files changed, 582 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 14da0d3..74f3ada 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2662,6 +2662,16 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
+ * Unset the storage policy set for a given file or directory.
+ * @param src file or directory path.
+ * @throws IOException
+ */
+ public void unsetStoragePolicy(Path src) throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support unsetStoragePolicy");
+ }
+
+ /**
* Query the effective storage policy ID for the given file or directory.
*
* @param src file or directory path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 4ee7514..3f9aaa4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -633,6 +633,11 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public void unsetStoragePolicy(Path src) throws IOException {
+ fs.unsetStoragePolicy(src);
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException {
return fs.getStoragePolicy(src);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 95ba663..d2020b9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -210,6 +210,8 @@ public class TestHarFileSystem {
public void setStoragePolicy(Path src, String policyName)
throws IOException;
+ public void unsetStoragePolicy(Path src) throws IOException;
+
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index cde89fb..24fc364 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1502,6 +1502,24 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
+ * Unset storage policy set for a given file/directory.
+ * @param src file/directory name
+ */
+ public void unsetStoragePolicy(String src) throws IOException {
+ checkOpen();
+ try (TraceScope ignored = newPathTraceScope("unsetStoragePolicy", src)) {
+ namenode.unsetStoragePolicy(src);
+ } catch (RemoteException e) {
+ throw e.unwrapRemoteException(AccessControlException.class,
+ FileNotFoundException.class,
+ SafeModeException.class,
+ NSQuotaExceededException.class,
+ UnresolvedPathException.class,
+ SnapshotAccessControlException.class);
+ }
+ }
+
+ /**
* @param path file/directory name
* @return Get the storage policy for specified path
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 1192b80..737afdb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -574,6 +574,31 @@ public class DistributedFileSystem extends FileSystem {
}
@Override
+ public void unsetStoragePolicy(final Path src)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ Path absF = fixRelativePart(src);
+ new FileSystemLinkResolver<Void>() {
+ @Override
+ public Void doCall(final Path p) throws IOException {
+ dfs.unsetStoragePolicy(getPathName(p));
+ return null;
+ }
+ @Override
+ public Void next(final FileSystem fs, final Path p) throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).unsetStoragePolicy(p);
+ return null;
+ } else {
+ throw new UnsupportedOperationException(
+ "Cannot perform unsetStoragePolicy on a "
+ + "non-DistributedFileSystem: " + src + " -> " + p);
+ }
+ }
+ }.resolve(this, absF);
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 01660db..a0371dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -283,6 +283,19 @@ public interface ClientProtocol {
throws IOException;
/**
+ * Unset the storage policy set for a given file or directory.
+ * @param src Path of an existing file/directory.
+ * @throws SnapshotAccessControlException If access is denied
+ * @throws org.apache.hadoop.fs.UnresolvedLinkException if <code>src</code>
+ * contains a symlink
+ * @throws java.io.FileNotFoundException If file/dir <code>src</code> is not
+ * found
+ * @throws QuotaExceededException If changes violate the quota restriction
+ */
+ @Idempotent
+ void unsetStoragePolicy(String src) throws IOException;
+
+ /**
* Get the storage policy for a file/directory.
* @param path
* Path of an existing file/directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index dd87b0b..e59f2a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -139,6 +139,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Recove
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RefreshNodesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCacheDirectiveRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UnsetStoragePolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameSnapshotRequestProto;
@@ -1512,6 +1513,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public void unsetStoragePolicy(String src) throws IOException {
+ UnsetStoragePolicyRequestProto req = UnsetStoragePolicyRequestProto
+ .newBuilder().setSrc(src).build();
+ try {
+ rpcProxy.unsetStoragePolicy(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
GetStoragePolicyRequestProto request = GetStoragePolicyRequestProto
.newBuilder().setPath(path).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 67365c3..c0c02f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -112,6 +112,13 @@ message SetStoragePolicyRequestProto {
message SetStoragePolicyResponseProto { // void response
}
+message UnsetStoragePolicyRequestProto {
+ required string src = 1;
+}
+
+message UnsetStoragePolicyResponseProto {
+}
+
message GetStoragePolicyRequestProto {
required string path = 1;
}
@@ -746,6 +753,8 @@ service ClientNamenodeProtocol {
returns(SetReplicationResponseProto);
rpc setStoragePolicy(SetStoragePolicyRequestProto)
returns(SetStoragePolicyResponseProto);
+ rpc unsetStoragePolicy(UnsetStoragePolicyRequestProto)
+ returns(UnsetStoragePolicyResponseProto);
rpc getStoragePolicy(GetStoragePolicyRequestProto)
returns(GetStoragePolicyResponseProto);
rpc getStoragePolicies(GetStoragePoliciesRequestProto)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index ca16c12..de5a4dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -158,6 +158,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Remove
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCacheDirectiveResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UnsetStoragePolicyRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UnsetStoragePolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2ResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto;
@@ -256,6 +258,9 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
GetSnapshottableDirListingResponseProto.newBuilder().build();
static final SetStoragePolicyResponseProto VOID_SET_STORAGE_POLICY_RESPONSE =
SetStoragePolicyResponseProto.newBuilder().build();
+ static final UnsetStoragePolicyResponseProto
+ VOID_UNSET_STORAGE_POLICY_RESPONSE =
+ UnsetStoragePolicyResponseProto.newBuilder().build();
private static final CreateResponseProto VOID_CREATE_RESPONSE =
CreateResponseProto.newBuilder().build();
@@ -1470,6 +1475,18 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public UnsetStoragePolicyResponseProto unsetStoragePolicy(
+ RpcController controller, UnsetStoragePolicyRequestProto request)
+ throws ServiceException {
+ try {
+ server.unsetStoragePolicy(request.getSrc());
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return VOID_UNSET_STORAGE_POLICY_RESPONSE;
+ }
+
+ @Override
public GetStoragePolicyResponseProto getStoragePolicy(
RpcController controller, GetStoragePolicyRequestProto request)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 7f41078..7575f5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.security.AccessControlException;
+import com.google.common.collect.Lists;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
@@ -158,13 +160,31 @@ public class FSDirAttrOp {
return isFile;
}
- static HdfsFileStatus setStoragePolicy(
- FSDirectory fsd, BlockManager bm, String src, final String policyName)
+ static HdfsFileStatus unsetStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src) throws IOException {
+ return setStoragePolicy(fsd, bm, src,
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, "unset");
+ }
+
+ static HdfsFileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src, final String policyName) throws IOException {
+ // get the corresponding policy and make sure the policy name is valid
+ BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
+ if (policy == null) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot find a block policy with the name " + policyName);
+ }
+
+ return setStoragePolicy(fsd, bm, src, policy.getId(), "set");
+ }
+
+ static HdfsFileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src, final byte policyId, final String operation)
throws IOException {
if (!fsd.isStoragePolicyEnabled()) {
- throw new IOException(
- "Failed to set storage policy since "
- + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
+ throw new IOException(String.format(
+ "Failed to %s storage policy since %s is set to false.", operation,
+ DFS_STORAGE_POLICY_ENABLED_KEY));
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
@@ -178,14 +198,8 @@ public class FSDirAttrOp {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
}
- // get the corresponding policy and make sure the policy name is valid
- BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
- if (policy == null) {
- throw new HadoopIllegalArgumentException(
- "Cannot find a block policy with the name " + policyName);
- }
- unprotectedSetStoragePolicy(fsd, bm, iip, policy.getId());
- fsd.getEditLog().logSetStoragePolicy(src, policy.getId());
+ unprotectedSetStoragePolicy(fsd, bm, iip, policyId);
+ fsd.getEditLog().logSetStoragePolicy(src, policyId);
} finally {
fsd.writeUnlock();
}
@@ -445,8 +459,8 @@ public class FSDirAttrOp {
return file.getBlocks();
}
- static void unprotectedSetStoragePolicy(
- FSDirectory fsd, BlockManager bm, INodesInPath iip, byte policyId)
+ static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
+ INodesInPath iip, final byte policyId)
throws IOException {
assert fsd.hasWriteLock();
final INode inode = iip.getLastINode();
@@ -456,10 +470,12 @@ public class FSDirAttrOp {
}
final int snapshotId = iip.getLatestSnapshotId();
if (inode.isFile()) {
- BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
- if (newPolicy.isCopyOnCreateFile()) {
- throw new HadoopIllegalArgumentException(
- "Policy " + newPolicy + " cannot be set after file creation.");
+ if (policyId != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
+ BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
+ if (newPolicy.isCopyOnCreateFile()) {
+ throw new HadoopIllegalArgumentException("Policy " + newPolicy
+ + " cannot be set after file creation.");
+ }
}
BlockStoragePolicy currentPolicy =
@@ -472,7 +488,8 @@ public class FSDirAttrOp {
}
inode.asFile().setStoragePolicyID(policyId, snapshotId);
} else if (inode.isDirectory()) {
- setDirStoragePolicy(fsd, inode.asDirectory(), policyId, snapshotId);
+ setDirStoragePolicy(fsd, inode.asDirectory(), policyId,
+ snapshotId);
} else {
throw new FileNotFoundException(iip.getPath()
+ " is not a file or directory");
@@ -484,11 +501,18 @@ public class FSDirAttrOp {
int latestSnapshotId) throws IOException {
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId);
- List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
- Arrays.asList(xAttr),
- EnumSet.of(
- XAttrSetFlag.CREATE,
- XAttrSetFlag.REPLACE));
+ List<XAttr> newXAttrs = null;
+ if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
+ List<XAttr> toRemove = Lists.newArrayList();
+ toRemove.add(xAttr);
+ List<XAttr> removed = Lists.newArrayList();
+ newXAttrs = FSDirXAttrOp.filterINodeXAttrs(existingXAttrs, toRemove,
+ removed);
+ } else {
+ newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
+ Arrays.asList(xAttr),
+ EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ }
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d83b4ef..2948ece 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2028,6 +2028,28 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * unset storage policy set for a given file or a directory.
+ *
+ * @param src file/directory path
+ */
+ void unsetStoragePolicy(String src) throws IOException {
+ HdfsFileStatus auditStat;
+ checkOperation(OperationCategory.WRITE);
+ writeLock();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot unset storage policy for " + src);
+ auditStat = FSDirAttrOp.unsetStoragePolicy(dir, blockManager, src);
+ } catch (AccessControlException e) {
+ logAuditEvent(false, "unsetStoragePolicy", src);
+ throw e;
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ logAuditEvent(true, "unsetStoragePolicy", src, null, auditStat);
+ }
+ /**
* Get the storage policy for a file or a directory.
*
* @param src
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index cfb470a..2996fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -781,6 +781,13 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
@Override
+ public void unsetStoragePolicy(String src)
+ throws IOException {
+ checkNNStartup();
+ namesystem.unsetStoragePolicy(src);
+ }
+
+ @Override
public void setStoragePolicy(String src, String policyName)
throws IOException {
checkNNStartup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index 98c8a6b..24079b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -224,9 +224,55 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
}
+ /* Command to unset the storage policy set for a file/directory */
+ private static class UnsetStoragePolicyCommand
+ implements AdminHelper.Command {
+
+ @Override
+ public String getName() {
+ return "-unsetStoragePolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>", "The path of the file/directory "
+ + "from which the storage policy will be unset.");
+ return getShortUsage() + "\n"
+ + "Unset the storage policy set for a file/directory.\n\n"
+ + listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify the path from which "
+ + "the storage policy will be unsetd.\nUsage: " + getLongUsage());
+ return 1;
+ }
+
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ try {
+ dfs.unsetStoragePolicy(new Path(path));
+ System.out.println("Unset storage policy from " + path);
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
private static final AdminHelper.Command[] COMMANDS = {
new ListStoragePoliciesCommand(),
new SetStoragePolicyCommand(),
- new GetStoragePolicyCommand()
+ new GetStoragePolicyCommand(),
+ new UnsetStoragePolicyCommand()
};
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
index cbfbaa1..803cc91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
@@ -26,6 +26,7 @@ Archival Storage, SSD & Memory
* [Storage Policy Commands](#Storage_Policy_Commands)
* [List Storage Policies](#List_Storage_Policies)
* [Set Storage Policy](#Set_Storage_Policy)
+ * [Unset Storage Policy](#Unset_Storage_Policy)
* [Get Storage Policy](#Get_Storage_Policy)
Introduction
@@ -150,6 +151,20 @@ Set a storage policy to a file or a directory.
| `-path <path>` | The path referring to either a directory or a file. |
| `-policy <policy>` | The name of the storage policy. |
+### Unset Storage Policy
+
+Unset a storage policy to a file or a directory. After the unset command the storage policy of the nearest ancestor will apply, and if there is no policy on any ancestor then the default storage policy will apply.
+
+* Command:
+
+ hdfs storagepolicies -unsetStoragePolicy -path <path>
+
+* Arguments:
+
+| | |
+|:---- |:---- |
+| `-path <path>` | The path referring to either a directory or a file. |
+
### Get Storage Policy
Get the storage policy of a file or a directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java
new file mode 100644
index 0000000..200fab6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestApplyingStoragePolicy {
+ private static final short REPL = 1;
+ private static final int SIZE = 128;
+
+ private static Configuration conf;
+ private static MiniDFSCluster cluster;
+ private static DistributedFileSystem fs;
+
+ @Before
+ public void clusterSetUp() throws IOException {
+ conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void clusterShutdown() throws IOException{
+ if(fs != null) {
+ fs.close();
+ fs = null;
+ }
+ if(cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testStoragePolicyByDefault() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ final Path fooz = new Path(bar, "/fooz");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: storage policy is HOT by default or inherited from nearest
+ * ancestor, if not explicitly specified for newly created dir/file.
+ */
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+
+ @Test
+ public void testSetAndUnsetStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ final Path fooz = new Path(bar, "/fooz");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: set storage policy
+ */
+ fs.setStoragePolicy(foo, warm.getName());
+ fs.setStoragePolicy(bar, cold.getName());
+ fs.setStoragePolicy(wow, hot.getName());
+ try {
+ fs.setStoragePolicy(fooz, warm.getName());
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(fs.getStoragePolicy(foo), warm);
+ assertEquals(fs.getStoragePolicy(bar), cold);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: unset storage policy
+ */
+ fs.unsetStoragePolicy(foo);
+ fs.unsetStoragePolicy(bar);
+ fs.unsetStoragePolicy(wow);
+ try {
+ fs.unsetStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after unset
+ */
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+
+ @Test
+ public void testNestedStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ final Path fooz = new Path("/foos");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: set storage policy
+ */
+ fs.setStoragePolicy(foo, warm.getName());
+ fs.setStoragePolicy(bar, cold.getName());
+ fs.setStoragePolicy(wow, hot.getName());
+ try {
+ fs.setStoragePolicy(fooz, warm.getName());
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(fs.getStoragePolicy(foo), warm);
+ assertEquals(fs.getStoragePolicy(bar), cold);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: unset storage policy in the case of being nested
+ */
+ // unset wow
+ fs.unsetStoragePolicy(wow);
+ // inherit storage policy from wow's nearest ancestor
+ assertEquals(fs.getStoragePolicy(wow), cold);
+ // unset bar
+ fs.unsetStoragePolicy(bar);
+ // inherit storage policy from bar's nearest ancestor
+ assertEquals(fs.getStoragePolicy(bar), warm);
+ // unset foo
+ fs.unsetStoragePolicy(foo);
+ // default storage policy is applied, since no more available ancestors
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ // unset fooz
+ try {
+ fs.unsetStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: default storage policy is applied, since no explicit policies from
+ * ancestors are available
+ */
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+
+ @Test
+ public void testSetAndGetStoragePolicy() throws IOException {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path fooz = new Path("/fooz");
+ DFSTestUtil.createFile(fs, bar, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: set storage policy
+ */
+ fs.setStoragePolicy(foo, warm.getName());
+ fs.setStoragePolicy(bar, cold.getName());
+ try {
+ fs.setStoragePolicy(fooz, warm.getName());
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(fs.getStoragePolicy(foo), warm);
+ assertEquals(fs.getStoragePolicy(bar), cold);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbf711c0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
index ec0bb66..63f8484 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
@@ -62,6 +62,69 @@ public class TestStoragePolicyCommands {
}
}
+
+ @Test
+ public void testSetAndUnsetStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ /*
+ * test: set storage policy
+ */
+ final StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo -policy WARM", 0,
+ "Set storage policy WARM on " + foo.toString());
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo/bar -policy COLD",
+ 0, "Set storage policy COLD on " + bar.toString());
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo/bar/wow -policy HOT",
+ 0, "Set storage policy HOT on " + wow.toString());
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /fooz -policy WARM",
+ 2, "File/Directory does not exist: /fooz");
+
+ /*
+ * test: get storage policy after set
+ */
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0,
+ "The storage policy of " + foo.toString() + ":\n" + warm);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar", 0,
+ "The storage policy of " + bar.toString() + ":\n" + cold);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar/wow", 0,
+ "The storage policy of " + wow.toString() + ":\n" + hot);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /fooz", 2,
+ "File/Directory does not exist: /fooz");
+
+ /*
+ * test: unset storage policy
+ */
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /foo", 0,
+ "Unset storage policy from " + foo.toString());
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /foo/bar", 0,
+ "Unset storage policy from " + bar.toString());
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /foo/bar/wow", 0,
+ "Unset storage policy from " + wow.toString());
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /fooz", 2,
+ "File/Directory does not exist: /fooz");
+
+ /*
+ * test: get storage policy after unset
+ */
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0,
+ "The storage policy of " + foo.toString() + " is unspecified");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar", 0,
+ "The storage policy of " + bar.toString() + " is unspecified");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar/wow", 0,
+ "The storage policy of " + wow.toString() + " is unspecified");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /fooz", 2,
+ "File/Directory does not exist: /fooz");
+ }
+
@Test
public void testSetAndGetStoragePolicy() throws Exception {
final Path foo = new Path("/foo");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HDFS-9894. Add unsetStoragePolicy API to
FileContext/AbstractFileSystem and derivatives. Contributed by Xiaobing Zhou.
Posted by ar...@apache.org.
HDFS-9894. Add unsetStoragePolicy API to FileContext/AbstractFileSystem and derivatives. Contributed by Xiaobing Zhou.
(cherry picked from commit 7149cdb3c2d9dd390cd8668883cbe5db94090e0a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54be36a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54be36a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54be36a6
Branch: refs/heads/branch-2.8
Commit: 54be36a6d6d46eaa6d88b4693ebfeee11c9445b1
Parents: bbf711c
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Apr 21 18:07:35 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jul 25 09:51:59 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/AbstractFileSystem.java | 11 +++++++++++
.../java/org/apache/hadoop/fs/FileContext.java | 17 +++++++++++++++++
.../main/java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/FilterFs.java | 6 ++++++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 ++++++
.../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 8 ++++++++
.../src/main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++
7 files changed, 54 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index 1fce04c..d1e7a1b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1236,6 +1236,17 @@ public abstract class AbstractFileSystem {
+ " doesn't support setStoragePolicy");
}
+
+ /**
+ * Unset the storage policy set for a given file or directory.
+ * @param src file or directory path.
+ * @throws IOException
+ */
+ public void unsetStoragePolicy(final Path src) throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support unsetStoragePolicy");
+ }
+
/**
* Retrieve the storage policy for a given file or directory.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 5866c49..568bf04 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -2706,6 +2706,23 @@ public class FileContext {
}
/**
+ * Unset the storage policy set for a given file or directory.
+ * @param src file or directory path.
+ * @throws IOException
+ */
+ public void unsetStoragePolicy(final Path src) throws IOException {
+ final Path absF = fixRelativePart(src);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ fs.unsetStoragePolicy(src);
+ return null;
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
* Query the effective storage policy ID for the given file or directory.
*
* @param src file or directory path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 74f3ada..a2e626c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2666,7 +2666,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param src file or directory path.
* @throws IOException
*/
- public void unsetStoragePolicy(Path src) throws IOException {
+ public void unsetStoragePolicy(final Path src) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support unsetStoragePolicy");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index 248377c..d7e313f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -406,6 +406,12 @@ public abstract class FilterFs extends AbstractFileSystem {
}
@Override
+ public void unsetStoragePolicy(final Path src)
+ throws IOException {
+ myFs.unsetStoragePolicy(src);
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException {
return myFs.getStoragePolicy(src);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 568b9a0..f07e07f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -386,6 +386,12 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
+ public void unsetStoragePolicy(final Path src)
+ throws IOException {
+ myFs.unsetStoragePolicy(fullPath(src));
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException {
return myFs.getStoragePolicy(src);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 6f05e77..2f93296 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -749,6 +749,14 @@ public class ViewFs extends AbstractFileSystem {
res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName);
}
+ @Override
+ public void unsetStoragePolicy(final Path src)
+ throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(src), true);
+ res.targetFileSystem.unsetStoragePolicy(res.remainingPath);
+ }
+
/**
* Retrieve the storage policy for a given file or directory.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54be36a6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 6dd8884..82ee41a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -474,6 +474,11 @@ public class Hdfs extends AbstractFileSystem {
}
@Override
+ public void unsetStoragePolicy(final Path src) throws IOException {
+ dfs.unsetStoragePolicy(getUriPath(src));
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
return dfs.getStoragePolicy(getUriPath(src));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org