You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/07/25 08:25:33 UTC
[45/50] [abbrv] hadoop git commit: HADOOP-13272. ViewFileSystem
should support storage policy related API. Contributed by Peter Shi
HADOOP-13272. ViewFileSystem should support storage policy related API. Contributed by Peter Shi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22a9a6b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22a9a6b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22a9a6b0
Branch: refs/heads/YARN-4757
Commit: 22a9a6b0d412f31ea0f15ac389e56b9de9eeadb1
Parents: 5fda66e
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Jul 22 18:07:07 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Jul 22 18:08:12 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 23 +++++-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 75 +++++++++++++++++++-
.../fs/viewfs/TestChRootedFileSystem.java | 65 +++++++++++++++++
.../fs/viewfs/ViewFileSystemBaseTest.java | 54 ++++++++++----
4 files changed, 201 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22a9a6b0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 46e8f2d..747ba20 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -39,12 +40,12 @@ import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
@@ -425,5 +426,21 @@ class ChRootedFileSystem extends FilterFileSystem {
@Override
public FsServerDefaults getServerDefaults(Path f) throws IOException {
return super.getServerDefaults(fullPath(f));
- }
+ }
+
+ @Override
+ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
+ return super.getStoragePolicy(fullPath(src));
+ }
+
+ @Override
+ public void setStoragePolicy(Path src, String policyName) throws IOException {
+ super.setStoragePolicy(fullPath(src), policyName);
+ }
+
+ @Override
+ public void unsetStoragePolicy(Path src) throws IOException {
+ super.unsetStoragePolicy(fullPath(src));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22a9a6b0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 5f42e70..f7047b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@@ -35,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -48,6 +50,7 @@ import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -56,7 +59,6 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
import org.apache.hadoop.security.AccessControlException;
@@ -760,6 +762,43 @@ public class ViewFileSystem extends FileSystem {
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
+ @Override
+ public void setStoragePolicy(Path src, String policyName) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
+ true);
+ res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName);
+ }
+
+ @Override
+ public void unsetStoragePolicy(Path src) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
+ true);
+ res.targetFileSystem.unsetStoragePolicy(res.remainingPath);
+ }
+
+ @Override
+ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
+ true);
+ return res.targetFileSystem.getStoragePolicy(res.remainingPath);
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ Collection<BlockStoragePolicySpi> allPolicies = new HashSet<>();
+ for (FileSystem fs : getChildFileSystems()) {
+ try {
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fs.getAllStoragePolicies();
+ allPolicies.addAll(policies);
+ } catch (UnsupportedOperationException e) {
+ // ignored
+ }
+ }
+ return allPolicies;
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
@@ -1079,5 +1118,39 @@ public class ViewFileSystem extends FileSystem {
public QuotaUsage getQuotaUsage(Path f) throws IOException {
throw new NotInMountpointException(f, "getQuotaUsage");
}
+
+ @Override
+ public void setStoragePolicy(Path src, String policyName)
+ throws IOException {
+ checkPathIsSlash(src);
+ throw readOnlyMountTable("setStoragePolicy", src);
+ }
+
+ @Override
+ public void unsetStoragePolicy(Path src) throws IOException {
+ checkPathIsSlash(src);
+ throw readOnlyMountTable("unsetStoragePolicy", src);
+ }
+
+ @Override
+ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
+ throw new NotInMountpointException(src, "getStoragePolicy");
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ Collection<BlockStoragePolicySpi> allPolicies = new HashSet<>();
+ for (FileSystem fs : getChildFileSystems()) {
+ try {
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fs.getAllStoragePolicies();
+ allPolicies.addAll(policies);
+ } catch (UnsupportedOperationException e) {
+ // ignored
+ }
+ }
+ return allPolicies;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22a9a6b0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
index e4a1ac9..780f409 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
@@ -468,4 +468,69 @@ public class TestChRootedFileSystem {
verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
"snapNewName");
}
+
+ @Test(timeout = 30000)
+ public void testSetStoragePolicy() throws Exception {
+ Path storagePolicyPath = new Path("/storagePolicy");
+ Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.setStoragePolicy(storagePolicyPath, "HOT");
+ verify(mockFs).setStoragePolicy(chRootedStoragePolicyPath, "HOT");
+ }
+
+ @Test(timeout = 30000)
+ public void testUnsetStoragePolicy() throws Exception {
+ Path storagePolicyPath = new Path("/storagePolicy");
+ Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.unsetStoragePolicy(storagePolicyPath);
+ verify(mockFs).unsetStoragePolicy(chRootedStoragePolicyPath);
+ }
+
+ @Test(timeout = 30000)
+ public void testGetStoragePolicy() throws Exception {
+ Path storagePolicyPath = new Path("/storagePolicy");
+ Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.getStoragePolicy(storagePolicyPath);
+ verify(mockFs).getStoragePolicy(chRootedStoragePolicyPath);
+ }
+
+ @Test(timeout = 30000)
+ public void testGetAllStoragePolicy() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.getAllStoragePolicies();
+ verify(mockFs).getAllStoragePolicies();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22a9a6b0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index ea4d9b1..02d4fd6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -19,42 +19,41 @@ package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
-
+import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
-import static org.apache.hadoop.fs.FileSystemTestHelper.*;
-import org.apache.hadoop.fs.permission.AclEntry;
-import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
-import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.ConfigUtil;
-import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import static org.apache.hadoop.fs.FileSystemTestHelper.*;
+import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.*;
/**
@@ -885,4 +884,35 @@ abstract public class ViewFileSystemBaseTest {
public void testInternalDeleteSnapshot() throws IOException {
fsView.deleteSnapshot(new Path("/internalDir"), "snap1");
}
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalSetStoragePolicy() throws IOException {
+ fsView.setStoragePolicy(new Path("/internalDir"), "HOT");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalUnsetStoragePolicy() throws IOException {
+ fsView.unsetStoragePolicy(new Path("/internalDir"));
+ }
+
+ @Test(expected = NotInMountpointException.class)
+ public void testInternalgetStoragePolicy() throws IOException {
+ fsView.getStoragePolicy(new Path("/internalDir"));
+ }
+
+ @Test
+ public void testInternalGetAllStoragePolicies() throws IOException {
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fsView.getAllStoragePolicies();
+ for (FileSystem fs : fsView.getChildFileSystems()) {
+ try {
+ for (BlockStoragePolicySpi s : fs.getAllStoragePolicies()) {
+ assertTrue("Missing policy: " + s, policies.contains(s));
+ }
+ } catch (UnsupportedOperationException e) {
+ // ignore
+ }
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org