You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/25 19:36:09 UTC
[02/29] hadoop git commit: HADOOP-12161. Add getStoragePolicy API to
the FileSystem interface. (Contributed by Brahma Reddy Battula)
HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adfa34ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adfa34ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adfa34ff
Branch: refs/heads/HADOOP-12111
Commit: adfa34ff9992295a6d2496b259d8c483ed90b566
Parents: 3bba180
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Jul 23 10:13:04 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jul 23 10:13:04 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/fs/AbstractFileSystem.java | 13 +++++++++++++
.../java/org/apache/hadoop/fs/FileContext.java | 20 ++++++++++++++++++++
.../java/org/apache/hadoop/fs/FileSystem.java | 13 +++++++++++++
.../org/apache/hadoop/fs/FilterFileSystem.java | 6 ++++++
.../java/org/apache/hadoop/fs/FilterFs.java | 6 ++++++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 ++++++
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 +++++++++++++++
.../org/apache/hadoop/fs/TestHarFileSystem.java | 3 +++
.../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++++++++++++++++++
.../hadoop/hdfs/DistributedFileSystem.java | 19 +++++++++++++++++++
.../hadoop/hdfs/TestBlockStoragePolicy.java | 17 +++++++++++++++++
13 files changed, 144 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ff7d2ad..f1a3bc9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -716,6 +716,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin
Walsh via Colin P. McCabe)
+ HADOOP-12161. Add getStoragePolicy API to the FileSystem interface.
+ (Brahma Reddy Battula via Arpit Agarwal)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index cb3fb86..2bc3859 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1237,6 +1237,19 @@ public abstract class AbstractFileSystem {
}
/**
+ * Retrieve the storage policy for a given file or directory.
+ *
+ * @param src file or directory path.
+ * @return storage policy for give file.
+ * @throws IOException
+ */
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support getStoragePolicy");
+ }
+
+ /**
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 0f21a61..a98d662 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RpcClientException;
import org.apache.hadoop.ipc.RpcServerException;
@@ -2692,6 +2693,25 @@ public class FileContext {
}
/**
+ * Query the effective storage policy ID for the given file or directory.
+ *
+ * @param src file or directory path.
+ * @return storage policy for give file.
+ * @throws IOException
+ */
+ public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException {
+ final Path absF = fixRelativePart(path);
+ return new FSLinkResolver<BlockStoragePolicySpi>() {
+ @Override
+ public BlockStoragePolicySpi next(final AbstractFileSystem fs,
+ final Path p)
+ throws IOException {
+ return fs.getStoragePolicy(p);
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 5e03e88..a01d3ea 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2629,6 +2629,19 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
+ * Query the effective storage policy ID for the given file or directory.
+ *
+ * @param src file or directory path.
+ * @return storage policy for give file.
+ * @throws IOException
+ */
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support getStoragePolicy");
+ }
+
+ /**
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 11f3b23..815ef69 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -628,6 +628,12 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ return fs.getStoragePolicy(src);
+ }
+
+ @Override
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {
return fs.getAllStoragePolicies();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index 539b26e..248377c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -406,6 +406,12 @@ public abstract class FilterFs extends AbstractFileSystem {
}
@Override
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ return myFs.getStoragePolicy(src);
+ }
+
+ @Override
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {
return myFs.getAllStoragePolicies();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 4e5a0d5..568b9a0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -386,6 +386,12 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ return myFs.getStoragePolicy(src);
+ }
+
+ @Override
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {
return myFs.getAllStoragePolicies();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index bec292c..6f05e77 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -748,6 +749,20 @@ public class ViewFs extends AbstractFileSystem {
res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName);
}
+ /**
+ * Retrieve the storage policy for a given file or directory.
+ *
+ * @param src file or directory path.
+ * @return storage policy for give file.
+ * @throws IOException
+ */
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(src), true);
+ return res.targetFileSystem.getStoragePolicy(res.remainingPath);
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* ie internal dir of the mount table.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 46f24fc..1710ba4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -210,6 +210,9 @@ public class TestHarFileSystem {
public void setStoragePolicy(Path src, String policyName)
throws IOException;
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException;
+
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 3f78b31..ba5687c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -473,6 +473,11 @@ public class Hdfs extends AbstractFileSystem {
}
@Override
+ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
+ return dfs.getStoragePolicy(getUriPath(src));
+ }
+
+ @Override
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {
return Arrays.asList(dfs.getStoragePolicies());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 6f9e613..44713a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1574,6 +1574,24 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
+ * @return Get the storage policy for specified path
+ */
+ public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
+ HdfsFileStatus status = getFileInfo(path);
+ if (status == null) {
+ throw new FileNotFoundException("File does not exist: " + path);
+ }
+ byte storagePolicyId = status.getStoragePolicy();
+ BlockStoragePolicy[] policies = getStoragePolicies();
+ for (BlockStoragePolicy policy : policies) {
+ if (policy.getId() == storagePolicyId) {
+ return policy;
+ }
+ }
+ return null;
+ }
+
+ /**
* @return All the existing storage policies
*/
public BlockStoragePolicy[] getStoragePolicies() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 902636c..4d5c0f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -557,6 +557,25 @@ public class DistributedFileSystem extends FileSystem {
}
@Override
+ public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException {
+ statistics.incrementReadOps(1);
+ Path absF = fixRelativePart(path);
+
+ return new FileSystemLinkResolver<BlockStoragePolicySpi>() {
+ @Override
+ public BlockStoragePolicySpi doCall(final Path p) throws IOException {
+ return getClient().getStoragePolicy(getPathName(p));
+ }
+
+ @Override
+ public BlockStoragePolicySpi next(final FileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.getStoragePolicy(p);
+ }
+ }.resolve(this, absF);
+ }
+
+ @Override
public Collection<BlockStoragePolicy> getAllStoragePolicies()
throws IOException {
return Arrays.asList(dfs.getStoragePolicies());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 0d59ded..afd2597 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -871,9 +871,25 @@ public class TestBlockStoragePolicy {
GenericTestUtils.assertExceptionContains(invalidPath.toString(), e);
}
+ try {
+ fs.getStoragePolicy(invalidPath);
+ Assert.fail("Should throw a FileNotFoundException");
+ } catch (FileNotFoundException e) {
+ GenericTestUtils.assertExceptionContains(invalidPath.toString(), e);
+ }
+
fs.setStoragePolicy(fooFile, HdfsServerConstants.COLD_STORAGE_POLICY_NAME);
fs.setStoragePolicy(barDir, HdfsServerConstants.WARM_STORAGE_POLICY_NAME);
fs.setStoragePolicy(barFile2, HdfsServerConstants.HOT_STORAGE_POLICY_NAME);
+ Assert.assertEquals("File storage policy should be COLD",
+ HdfsServerConstants.COLD_STORAGE_POLICY_NAME,
+ fs.getStoragePolicy(fooFile).getName());
+ Assert.assertEquals("File storage policy should be WARM",
+ HdfsServerConstants.WARM_STORAGE_POLICY_NAME,
+ fs.getStoragePolicy(barDir).getName());
+ Assert.assertEquals("File storage policy should be HOT",
+ HdfsServerConstants.HOT_STORAGE_POLICY_NAME,
+ fs.getStoragePolicy(barFile2).getName());
dirList = fs.getClient().listPaths(dir.toString(),
HdfsFileStatus.EMPTY_NAME).getPartialListing();
@@ -1306,4 +1322,5 @@ public class TestBlockStoragePolicy {
Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey());
}
}
+
}