You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by um...@apache.org on 2020/08/26 17:52:56 UTC
[hadoop] branch branch-3.2 updated: HDFS-14096. [SPS] : Add Support
for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 3bacea2 HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.
3bacea2 is described below
commit 3bacea2e5ebd330964bfca4c1064d8f07d09112d
Author: Surendra Singh Lilhore <su...@apache.org>
AuthorDate: Mon Dec 17 11:24:57 2018 +0530
HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.
(cherry picked from commit 788e7473a404fa074b3af522416ee3d2fae865a0)
---
.../java/org/apache/hadoop/fs/AbstractFileSystem.java | 10 ++++++++++
.../main/java/org/apache/hadoop/fs/FileContext.java | 18 ++++++++++++++++++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 10 ++++++++++
.../java/org/apache/hadoop/fs/FilterFileSystem.java | 5 +++++
.../src/main/java/org/apache/hadoop/fs/FilterFs.java | 5 +++++
.../apache/hadoop/fs/viewfs/ChRootedFileSystem.java | 5 +++++
.../java/org/apache/hadoop/fs/viewfs/ChRootedFs.java | 5 +++++
.../org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +++++++++++++
.../main/java/org/apache/hadoop/fs/viewfs/ViewFs.java | 12 ++++++++++++
.../java/org/apache/hadoop/fs/TestHarFileSystem.java | 2 ++
.../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 5 +++++
.../src/main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++
.../org/apache/hadoop/hdfs/DistributedFileSystem.java | 6 +-----
13 files changed, 96 insertions(+), 5 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index c7b21fc..9926a74 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1250,6 +1250,16 @@ public abstract class AbstractFileSystem implements PathCapabilities {
}
/**
+ * Set the source path to satisfy storage policy.
+ * @param path The source path referring to either a directory or a file.
+ * @throws IOException
+ */
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ throw new UnsupportedOperationException(
+ getClass().getSimpleName() + " doesn't support satisfyStoragePolicy");
+ }
+
+ /**
* Set the storage policy for a given file or directory.
*
* @param path file or directory path.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index ace892d..4357c88 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -2778,6 +2778,24 @@ public class FileContext implements PathCapabilities {
}
/**
+ * Set the source path to satisfy storage policy.
+ * @param path The source path referring to either a directory or a file.
+ * @throws IOException
+ */
+ public void satisfyStoragePolicy(final Path path)
+ throws IOException {
+ final Path absF = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ fs.satisfyStoragePolicy(path);
+ return null;
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
* Set the storage policy for a given file or directory.
*
* @param path file or directory path.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index bac398b..22586b2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -3116,6 +3116,16 @@ public abstract class FileSystem extends Configured
}
/**
+ * Set the source path to satisfy storage policy.
+ * @param path The source path referring to either a directory or a file.
+ * @throws IOException
+ */
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ throw new UnsupportedOperationException(
+ getClass().getSimpleName() + " doesn't support setStoragePolicy");
+ }
+
+ /**
* Set the storage policy for a given file or directory.
*
* @param src file or directory path.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index c496748..90584be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -650,6 +650,11 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ fs.satisfyStoragePolicy(src);
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {
fs.setStoragePolicy(src, policyName);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index d037071..4f71c26 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -406,6 +406,11 @@ public abstract class FilterFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ myFs.satisfyStoragePolicy(path);
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {
myFs.setStoragePolicy(path, policyName);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 42158dd..8aab4b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -465,6 +465,11 @@ class ChRootedFileSystem extends FilterFileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ super.satisfyStoragePolicy(fullPath(src));
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
super.setStoragePolicy(fullPath(src), policyName);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 6168df0..5e14be1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -399,6 +399,11 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ myFs.satisfyStoragePolicy(path);
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {
myFs.setStoragePolicy(fullPath(path), policyName);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 2ffa8bd..4f172c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -979,6 +979,13 @@ public class ViewFileSystem extends FileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(src), true);
+ res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
true);
@@ -1626,6 +1633,12 @@ public class ViewFileSystem extends FileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ checkPathIsSlash(src);
+ throw readOnlyMountTable("satisfyStoragePolicy", src);
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {
checkPathIsSlash(src);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 4e40bc6..8168a6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -812,6 +812,13 @@ public class ViewFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
+ }
+
+ @Override
public void setStoragePolicy(final Path path, final String policyName)
throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res =
@@ -1378,6 +1385,11 @@ public class ViewFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ throw readOnlyMountTable("satisfyStoragePolicy", path);
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {
throw readOnlyMountTable("setStoragePolicy", path);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index dadbcb7..2f7f2b1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -215,6 +215,8 @@ public class TestHarFileSystem {
public void access(Path path, FsAction mode) throws IOException;
+ void satisfyStoragePolicy(Path src) throws IOException;
+
public void setStoragePolicy(Path src, String policyName)
throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 3d9637a..8985304 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -981,6 +981,11 @@ abstract public class ViewFileSystemBaseTest {
fsView.unsetStoragePolicy(new Path("/internalDir"));
}
+ @Test(expected = AccessControlException.class)
+ public void testInternalSatisfyStoragePolicy() throws IOException {
+ fsView.satisfyStoragePolicy(new Path("/internalDir"));
+ }
+
@Test(expected = NotInMountpointException.class)
public void testInternalgetStoragePolicy() throws IOException {
fsView.getStoragePolicy(new Path("/internalDir"));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 2638c7a..4162b19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -512,6 +512,11 @@ public class Hdfs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path path) throws IOException {
+ dfs.satisfyStoragePolicy(getUriPath(path));
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName) throws IOException {
dfs.setStoragePolicy(getUriPath(path), policyName);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index fd98625..7a218bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2914,11 +2914,7 @@ public class DistributedFileSystem extends FileSystem
}
/**
- * Set the source path to satisfy storage policy. This API is non-recursive
- * in nature, i.e., if the source path is a directory then all the files
- * immediately under the directory would be considered for satisfying the
- * policy and the sub-directories if any under this path will be skipped.
- *
+ * Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org