You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by um...@apache.org on 2020/07/09 23:26:38 UTC
[hadoop] branch branch-3.3 updated: HDFS-15394. Add all available
fs.viewfs.overload.scheme.target..impl classes in core-default.xml
bydefault. Contributed by Uma Maheswara Rao G.
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new f85ce25 HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G.
f85ce25 is described below
commit f85ce2570e796e0ca838535340cd3c049634a225
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Sat Jun 6 08:11:57 2020 -0700
HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G.
(cherry picked from commit 3ca15292c5584ec220b3eeaf76da85d228bcbd8b)
---
.../src/main/resources/core-default.xml | 110 +++++++++++++++++++++
.../hadoop/conf/TestCommonConfigurationFields.java | 18 ++++
...ViewFileSystemOverloadSchemeWithHdfsScheme.java | 9 +-
3 files changed, 136 insertions(+), 1 deletion(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 0d583cc..accb1b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -953,6 +953,116 @@
</property>
<property>
+ <name>fs.viewfs.overload.scheme.target.hdfs.impl</name>
+ <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+ <description>The DistributedFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are hdfs.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.s3a.impl</name>
+ <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+ <description>The S3AFileSystem for view file system overload scheme when
+ child file system and ViewFSOverloadScheme's schemes are s3a.</description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.o3fs.impl</name>
+ <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
+ <description>The OzoneFileSystem for view file system overload scheme when
+ child file system and ViewFSOverloadScheme's schemes are o3fs.</description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.ftp.impl</name>
+ <value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
+ <description>The FTPFileSystem for view file system overload scheme when
+ child file system and ViewFSOverloadScheme's schemes are ftp.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.webhdfs.impl</name>
+ <value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value>
+ <description>The WebHdfsFileSystem for view file system overload scheme when
+ child file system and ViewFSOverloadScheme's schemes are webhdfs.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.swebhdfs.impl</name>
+ <value>org.apache.hadoop.hdfs.web.SWebHdfsFileSystem</value>
+ <description>The SWebHdfsFileSystem for view file system overload scheme when
+ child file system and ViewFSOverloadScheme's schemes are swebhdfs.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.file.impl</name>
+ <value>org.apache.hadoop.fs.LocalFileSystem</value>
+ <description>The LocalFileSystem for view file system overload scheme when
+ child file system and ViewFSOverloadScheme's schemes are file.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.abfs.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value>
+ <description>The AzureBlobFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are abfs.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.abfss.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value>
+ <description>The SecureAzureBlobFileSystem for view file system overload
+ scheme when child file system and ViewFSOverloadScheme's schemes are abfss.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.wasb.impl</name>
+ <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
+ <description>The NativeAzureFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are wasb.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.swift.impl</name>
+ <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
+ <description>The SwiftNativeFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are swift.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.oss.impl</name>
+ <value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
+ <description>The AliyunOSSFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are oss.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.http.impl</name>
+ <value>org.apache.hadoop.fs.http.HttpFileSystem</value>
+ <description>The HttpFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are http.
+ </description>
+</property>
+
+<property>
+ <name>fs.viewfs.overload.scheme.target.https.impl</name>
+ <value>org.apache.hadoop.fs.http.HttpsFileSystem</value>
+ <description>The HttpsFileSystem for view file system overload scheme
+ when child file system and ViewFSOverloadScheme's schemes are https.
+ </description>
+</property>
+
+<property>
<name>fs.AbstractFileSystem.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
<description>The FileSystem for Ftp: uris.</description>
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 1ce23a0..3b9947e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -124,6 +124,24 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPrefixToSkipCompare.add("fs.adl.");
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
+ // ViewfsOverloadScheme target fs impl property keys are dynamically
+ // constructed and they are advanced props.
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfs.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.o3fs.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.oss.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
+ xmlPropsToSkipCompare.
+ add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
+ xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
+
// Azure properties are in a different class
// - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore
// - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
index 6e101a6..f0f3aae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
@@ -382,13 +382,17 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme {
* Note: Above links created because to make fs initialization success.
* Otherwise will not proceed if no mount links.
*
- * Don't set fs.viewfs.overload.scheme.target.hdfs.impl property.
+ * Unset fs.viewfs.overload.scheme.target.hdfs.impl property.
* So, OverloadScheme target fs initialization will fail.
*/
@Test(expected = UnsupportedFileSystemException.class, timeout = 30000)
public void testInvalidOverloadSchemeTargetFS() throws Exception {
final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER);
+ String mountTableIfSet = conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH);
conf = new Configuration();
+ if (mountTableIfSet != null) {
+ conf.set(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH, mountTableIfSet);
+ }
addMountLinks(defaultFSURI.getAuthority(),
new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER,
Constants.CONFIG_VIEWFS_LINK_FALLBACK },
@@ -400,6 +404,9 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme {
defaultFSURI.toString());
conf.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME),
ViewFileSystemOverloadScheme.class.getName());
+ conf.unset(String.format(
+ FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
+ HDFS_SCHEME));
try (FileSystem fs = FileSystem.get(conf)) {
fs.createNewFile(new Path("/onRootWhenFallBack"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org