You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2016/12/07 02:17:12 UTC
[1/2] hadoop git commit: HADOOP-13859. TestConfigurationFieldsBase
fails for fields that are DEFAULT values of skipped properties. (Haibo Chen
via kasha)
Repository: hadoop
Updated Branches:
refs/heads/YARN-5355 3b2e80881 -> 12bce022e
refs/heads/YARN-5355-branch-2 4537f5665 -> 5c58e55ea
HADOOP-13859. TestConfigurationFieldsBase fails for fields that are DEFAULT values of skipped properties. (Haibo Chen via kasha)
(cherry picked from commit c8d0a049b00536385f06fad412a2288f005bf2ce)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12bce022
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12bce022
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12bce022
Branch: refs/heads/YARN-5355
Commit: 12bce022ea1dd9743f441a60ac20b95da411be93
Parents: 3b2e808
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Tue Dec 6 14:08:08 2016 -0800
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Dec 6 18:07:23 2016 -0800
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 20 ++++++++++++++--
.../hadoop/tools/TestHdfsConfigFields.java | 6 -----
.../yarn/conf/TestYarnConfigurationFields.java | 24 --------------------
3 files changed, 18 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12bce022/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index eab0161..11da129 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -196,6 +196,12 @@ public abstract class TestConfigurationFieldsBase {
if (!f.getType().getName().equals("java.lang.String")) {
continue;
}
+
+ // filter out default-value fields
+ if (isFieldADefaultValue(f)) {
+ continue;
+ }
+
// Convert found member into String
try {
value = (String) f.get(null);
@@ -323,6 +329,17 @@ public abstract class TestConfigurationFieldsBase {
}
/**
+ * Test if a field is a default value of another property by
+ * checking if its name starts with "DEFAULT_" or ends with
+ * "_DEFAULT".
+ * @param field the field to check
+ */
+ private static boolean isFieldADefaultValue(Field field) {
+ return field.getName().startsWith("DEFAULT_") ||
+ field.getName().endsWith("_DEFAULT");
+ }
+
+ /**
* Utility function to extract "public static final" default
* member variables from a Configuration type class.
*
@@ -354,8 +371,7 @@ public abstract class TestConfigurationFieldsBase {
}
// Special: Stuff any property beginning with "DEFAULT_" into a
// different hash for later processing
- if (f.getName().startsWith("DEFAULT_") ||
- f.getName().endsWith("_DEFAULT")) {
+ if (isFieldADefaultValue(f)) {
if (retVal.containsKey(f.getName())) {
continue;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12bce022/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index bf29428..95ba822 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -56,12 +56,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
// Remove deprecated properties listed in Configuration#DeprecationDelta
configurationPropsToSkipCompare.add(DFSConfigKeys.DFS_DF_INTERVAL_KEY);
- // Remove default properties
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT);
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT);
-
// Remove support property
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12bce022/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 0c40fa9..756b164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -49,22 +49,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Specific properties to skip
configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_CLIENT_FACTORY_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_SERVER_FACTORY_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_RPC_IMPL);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_RM_SCHEDULER);
- configurationPropsToSkipCompare
.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL);
configurationPropsToSkipCompare
@@ -82,14 +66,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
// Ignore blacklisting nodes for AM failures feature since it is still a
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-13859. TestConfigurationFieldsBase
fails for fields that are DEFAULT values of skipped properties. (Haibo Chen
via kasha)
Posted by sj...@apache.org.
HADOOP-13859. TestConfigurationFieldsBase fails for fields that are DEFAULT values of skipped properties. (Haibo Chen via kasha)
(cherry picked from commit c8d0a049b00536385f06fad412a2288f005bf2ce)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c58e55e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c58e55e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c58e55e
Branch: refs/heads/YARN-5355-branch-2
Commit: 5c58e55ea8a79ba8415f5cb1ae24ff10fcb1aefc
Parents: 4537f56
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Tue Dec 6 14:08:08 2016 -0800
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Dec 6 18:16:36 2016 -0800
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 20 ++++++++++++++--
.../hadoop/tools/TestHdfsConfigFields.java | 6 -----
.../yarn/conf/TestYarnConfigurationFields.java | 24 --------------------
3 files changed, 18 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c58e55e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index eab0161..11da129 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -196,6 +196,12 @@ public abstract class TestConfigurationFieldsBase {
if (!f.getType().getName().equals("java.lang.String")) {
continue;
}
+
+ // filter out default-value fields
+ if (isFieldADefaultValue(f)) {
+ continue;
+ }
+
// Convert found member into String
try {
value = (String) f.get(null);
@@ -323,6 +329,17 @@ public abstract class TestConfigurationFieldsBase {
}
/**
+ * Test if a field is a default value of another property by
+ * checking if its name starts with "DEFAULT_" or ends with
+ * "_DEFAULT".
+ * @param field the field to check
+ */
+ private static boolean isFieldADefaultValue(Field field) {
+ return field.getName().startsWith("DEFAULT_") ||
+ field.getName().endsWith("_DEFAULT");
+ }
+
+ /**
* Utility function to extract "public static final" default
* member variables from a Configuration type class.
*
@@ -354,8 +371,7 @@ public abstract class TestConfigurationFieldsBase {
}
// Special: Stuff any property beginning with "DEFAULT_" into a
// different hash for later processing
- if (f.getName().startsWith("DEFAULT_") ||
- f.getName().endsWith("_DEFAULT")) {
+ if (isFieldADefaultValue(f)) {
if (retVal.containsKey(f.getName())) {
continue;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c58e55e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index ae13f06..92019f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -59,12 +59,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
// Remove deprecated properties in DeprecatedProperties.md
configurationPropsToSkipCompare.add(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY);
- // Remove default properties
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT);
- configurationPropsToSkipCompare
- .add(DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT);
-
// Remove support property
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c58e55e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 0c40fa9..756b164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -49,22 +49,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Specific properties to skip
configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_CLIENT_FACTORY_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_SERVER_FACTORY_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_IPC_RPC_IMPL);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_RM_SCHEDULER);
- configurationPropsToSkipCompare
.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL);
configurationPropsToSkipCompare
@@ -82,14 +66,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
// Ignore blacklisting nodes for AM failures feature since it is still a
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org