You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/05/04 17:05:04 UTC
[11/16] hadoop git commit: HADOOP-12101. Add automatic search of
default Configuration variables to TestConfigurationFieldsBase. Contributed
by Ray Chiang.
HADOOP-12101. Add automatic search of default Configuration variables to TestConfigurationFieldsBase. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/355325bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/355325bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/355325bc
Branch: refs/heads/HDFS-7240
Commit: 355325bcc7111fa4aac801fd23a26422ffabaf7c
Parents: 75e0450
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed May 4 15:12:51 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed May 4 15:12:51 2016 +0900
----------------------------------------------------------------------
dev-support/verify-xml.sh | 150 +++++++++++
.../conf/TestConfigurationFieldsBase.java | 255 +++++++++++++++++++
2 files changed, 405 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/355325bc/dev-support/verify-xml.sh
----------------------------------------------------------------------
diff --git a/dev-support/verify-xml.sh b/dev-support/verify-xml.sh
new file mode 100755
index 0000000..abab4e6
--- /dev/null
+++ b/dev-support/verify-xml.sh
@@ -0,0 +1,150 @@
+#!/bin/bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# Script to run unit tests for xml <-> 1 or more Configuration file verification
+# usage: ./verify-xml.sh <mode>
+#
+
+# Utility functions
+function find_test_output_file() {
+ echo "Found test output file(s) at"
+ echo ""
+ if [ -n "$1" ] && [ -e "$1" ] ; then
+ echo " $1"
+ fi
+ if [ -n "$2" ] && [ -e "$2" ] ; then
+ echo " $2"
+ fi
+ if [ -n "$3" ] && [ -e "$3" ] ; then
+ echo " $3"
+ fi
+ if [ -n "$4" ] && [ -e "$4" ] ; then
+ echo " $4"
+ fi
+ echo ""
+ echo "Examine the file for specific information xml/Configuration mismatches."
+ echo ""
+}
+
+function print_test_banner() {
+ local banner_text=$1
+ local banner_length=${#banner_text}
+ local banner
+ banner=$( printf "%${banner_length}s" ' ' )
+ echo ""
+ echo "${banner// /=}"
+ echo "${banner_text}"
+ echo "${banner// /=}"
+ echo ""
+}
+
+# Wrapper functions for running unit tests
+function run_all_xml_test() {
+ mvn test -Dtest=TestCommonConfigurationFields,TestHdfsConfigFields,TestMapreduceConfigFields,TestYarnConfigurationFields
+ if [ $? -ne 0 ] ; then
+ print_test_banner "All Test*ConfigFields FAIL"
+ else
+ print_test_banner "All Test*ConfigFields SUCCESS"
+ fi
+}
+
+function run_common_xml_test() {
+ mvn test -Dtest=TestCommonConfigFields
+ if [ $? -ne 0 ] ; then
+ print_test_banner "TestCommonConfigurationFields FAIL"
+ else
+ print_test_banner "TestCommonConfigurationFields SUCCESS"
+ fi
+}
+
+function run_hdfs_xml_test() {
+ mvn test -Dtest=TestHdfsConfigFields
+ if [ $? -ne 0 ] ; then
+ print_test_banner "TestHdfsConfigFields FAIL"
+ else
+ print_test_banner "TestHdfsConfigFields SUCCESS"
+ fi
+}
+
+function run_mapreduce_xml_test() {
+ mvn test -Dtest=TestMapreduceConfigFields
+ if [ $? -ne 0 ] ; then
+ print_test_banner "TestMapreduceConfigFields FAIL"
+ else
+ print_test_banner "TestMapreduceConfigFields SUCCESS"
+ fi
+}
+
+function run_yarn_xml_test() {
+ mvn test -Dtest=TestYarnConfigurationFields
+ if [ $? -ne 0 ] ; then
+ print_test_banner "TestYarnConfigurationFields FAIL"
+ else
+ print_test_banner "TestYarnConfigurationFields SUCCESS"
+ fi
+}
+
+# Main body
+cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")/.." || exit
+dir="$(pwd -P)"
+
+# - Create unit test file names
+export commonOutputFile
+commonOutputFile="$(find "${dir}" -name org.apache.hadoop.conf.TestCommonConfigurationFields-output.txt)"
+export hdfsOutputFile
+hdfsOutputFile="$(find "${dir}" -name org.apache.hadoop.tools.TestHdfsConfigFields-output.txt)"
+export mrOutputFile
+mrOutputFile="$(find "${dir}" -name org.apache.hadoop.mapreduce.TestMapreduceConfigFields-output.txt)"
+export yarnOutputFile
+yarnOutputFile="$(find "${dir}" -name org.apache.hadoop.yarn.conf.TestYarnConfigurationFields-output.txt)"
+
+# - Determine which tests to run
+case "$1" in
+
+ all)
+ run_all_xml_test
+ find_test_output_file "${commonOutputFile}" "${hdfsOutputFile}" "${mrOutputFile}" "${yarnOutputFile}"
+ ;;
+
+ common)
+ run_common_xml_test
+ find_test_output_file "${commonOutputFile}"
+ ;;
+
+ hdfs)
+ run_hdfs_xml_test
+ find_test_output_file "${hdfsOutputFile}"
+ ;;
+
+ mr)
+ run_mapreduce_xml_test
+ find_test_output_file "${mrOutputFile}"
+ ;;
+
+ yarn)
+ run_yarn_xml_test
+ find_test_output_file "${yarnOutputFile}"
+ ;;
+
+ *)
+ echo "Usage: $0 <mode>"
+ echo " where <mode> is one of all, common, hdfs, mr, yarn"
+ exit 1
+ ;;
+
+esac
http://git-wip-us.apache.org/repos/asf/hadoop/blob/355325bc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index e528602..eab0161 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -125,6 +125,11 @@ public abstract class TestConfigurationFieldsBase {
private Map<String,String> configurationMemberVariables = null;
/**
+ * Member variable to store Configuration variables for later reference.
+ */
+ private Map<String,String> configurationDefaultVariables = null;
+
+ /**
* Member variable to store XML properties for later comparison.
*/
private Map<String,String> xmlKeyValueMap = null;
@@ -146,6 +151,7 @@ public abstract class TestConfigurationFieldsBase {
*/
protected boolean configDebug = false;
protected boolean xmlDebug = false;
+ protected boolean defaultDebug = false;
/**
* Abstract method to be used by subclasses for initializing base
@@ -317,6 +323,79 @@ public abstract class TestConfigurationFieldsBase {
}
/**
+ * Utility function to extract "public static final" default
+ * member variables from a Configuration type class.
+ *
+ * @param fields The class member variables
+ * @return HashMap containing <DefaultVariableName,DefaultValue> entries
+ */
+ private HashMap<String,String>
+ extractDefaultVariablesFromConfigurationFields(Field[] fields) {
+ // Sanity Check
+ if (fields==null) {
+ return null;
+ }
+
+ HashMap<String,String> retVal = new HashMap<String,String>();
+
+ // Setup regexp for valid properties
+ String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
+ Pattern p = Pattern.compile(propRegex);
+
+ // Iterate through class member variables
+ int totalFields = 0;
+ String value;
+ for (Field f : fields) {
+ // Filter out anything that isn't "public static final"
+ if (!Modifier.isStatic(f.getModifiers()) ||
+ !Modifier.isPublic(f.getModifiers()) ||
+ !Modifier.isFinal(f.getModifiers())) {
+ continue;
+ }
+ // Special: Stuff any property beginning with "DEFAULT_" into a
+ // different hash for later processing
+ if (f.getName().startsWith("DEFAULT_") ||
+ f.getName().endsWith("_DEFAULT")) {
+ if (retVal.containsKey(f.getName())) {
+ continue;
+ }
+ try {
+ if (f.getType().getName().equals("java.lang.String")) {
+ String sValue = (String) f.get(null);
+ retVal.put(f.getName(),sValue);
+ } else if (f.getType().getName().equals("short")) {
+ short shValue = (short) f.get(null);
+ retVal.put(f.getName(),Integer.toString(shValue));
+ } else if (f.getType().getName().equals("int")) {
+ int iValue = (int) f.get(null);
+ retVal.put(f.getName(),Integer.toString(iValue));
+ } else if (f.getType().getName().equals("long")) {
+ long lValue = (long) f.get(null);
+ retVal.put(f.getName(),Long.toString(lValue));
+ } else if (f.getType().getName().equals("float")) {
+ float fValue = (float) f.get(null);
+ retVal.put(f.getName(),Float.toString(fValue));
+ } else if (f.getType().getName().equals("double")) {
+ double dValue = (double) f.get(null);
+ retVal.put(f.getName(),Double.toString(dValue));
+ } else if (f.getType().getName().equals("boolean")) {
+ boolean bValue = (boolean) f.get(null);
+ retVal.put(f.getName(),Boolean.toString(bValue));
+ } else {
+ if (defaultDebug) {
+ System.out.println("Config variable " + f.getName() + " has unknown type " + f.getType().getName());
+ }
+ }
+ } catch (IllegalAccessException iaException) {
+ iaException.printStackTrace();
+ }
+ }
+ }
+
+ return retVal;
+ }
+
+ /**
* Perform set difference operation on keyMap2 from keyMap1.
*
* @param keyMap1 The initial set
@@ -374,6 +453,26 @@ public abstract class TestConfigurationFieldsBase {
System.out.println("");
}
+ // Create default configuration variable key/value map
+ if (defaultDebug) {
+ System.out.println("Reading Config property files for defaults");
+ System.out.println("");
+ }
+ configurationDefaultVariables = new HashMap<String,String>();
+ for (Class c : configurationClasses) {
+ Field[] fields = c.getDeclaredFields();
+ Map<String,String> defaultMap =
+ extractDefaultVariablesFromConfigurationFields(fields);
+ if (defaultMap!=null) {
+ configurationDefaultVariables.putAll(defaultMap);
+ }
+ }
+ if (defaultDebug) {
+ System.out.println("");
+ System.out.println("=====");
+ System.out.println("");
+ }
+
// Find class members not in the XML file
configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields
(configurationMemberVariables, xmlKeyValueMap);
@@ -464,4 +563,160 @@ public abstract class TestConfigurationFieldsBase {
assertTrue(configErrorMsg.toString(), missingConfigSize==0);
}
}
+
+ /**
+ * For each property in the XML file, verify that the value matches
+ * up to the default if one exists.
+ */
+ @Test
+ public void testXmlAgainstDefaultValuesInConfigurationClass() {
+ // Error if subclass hasn't set class members
+ assertTrue(xmlFilename!=null);
+ assertTrue(configurationMemberVariables!=null);
+ assertTrue(configurationDefaultVariables!=null);
+
+ HashSet<String> xmlPropertiesWithEmptyValue = new HashSet<String>();
+ HashSet<String> configPropertiesWithNoDefaultConfig = new HashSet<String>();
+ HashMap<String,String> xmlPropertiesMatchingConfigDefault =
+ new HashMap<String,String>();
+ // Ugly solution. Should have tuple-based solution.
+ HashMap<HashMap<String,String>,HashMap<String,String>> mismatchingXmlConfig =
+ new HashMap<HashMap<String,String>,HashMap<String,String>>();
+
+ for (Map.Entry<String,String> xEntry : xmlKeyValueMap.entrySet()) {
+ String xmlProperty = xEntry.getKey();
+ String xmlDefaultValue = xEntry.getValue();
+ String configProperty = configurationMemberVariables.get(xmlProperty);
+ if (configProperty!=null) {
+ String defaultConfigName = null;
+ String defaultConfigValue = null;
+
+ // Type 1: Prepend DEFAULT_
+ String defaultNameCheck1 = "DEFAULT_" + configProperty;
+ String defaultValueCheck1 = configurationDefaultVariables
+ .get(defaultNameCheck1);
+ // Type 2: Swap _KEY suffix with _DEFAULT suffix
+ String defaultNameCheck2 = null;
+ if (configProperty.endsWith("_KEY")) {
+ defaultNameCheck2 = configProperty
+ .substring(0,configProperty.length()-4) + "_DEFAULT";
+ }
+ String defaultValueCheck2 = configurationDefaultVariables
+ .get(defaultNameCheck2);
+ // Type Last: Append _DEFAULT suffix
+ String defaultNameCheck3 = configProperty + "_DEFAULT";
+ String defaultValueCheck3 = configurationDefaultVariables
+ .get(defaultNameCheck3);
+
+ // Pick the default value that exists
+ if (defaultValueCheck1!=null) {
+ defaultConfigName = defaultNameCheck1;
+ defaultConfigValue = defaultValueCheck1;
+ } else if (defaultValueCheck2!=null) {
+ defaultConfigName = defaultNameCheck2;
+ defaultConfigValue = defaultValueCheck2;
+ } else if (defaultValueCheck3!=null) {
+ defaultConfigName = defaultNameCheck3;
+ defaultConfigValue = defaultValueCheck3;
+ }
+
+ if (defaultConfigValue!=null) {
+ if (xmlDefaultValue==null) {
+ xmlPropertiesWithEmptyValue.add(xmlProperty);
+ } else if (!xmlDefaultValue.equals(defaultConfigValue)) {
+ HashMap<String,String> xmlEntry =
+ new HashMap<String,String>();
+ xmlEntry.put(xmlProperty,xmlDefaultValue);
+ HashMap<String,String> configEntry =
+ new HashMap<String,String>();
+ configEntry.put(defaultConfigName,defaultConfigValue);
+ mismatchingXmlConfig.put(xmlEntry,configEntry);
+ } else {
+ xmlPropertiesMatchingConfigDefault
+ .put(xmlProperty, defaultConfigName);
+ }
+ } else {
+ configPropertiesWithNoDefaultConfig.add(configProperty);
+ }
+ } else {
+ }
+ }
+
+ // Print out any unknown mismatching XML value/Config default value
+ System.out.println(this.xmlFilename + " has " +
+ mismatchingXmlConfig.size() +
+ " properties that do not match the default Config value");
+ if (mismatchingXmlConfig.size()==0) {
+ System.out.println(" (None)");
+ } else {
+ for (Map.Entry<HashMap<String,String>,HashMap<String,String>> xcEntry :
+ mismatchingXmlConfig.entrySet()) {
+ HashMap<String,String> xmlMap = xcEntry.getKey();
+ HashMap<String,String> configMap = xcEntry.getValue();
+ for (Map.Entry<String,String> xmlEntry : xmlMap.entrySet()) {
+ System.out.println(" XML Property: " + xmlEntry.getKey());
+ System.out.println(" XML Value: " + xmlEntry.getValue());
+ }
+ for (Map.Entry<String,String> configEntry : configMap.entrySet()) {
+ System.out.println(" Config Name: " + configEntry.getKey());
+ System.out.println(" Config Value: " + configEntry.getValue());
+ }
+ System.out.println("");
+ }
+ }
+ System.out.println();
+
+ // Print out Config properties that have no corresponding DEFAULT_*
+ // variable and cannot do any XML comparison (i.e. probably needs to
+ // be checked by hand)
+ System.out.println("Configuration(s) have " +
+ configPropertiesWithNoDefaultConfig.size() +
+ " properties with no corresponding default member variable. These" +
+ " will need to be verified manually.");
+ if (configPropertiesWithNoDefaultConfig.size()==0) {
+ System.out.println(" (None)");
+ } else {
+ Iterator<String> cItr = configPropertiesWithNoDefaultConfig.iterator();
+ while (cItr.hasNext()) {
+ System.out.println(" " + cItr.next());
+ }
+ }
+ System.out.println();
+
+ // MAYBE TODO Print out any known mismatching XML value/Config default
+
+ // Print out XML properties that have empty values (i.e. should result
+ // in code-based default)
+ System.out.println(this.xmlFilename + " has " +
+ xmlPropertiesWithEmptyValue.size() + " properties with empty values");
+ if (xmlPropertiesWithEmptyValue.size()==0) {
+ System.out.println(" (None)");
+ } else {
+ Iterator<String> xItr = xmlPropertiesWithEmptyValue.iterator();
+ while (xItr.hasNext()) {
+ System.out.println(" " + xItr.next());
+ }
+ }
+ System.out.println();
+
+ // Print out any matching XML value/Config default value
+ System.out.println(this.xmlFilename + " has " +
+ xmlPropertiesMatchingConfigDefault.size() +
+ " properties which match a corresponding Config variable");
+ if (xmlPropertiesMatchingConfigDefault.size()==0) {
+ System.out.println(" (None)");
+ } else {
+ for (Map.Entry<String,String> xcEntry :
+ xmlPropertiesMatchingConfigDefault.entrySet()) {
+ System.out.println(" " + xcEntry.getKey() + " / " +
+ xcEntry.getValue());
+ }
+ }
+ System.out.println();
+
+ // Test separator
+ System.out.println();
+ System.out.println("=====");
+ System.out.println();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org