You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by rk...@apache.org on 2015/05/05 23:50:14 UTC
hadoop git commit: MAPREDUCE-6192. Create unit test to automatically
compare MR related classes and mapred-default.xml (rchiang via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0100b1550 -> 9809a16d3
MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9809a16d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9809a16d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9809a16d
Branch: refs/heads/trunk
Commit: 9809a16d3c8068beccbf0106e99c7ede6ba11e0f
Parents: 0100b15
Author: Robert Kanter <rk...@apache.org>
Authored: Mon May 4 17:48:10 2015 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue May 5 14:43:28 2015 -0700
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 58 ++++++++++++++-
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapred/TestMapreduceConfigFields.java | 76 ++++++++++++++++++++
3 files changed, 135 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index c3fe3a3..2e4d8b1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -147,6 +147,12 @@ public abstract class TestConfigurationFieldsBase {
private Set<String> xmlFieldsMissingInConfiguration = null;
/**
+ * Member variable for debugging base class operation
+ */
+ protected boolean configDebug = false;
+ protected boolean xmlDebug = false;
+
+ /**
* Abstract method to be used by subclasses for initializing base
* members.
*/
@@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
HashMap<String,String> retVal = new HashMap<String,String>();
// Setup regexp for valid properties
- String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$";
+ String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
Pattern p = Pattern.compile(propRegex);
// Iterate through class member variables
int totalFields = 0;
String value;
for (Field f : fields) {
+ if (configDebug) {
+ System.out.println("Field: " + f);
+ }
// Filter out anything that isn't "public static final"
if (!Modifier.isStatic(f.getModifiers()) ||
!Modifier.isPublic(f.getModifiers()) ||
@@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
} catch (IllegalAccessException iaException) {
continue;
}
+ if (configDebug) {
+ System.out.println(" Value: " + value);
+ }
// Special Case: Detect and ignore partial properties (ending in x)
// or file properties (ending in .xml)
if (value.endsWith(".xml") ||
@@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
// something like: blah.blah2(.blah3.blah4...)
Matcher m = p.matcher(value);
if (!m.find()) {
+ if (configDebug) {
+ System.out.println(" Passes Regex: false");
+ }
continue;
}
+ if (configDebug) {
+ System.out.println(" Passes Regex: true");
+ }
// Save member variable/value as hash
- retVal.put(value,f.getName());
+ if (!retVal.containsKey(value)) {
+ retVal.put(value,f.getName());
+ } else {
+ if (configDebug) {
+ System.out.println("ERROR: Already found key for property " + value);
+ }
+ }
}
return retVal;
@@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
// Ignore known xml props
if (xmlPropsToSkipCompare != null) {
if (xmlPropsToSkipCompare.contains(key)) {
+ if (xmlDebug) {
+ System.out.println(" Skipping Full Key: " + key);
+ }
continue;
}
}
@@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
}
}
if (skipPrefix) {
+ if (xmlDebug) {
+ System.out.println(" Skipping Prefix Key: " + key);
+ }
continue;
}
if (conf.onlyKeyExists(key)) {
retVal.put(key,null);
+ if (xmlDebug) {
+ System.out.println(" XML Key,Null Value: " + key);
+ }
} else {
String value = conf.get(key);
if (value!=null) {
retVal.put(key,entry.getValue());
+ if (xmlDebug) {
+ System.out.println(" XML Key,Valid Value: " + key);
+ }
}
}
kvItr.remove();
@@ -312,6 +348,10 @@ public abstract class TestConfigurationFieldsBase {
// Create class member/value map
configurationMemberVariables = new HashMap<String,String>();
+ if (configDebug) {
+ System.out.println("Reading configuration classes");
+ System.out.println("");
+ }
for (Class c : configurationClasses) {
Field[] fields = c.getDeclaredFields();
Map<String,String> memberMap =
@@ -320,9 +360,23 @@ public abstract class TestConfigurationFieldsBase {
configurationMemberVariables.putAll(memberMap);
}
}
+ if (configDebug) {
+ System.out.println("");
+ System.out.println("=====");
+ System.out.println("");
+ }
// Create XML key/value map
+ if (xmlDebug) {
+ System.out.println("Reading XML property files");
+ System.out.println("");
+ }
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+ if (xmlDebug) {
+ System.out.println("");
+ System.out.println("=====");
+ System.out.println("");
+ }
// Find class members not in the XML file
configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 002fbe6..99621cb 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -304,6 +304,9 @@ Release 2.8.0 - UNRELEASED
mapreduce.tasktracker.taskmemorymanager.monitoringinterval.
(J.Andreina via aajisaka)
+ MAPREDUCE-6192. Create unit test to automatically compare
+ MR related classes and mapred-default.xml (rchiang via rkanter)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
new file mode 100644
index 0000000..7f18714
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+
+/**
+ * Unit test class to compare the following MR Configuration classes:
+ * <p></p>
+ * {@link org.apache.hadoop.mapreduce.MRJobConfig}
+ * {@link org.apache.hadoop.mapreduce.MRConfig}
+ * {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig}
+ * {@link org.apache.hadoop.mapred.ShuffleHandler}
+ * {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat}
+ * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
+ * {@link org.apache.hadoop.mapreduce.Job}
+ * {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat}
+ * {@link org.apache.hadoop.mapred.JobConf}
+ * <p></p>
+ * against mapred-default.xml for missing properties. Currently only
+ * throws an error if the class is missing a property.
+ * <p></p>
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void initializeMemberVariables() {
+ xmlFilename = new String("mapred-default.xml");
+ configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class,
+ JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class,
+ FileInputFormat.class, Job.class, NLineInputFormat.class,
+ JobConf.class, FileOutputCommitter.class };
+
+ // Initialize used variables
+ configurationPropsToSkipCompare = new HashSet<String>();
+
+ // Set error modes
+ errorIfMissingConfigProps = true;
+ errorIfMissingXmlProps = false;
+
+ // Ignore deprecated MR1 properties in JobConf
+ configurationPropsToSkipCompare
+ .add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY);
+ configurationPropsToSkipCompare
+ .add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+ }
+
+}