You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2014/10/29 18:29:33 UTC
git commit: MAPREDUCE-6022. map_input_file is missing from streaming
job environment. Contributed by Jason Lowe.
Repository: hadoop
Updated Branches:
refs/heads/trunk 782971ae7 -> b05604811
MAPREDUCE-6022. map_input_file is missing from streaming job
environment. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0560481
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0560481
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0560481
Branch: refs/heads/trunk
Commit: b056048114bf4701ef9dd22486db937cb589e81b
Parents: 782971a
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Oct 29 12:28:32 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Oct 29 12:29:07 2014 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 26 ++++++++++++++++++++
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/streaming/PipeMapRed.java | 4 ++-
.../java/org/apache/hadoop/streaming/TrApp.java | 3 +++
4 files changed, 35 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 98fa4f6..a3fae19 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -559,6 +559,32 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
/**
+ * Sets all deprecated properties that are not currently set but have a
+ * corresponding new property that is set. Useful for iterating the
+ * properties when all deprecated properties for currently set properties
+ * need to be present.
+ */
+ public void setDeprecatedProperties() {
+ DeprecationContext deprecations = deprecationContext.get();
+ Properties props = getProps();
+ Properties overlay = getOverlay();
+ for (Map.Entry<String, DeprecatedKeyInfo> entry :
+ deprecations.getDeprecatedKeyMap().entrySet()) {
+ String depKey = entry.getKey();
+ if (!overlay.contains(depKey)) {
+ for (String newKey : entry.getValue().newKeys) {
+ String val = overlay.getProperty(newKey);
+ if (val != null) {
+ props.setProperty(depKey, val);
+ overlay.setProperty(depKey, val);
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ /**
* Checks for the presence of the property <code>name</code> in the
* deprecation map. Returns the first of the list of new keys if present
* in the deprecation map or the <code>name</code> itself. If the property
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index c0edb37..2c06aa3 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -450,6 +450,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
+ MAPREDUCE-6022. map_input_file is missing from streaming job environment.
+ (jlowe via kihwal)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
index c8d099e..f47e756 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/PipeMapRed.java
@@ -235,7 +235,9 @@ public abstract class PipeMapRed {
}
}
- void addJobConfToEnvironment(JobConf conf, Properties env) {
+ void addJobConfToEnvironment(JobConf jobconf, Properties env) {
+ JobConf conf = new JobConf(jobconf);
+ conf.setDeprecatedProperties();
Iterator it = conf.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0560481/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
index e413261..d058332 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
@@ -57,6 +57,9 @@ public class TrApp
// the FileSplit context properties are not available in local hadoop..
// so can't check them in this test.
+ // verify some deprecated properties appear for older stream jobs
+ expect("map_input_file", env.getProperty("mapreduce_map_input_file"));
+ expect("map_input_length", env.getProperty("mapreduce_map_input_length"));
}
// this runs in a subprocess; won't use JUnit's assertTrue()