You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tg...@apache.org on 2012/07/31 15:40:39 UTC
svn commit: r1367540 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/s...
Author: tgraves
Date: Tue Jul 31 13:40:39 2012
New Revision: 1367540
URL: http://svn.apache.org/viewvc?rev=1367540&view=rev
Log:
merge -r 1367538:1367539 from trunk. FIXES: MAPREDUCE-4375
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 13:40:39 2012
@@ -381,6 +381,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
+ MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby
+ via tgraves)
+
OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 13:40:39 2012
@@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac
Path confPath = getConfFile();
FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confPath));
+ jobConf.addResource(fc.open(confPath), confPath.toString());
return jobConf;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Tue Jul 31 13:40:39 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
tr().
th(_TH, "key").
th(_TH, "value").
+ th(_TH, "source chain").
_().
_().
tbody();
for (ConfEntryInfo entry : info.getProperties()) {
+ StringBuffer buffer = new StringBuffer();
+ String[] sources = entry.getSource();
+ //Skip the last entry, because it is always the same HDFS file, and
+ // output them in reverse order so most recent is output first
+ boolean first = true;
+ for(int i = (sources.length - 2); i >= 0; i--) {
+ if(!first) {
+ // \u2B05 is an arrow <--
+ buffer.append(" \u2B05 ");
+ }
+ first = false;
+ buffer.append(sources[i]);
+ }
tbody.
tr().
td(entry.getName()).
td(entry.getValue()).
+ td(buffer.toString()).
_();
}
tbody._().
@@ -93,6 +108,7 @@ public class ConfBlock extends HtmlBlock
tr().
th().input("search_init").$type(InputType.text).$name("key").$value("key")._()._().
th().input("search_init").$type(InputType.text).$name("value").$value("value")._()._().
+ th().input("search_init").$type(InputType.text).$name("source chain").$value("source chain")._()._().
_().
_().
_();
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java Tue Jul 31 13:40:39 2012
@@ -27,13 +27,19 @@ public class ConfEntryInfo {
protected String name;
protected String value;
+ protected String[] source;
public ConfEntryInfo() {
}
public ConfEntryInfo(String key, String value) {
+ this(key, value, null);
+ }
+
+ public ConfEntryInfo(String key, String value, String[] source) {
this.name = key;
this.value = value;
+ this.source = source;
}
public String getName() {
@@ -43,4 +49,8 @@ public class ConfEntryInfo {
public String getValue() {
return this.value;
}
+
+ public String[] getSource() {
+ return source;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java Tue Jul 31 13:40:39 2012
@@ -46,7 +46,8 @@ public class ConfInfo {
Configuration jobConf = job.loadConfFile();
this.path = job.getConfFile().toString();
for (Map.Entry<String, String> entry : jobConf) {
- this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue()));
+ this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue(),
+ jobConf.getPropertySources(entry.getKey())));
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Tue Jul 31 13:40:39 2012
@@ -603,7 +603,7 @@ public class MockJobs extends MockApps {
public Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(configFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(configFile));
+ jobConf.addResource(fc.open(configFile), configFile.toString());
return jobConf;
}
};
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java Tue Jul 31 13:40:39 2012
@@ -336,7 +336,7 @@ public class HistoryFileManager extends
public synchronized Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confFile));
+ jobConf.addResource(fc.open(confFile), confFile.toString());
return jobConf;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm Tue Jul 31 13:40:39 2012
@@ -1261,6 +1261,9 @@ History Server REST API's.
*---------------+--------------+-------------------------------+
| value | string | The value of the configuration property |
*---------------+--------------+-------------------------------+
+| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. |
+*---------------+--------------+-------------------------------+
+
*** Response Examples
@@ -1293,14 +1296,17 @@ History Server REST API's.
{
"value" : "/home/hadoop/hdfs/data",
"name" : "dfs.datanode.data.dir"
+ "source" : ["hdfs-site.xml", "job.xml"]
},
{
"value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
"name" : "hadoop.http.filter.initializers"
+ "source" : ["programatically", "job.xml"]
},
{
"value" : "/home/hadoop/tmp",
"name" : "mapreduce.cluster.temp.dir"
+ "source" : ["mapred-site.xml"]
},
...
]
@@ -1335,14 +1341,19 @@ History Server REST API's.
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/hdfs/data</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
+ <source>programatically</source>
+ <source>job.xml</source>
</property>
<property>
<name>mapreduce.cluster.temp.dir</name>
<value>/home/hadoop/tmp</value>
+ <source>mapred-site.xml</source>
</property>
...
</conf>
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm?rev=1367540&r1=1367539&r2=1367540&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm Tue Jul 31 13:40:39 2012
@@ -1296,6 +1296,8 @@ MapReduce Application Master REST API's.
*---------------+--------------+-------------------------------+
| value | string | The value of the configuration property |
*---------------+--------------+-------------------------------+
+| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. |
+*---------------+--------------+-------------------------------+
** Response Examples
@@ -1327,15 +1329,18 @@ MapReduce Application Master REST API's.
"property" : [
{
"value" : "/home/hadoop/hdfs/data",
- "name" : "dfs.datanode.data.dir"
+ "name" : "dfs.datanode.data.dir",
+ "source" : ["hdfs-site.xml", "job.xml"]
},
{
"value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
"name" : "hadoop.http.filter.initializers"
+ "source" : ["programatically", "job.xml"]
},
{
"value" : "/home/hadoop/tmp",
"name" : "mapreduce.cluster.temp.dir"
+ "source" : ["mapred-site.xml"]
},
...
]
@@ -1370,14 +1375,19 @@ MapReduce Application Master REST API's.
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/hdfs/data</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
+ <source>programatically</source>
+ <source>job.xml</source>
</property>
<property>
<name>mapreduce.cluster.temp.dir</name>
<value>/home/hadoop/tmp</value>
+ <source>mapred-site.xml</source>
</property>
...
</conf>