You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/05/12 18:09:04 UTC
svn commit: r1102364 - in /hadoop/mapreduce/branches/branch-0.22: ./
src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/
src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/
src/contrib/vaidya/src/java/org/apache/hadoo...
Author: tomwhite
Date: Thu May 12 16:09:03 2011
New Revision: 1102364
URL: http://svn.apache.org/viewvc?rev=1102364&view=rev
Log:
Merge -r 1102362:1102363 from trunk to branch-0.22. Fixes: MAPREDUCE-1626.
Modified:
hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
hadoop/mapreduce/branches/branch-0.22/build.xml
hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/PostExPerformanceDiagnoser.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatisticsInterface.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/util/XMLUtils.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaConfiguration.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputSplit.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaOutputFormat.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordReader.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordReader.java
Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Thu May 12 16:09:03 2011
@@ -200,6 +200,9 @@ Release 0.22.0 - Unreleased
MAPREDUCE-2383. Improve documentation of DistributedCache methods (Harsh J
Chouraria via todd)
+ MAPREDUCE-2222. Ivy resolve force mode should be turned off by default.
+ (Luke Lu via tomwhite)
+
OPTIMIZATIONS
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
Modified: hadoop/mapreduce/branches/branch-0.22/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/build.xml?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/build.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/build.xml Thu May 12 16:09:03 2011
@@ -968,6 +968,8 @@
<packageset dir="src/contrib/index/src/java"/>
<packageset dir="src/contrib/mumak/src/java"/>
<packageset dir="src/contrib/streaming/src/java"/>
+ <packageset dir="src/contrib/vaidya/src/java"/>
+ <packageset dir="src/contrib/vertica/src/java"/>
<link href="${javadoc.link.java}"/>
@@ -991,6 +993,8 @@
<group title="contrib: Gridmix" packages="org.apache.hadoop.mapred.gridmix*"/>
<group title="contrib: Index" packages="org.apache.hadoop.contrib.index*"/>
<group title="contrib: Streaming" packages="org.apache.hadoop.streaming*:org.apache.hadoop.typedbytes*"/>
+ <group title="contrib: Vaidya" packages="org.apache.hadoop.vaidya*"/>
+ <group title="contrib: Vertica" packages="org.apache.hadoop.vertica*"/>
</javadoc>
</target>
@@ -1029,6 +1033,8 @@
<packageset dir="src/contrib/gridmix/src/java"/>
<packageset dir="src/contrib/index/src/java"/>
<packageset dir="src/contrib/streaming/src/java"/>
+ <packageset dir="src/contrib/vaidya/src/java"/>
+ <packageset dir="src/contrib/vertica/src/java"/>
<packageset dir="${tools.src}"/>
<link href="${javadoc.link.java}"/>
@@ -1053,6 +1059,9 @@
<group title="contrib: Gridmix" packages="org.apache.hadoop.mapred.gridmix*"/>
<group title="contrib: Index" packages="org.apache.hadoop.contrib.index*"/>
<group title="contrib: Streaming" packages="org.apache.hadoop.streaming*:org.apache.hadoop.typedbytes*"/>
+ <group title="contrib: Vaidya" packages="org.apache.hadoop.vaidya*"/>
+ <group title="contrib: Vertica" packages="org.apache.hadoop.vertica*"/>
+
<doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet"
path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar"/>
</javadoc>
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/PostExPerformanceDiagnoser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/PostExPerformanceDiagnoser.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/PostExPerformanceDiagnoser.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/PostExPerformanceDiagnoser.java Thu May 12 16:09:03 2011
@@ -92,8 +92,8 @@ public class PostExPerformanceDiagnoser
/**
* @param jobConfFile - URL pointing to job configuration (job_conf.xml) file
- * @param jobHistoryLogFile - URL pointing to job history log file
- * @param testsConfFile - file path for test configuration file (optional).
+ * @param jobHistoryFile - URL pointing to job history log file
+ * @param testsConfFileIs - file path for test configuration file (optional).
* If not specified default path is:$HADOOP_HOME/contrib/vaidya/pxpd_tests_config.xml
* @param reportFile - file path for storing report (optional)
*/
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java Thu May 12 16:09:03 2011
@@ -69,9 +69,6 @@ public class JobStatistics implements Jo
*/
private java.util.Hashtable<Enum, String> _job;
- /**
- * @param jobConf the jobConf to set
- */
public JobConf getJobConf() {
return this._jobConf;
}
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatisticsInterface.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatisticsInterface.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatisticsInterface.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatisticsInterface.java Thu May 12 16:09:03 2011
@@ -61,7 +61,7 @@ public interface JobStatisticsInterface
/**
* @return mapTaskList : ArrayList of MapTaskStatistics
* @param mapTaskSortKey : Specific counter key used for sorting the task list
- * @param datatype : indicates the data type of the counter key used for sorting
+ * @param dataType : indicates the data type of the counter key used for sorting
* If sort key is null then by default map tasks are sorted using map task ids.
*/
public ArrayList<MapTaskStatistics> getMapTaskList(Enum mapTaskSortKey, KeyDataType dataType);
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/util/XMLUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/util/XMLUtils.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/util/XMLUtils.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/util/XMLUtils.java Thu May 12 16:09:03 2011
@@ -148,7 +148,6 @@ public class XMLUtils {
/**
* Parse the XML file and create Document
- * @param fileName
* @return Document
*/
public static Document parse(InputStream fs) {
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaConfiguration.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaConfiguration.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaConfiguration.java Thu May 12 16:09:03 2011
@@ -45,14 +45,7 @@ import org.apache.hadoop.util.StringUtil
* String, String)
* @see VerticaConfiguration#configureVertica(Configuration, String[], String,
* String, String, String[], String, String, String)
- * @see VerticaInputFormat#setInput(Job, String)
- * @see VerticaInputFormat#setInput(Job, String, Collection<List<Object>>)
- * @see VerticaInputFormat#setInput(Job, String, String)
- * @see VerticaInputFormat#setInput(Job, String, String...)
- * @see VerticaOutputFormat#setOutput(Job, String)
- * @see VerticaOutputFormat#setOutput(Job, String, Collection<VerticaTable>)
- * @see VerticaOutputFormat#setOutput(Job, String, boolean)
- * @see VerticaOutputFormat#setOutput(Job, String, boolean, String...)
+ *
*/
public class VerticaConfiguration {
/** Vertica Version Constants */
@@ -309,7 +302,7 @@ public class VerticaConfiguration {
* Sets a collection of lists. Each list is passed to an input split and used
* as arguments to the input query.
*
- * @param segmentParams
+ * @param segment_params
* @throws IOException
*/
public void setInputParams(Collection<List<Object>> segment_params)
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputSplit.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputSplit.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputSplit.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputSplit.java Thu May 12 16:09:03 2011
@@ -88,7 +88,6 @@ public class VerticaInputSplit extends I
/**
* Return the parameters used for input query
*
- * @return
*/
public List<Object> getSegmentParams() {
return segmentParams;
@@ -97,7 +96,6 @@ public class VerticaInputSplit extends I
/**
* Run the query that, when executed returns input for the mapper
*
- * @return
* @throws Exception
*/
public ResultSet executeQuery() throws Exception {
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaOutputFormat.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaOutputFormat.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaOutputFormat.java Thu May 12 16:09:03 2011
@@ -48,7 +48,7 @@ public class VerticaOutputFormat extends
/**
* Set the output table
*
- * @param conf
+ * @param job
* @param tableName
*/
public static void setOutput(Job job, String tableName) {
@@ -136,7 +136,6 @@ public class VerticaOutputFormat extends
}
}
- /** {@inheritDoc} */
public static VerticaRecord getValue(Configuration conf) throws Exception {
VerticaConfiguration config = new VerticaConfiguration(conf);
String table = config.getOutputTableName();
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordReader.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordReader.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordReader.java Thu May 12 16:09:03 2011
@@ -48,7 +48,6 @@ public class VerticaRecordReader extends
results = split.executeQuery();
}
- /** {@inheritDoc} */
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
key = new LongWritable();
@@ -71,12 +70,10 @@ public class VerticaRecordReader extends
}
}
- /** {@inheritDoc} */
public long getPos() throws IOException {
return pos;
}
- /** {@inheritDoc} */
public float getProgress() throws IOException {
// TODO: figure out why length would be 0
if (length == 0)
@@ -84,7 +81,6 @@ public class VerticaRecordReader extends
return pos / length;
}
- /** {@inheritDoc} */
public boolean next(LongWritable key, VerticaRecord value) throws IOException {
key.set(pos + start);
pos++;
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordReader.java?rev=1102364&r1=1102363&r2=1102364&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordReader.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordReader.java Thu May 12 16:09:03 2011
@@ -71,12 +71,10 @@ public class VerticaStreamingRecordReade
}
}
- /** {@inheritDoc} */
public long getPos() throws IOException {
return pos;
}
- /** {@inheritDoc} */
public float getProgress() throws IOException {
// TODO: figure out why length would be 0
if (length == 0)