You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2011/09/30 21:16:54 UTC
svn commit: r1177783 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/JobSplitWriter.java
Author: shv
Date: Fri Sep 30 19:16:54 2011
New Revision: 1177783
URL: http://svn.apache.org/viewvc?rev=1177783&view=rev
Log:
MAPREDUCE-2779. JobSplitWriter.java can't handle large job.split file. Contributed by Ming Ma.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/JobSplitWriter.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1177783&r1=1177782&r2=1177783&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Sep 30 19:16:54 2011
@@ -2080,6 +2080,9 @@ Release 0.21.1 - Unreleased
MAPREDUCE-2127. mapreduce trunk builds are filing on hudson.
(Bruno Mahé via eli)
+ MAPREDUCE-2779. JobSplitWriter.java can't handle large job.split file.
+ (Ming Ma via shv)
+
Release 0.21.0 - 2010-08-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/JobSplitWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/JobSplitWriter.java?rev=1177783&r1=1177782&r2=1177783&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/JobSplitWriter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/JobSplitWriter.java Fri Sep 30 19:16:54 2011
@@ -114,15 +114,15 @@ public class JobSplitWriter {
if (array.length != 0) {
SerializationFactory factory = new SerializationFactory(conf);
int i = 0;
- long offset = out.size();
+ long offset = out.getPos();
for(T split: array) {
- int prevCount = out.size();
+ long prevCount = out.getPos();
Text.writeString(out, split.getClass().getName());
Serializer<T> serializer =
factory.getSerializer((Class<T>) split.getClass());
serializer.open(out);
serializer.serialize(split);
- int currCount = out.size();
+ long currCount = out.getPos();
info[i++] =
new JobSplit.SplitMetaInfo(
split.getLocations(), offset,
@@ -139,12 +139,12 @@ public class JobSplitWriter {
SplitMetaInfo[] info = new SplitMetaInfo[splits.length];
if (splits.length != 0) {
int i = 0;
- long offset = out.size();
+ long offset = out.getPos();
for(org.apache.hadoop.mapred.InputSplit split: splits) {
- int prevLen = out.size();
+ long prevLen = out.getPos();
Text.writeString(out, split.getClass().getName());
split.write(out);
- int currLen = out.size();
+ long currLen = out.getPos();
info[i++] = new JobSplit.SplitMetaInfo(
split.getLocations(), offset,
split.getLength());