You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-user@hadoop.apache.org by Keith Thompson <kt...@binghamton.edu> on 2012/04/30 19:10:16 UTC
EOFException
I have been running several MapReduce jobs on some input text files. They
were working fine earlier and then I suddenly started getting EOFException
every time. Even the jobs that ran fine before (on the exact same input
files) aren't running now. I am a bit perplexed as to what is causing this
error. Here is the error:
12/04/30 12:55:55 INFO mapred.JobClient: Task Id :
attempt_201202240659_6328_m_000001_1, Status : FAILED
java.lang.RuntimeException: java.io.EOFException
at
org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:128)
at
org.apache.hadoop.mapred.MapTask$MapOutputBuffer.compare(MapTask.java:967)
at org.apache.hadoop.util.QuickSort.fix(QuickSort.java:30)
at org.apache.hadoop.util.QuickSort.sortInternal(QuickSort.java:83)
at org.apache.hadoop.util.QuickSort.sort(QuickSort.java:59)
at
org.apache.hadoop.mapred.MapTask$MapOutputBuffer.sortAndSpill(MapTask.java:1253)
at
org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush(MapTask.java:1154)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:391)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:324)
at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at org.apache.hadoop.mapred.Child.main(Child.java:262)
Caused by: java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:375)
at com.xerox.twitter.bin.UserTime.readFields(UserTime.java:31)
at
org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:122)
Since the compare function seems to be involved, here is my custom key
class. Note: I did not include year in the key because all keys have the
same year.
public class UserTime implements WritableComparable<UserTime> {
int id, month, day, year, hour, min, sec;
public UserTime() {
}
public UserTime(int u, int mon, int d, int y, int h, int m, int s) {
id = u;
month = mon;
day = d;
year = y;
hour = h;
min = m;
sec = s;
}
@Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
id = in.readInt();
month = in.readInt();
day = in.readInt();
year = in.readInt();
hour = in.readInt();
min = in.readInt();
sec = in.readInt();
}
@Override
public void write(DataOutput out) throws IOException {
// TODO Auto-generated method stub
out.write(id);
out.write(month);
out.write(day);
out.write(year);
out.write(hour);
out.write(min);
out.write(sec);
}
@Override
public int compareTo(UserTime that) {
// TODO Auto-generated method stub
if(compareUser(that) == 0)
return (compareTime(that));
else if(compareUser(that) == 1)
return 1;
else return -1;
}
private int compareUser(UserTime that) {
if(id > that.id)
return 1;
else if(id == that.id)
return 0;
else return -1;
}
//assumes all are from the same year
private int compareTime(UserTime that) {
if(month > that.month ||
(month == that.month && day > that.day) ||
(month == that.month && day == that.day && hour > that.hour) ||
(month == that.month && day == that.day && hour == that.hour && min >
that.min) ||
(month == that.month && day == that.day && hour == that.hour && min ==
that.min && sec > that.sec))
return 1;
else if(month == that.month && day == that.day && hour == that.hour && min
== that.min && sec == that.sec)
return 0;
else return -1;
}
public String toString() {
String h, m, s;
if(hour < 10)
h = "0"+hour;
else
h = Integer.toString(hour);
if(min < 10)
m = "0"+min;
else
m = Integer.toString(hour);
if(sec < 10)
s = "0"+min;
else
s = Integer.toString(hour);
return (id+"\t"+month+"/"+day+"/"+year+"\t"+h+":"+m+":"+s);
}
}
Thanks for any help.
Regards,
Keith
Re: EOFException
Posted by madhu phatak <ph...@gmail.com>.
Hi,
In write method ,use writeInt() rather than write method. It should solve
your problem.
On Mon, Apr 30, 2012 at 10:40 PM, Keith Thompson <kt...@binghamton.edu>wrote:
> I have been running several MapReduce jobs on some input text files. They
> were working fine earlier and then I suddenly started getting EOFException
> every time. Even the jobs that ran fine before (on the exact same input
> files) aren't running now. I am a bit perplexed as to what is causing this
> error. Here is the error:
>
> 12/04/30 12:55:55 INFO mapred.JobClient: Task Id :
> attempt_201202240659_6328_m_000001_1, Status : FAILED
> java.lang.RuntimeException: java.io.EOFException
> at
>
> org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:128)
> at
> org.apache.hadoop.mapred.MapTask$MapOutputBuffer.compare(MapTask.java:967)
> at org.apache.hadoop.util.QuickSort.fix(QuickSort.java:30)
> at org.apache.hadoop.util.QuickSort.sortInternal(QuickSort.java:83)
> at org.apache.hadoop.util.QuickSort.sort(QuickSort.java:59)
> at
>
> org.apache.hadoop.mapred.MapTask$MapOutputBuffer.sortAndSpill(MapTask.java:1253)
> at
> org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush(MapTask.java:1154)
> at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:391)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:324)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
>
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
> at org.apache.hadoop.mapred.Child.main(Child.java:262)
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readInt(DataInputStream.java:375)
> at com.xerox.twitter.bin.UserTime.readFields(UserTime.java:31)
> at
>
> org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:122)
>
> Since the compare function seems to be involved, here is my custom key
> class. Note: I did not include year in the key because all keys have the
> same year.
>
> public class UserTime implements WritableComparable<UserTime> {
>
> int id, month, day, year, hour, min, sec;
> public UserTime() {
>
> }
> public UserTime(int u, int mon, int d, int y, int h, int m, int s) {
> id = u;
> month = mon;
> day = d;
> year = y;
> hour = h;
> min = m;
> sec = s;
> }
> @Override
> public void readFields(DataInput in) throws IOException {
> // TODO Auto-generated method stub
> id = in.readInt();
> month = in.readInt();
> day = in.readInt();
> year = in.readInt();
> hour = in.readInt();
> min = in.readInt();
> sec = in.readInt();
> }
>
> @Override
> public void write(DataOutput out) throws IOException {
> // TODO Auto-generated method stub
> out.write(id);
> out.write(month);
> out.write(day);
> out.write(year);
> out.write(hour);
> out.write(min);
> out.write(sec);
> }
>
> @Override
> public int compareTo(UserTime that) {
> // TODO Auto-generated method stub
> if(compareUser(that) == 0)
> return (compareTime(that));
> else if(compareUser(that) == 1)
> return 1;
> else return -1;
> }
> private int compareUser(UserTime that) {
> if(id > that.id)
> return 1;
> else if(id == that.id)
> return 0;
> else return -1;
> }
> //assumes all are from the same year
> private int compareTime(UserTime that) {
> if(month > that.month ||
> (month == that.month && day > that.day) ||
> (month == that.month && day == that.day && hour > that.hour) ||
> (month == that.month && day == that.day && hour == that.hour && min >
> that.min) ||
> (month == that.month && day == that.day && hour == that.hour && min ==
> that.min && sec > that.sec))
> return 1;
> else if(month == that.month && day == that.day && hour == that.hour && min
> == that.min && sec == that.sec)
> return 0;
> else return -1;
> }
> public String toString() {
> String h, m, s;
> if(hour < 10)
> h = "0"+hour;
> else
> h = Integer.toString(hour);
> if(min < 10)
> m = "0"+min;
> else
> m = Integer.toString(hour);
> if(sec < 10)
> s = "0"+min;
> else
> s = Integer.toString(hour);
> return (id+"\t"+month+"/"+day+"/"+year+"\t"+h+":"+m+":"+s);
> }
> }
>
> Thanks for any help.
>
> Regards,
> Keith
>
--
https://github.com/zinnia-phatak-dev/Nectar