You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/08/17 07:38:53 UTC
svn commit: r986195 - in /hadoop/hive/branches/branch-0.6: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/io/
Author: namit
Date: Tue Aug 17 05:38:52 2010
New Revision: 986195
URL: http://svn.apache.org/viewvc?rev=986195&view=rev
Log:
HIVE-1543. Abort if record reader's next throws an exception
(Ning Zhang via namit)
Modified:
hadoop/hive/branches/branch-0.6/CHANGES.txt
hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java
hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
Modified: hadoop/hive/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/CHANGES.txt?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.6/CHANGES.txt Tue Aug 17 05:38:52 2010
@@ -587,6 +587,9 @@ Release 0.6.0 - Unreleased
metastore
(Pradeep Kamath via jvs)
+ HIVE-1543. Abort if record reader's next throws an exception
+ (Ning Zhang via namit)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Tue Aug 17 05:38:52 2010
@@ -51,7 +51,7 @@ public class ExecMapper extends MapReduc
private Map<String, FetchOperator> fetchOperators;
private OutputCollector oc;
private JobConf jc;
- private boolean abort = false;
+ private static boolean abort = false;
private Reporter rp;
public static final Log l4j = LogFactory.getLog("ExecMapper");
private static boolean done;
@@ -213,6 +213,9 @@ public class ExecMapper extends MapReduc
reportStats rps = new reportStats(rp);
mo.preorderMap(rps);
+
+ // reset abort flag so that ExecMapper instance can be potentially reused
+ setAbort(false);
return;
} catch (Exception e) {
if (!abort) {
@@ -231,8 +234,8 @@ public class ExecMapper extends MapReduc
return abort;
}
- public void setAbort(boolean abort) {
- this.abort = abort;
+ public static void setAbort(boolean abrt) {
+ abort = abrt;
}
public static void setDone(boolean done) {
Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveRecordReader.java Tue Aug 17 05:38:52 2010
@@ -24,11 +24,9 @@ import org.apache.hadoop.hive.ql.exec.Ex
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
/**
* BucketizedHiveRecordReader is a wrapper on a list of RecordReader. It behaves
@@ -87,12 +85,17 @@ public class BucketizedHiveRecordReader<
}
public boolean next(K key, V value) throws IOException {
- while ((curReader == null) || !curReader.next(key, value)) {
- if (!initNextRecordReader()) {
- return false;
+ try {
+ while ((curReader == null) || !curReader.next(key, value)) {
+ if (!initNextRecordReader()) {
+ return false;
+ }
}
+ return true;
+ } catch (IOException e) {
+ ExecMapper.setAbort(true);
+ throw e;
}
- return true;
}
/**
Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java Tue Aug 17 05:38:52 2010
@@ -92,6 +92,11 @@ public class CombineHiveRecordReader<K e
if (ExecMapper.getDone()) {
return false;
}
- return recordReader.next(key, value);
+ try {
+ return recordReader.next(key, value);
+ } catch (IOException e) {
+ ExecMapper.setAbort(true);
+ throw e;
+ }
}
}
Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java?rev=986195&r1=986194&r2=986195&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java Tue Aug 17 05:38:52 2010
@@ -28,7 +28,7 @@ import org.apache.hadoop.mapred.RecordRe
/**
* HiveRecordReader is a simple wrapper on RecordReader.
* It allows us to stop reading the data when some global flag
- * ExecMapper.getDone() is set.
+ * ExecMapper.getDone() is set.
*/
public class HiveRecordReader<K extends WritableComparable, V extends Writable>
implements RecordReader<K, V> {
@@ -63,6 +63,11 @@ public class HiveRecordReader<K extends
if (ExecMapper.getDone()) {
return false;
}
- return recordReader.next(key, value);
+ try {
+ return recordReader.next(key, value);
+ } catch (IOException e) {
+ ExecMapper.setAbort(true);
+ throw e;
+ }
}
}