You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2011/08/11 06:18:18 UTC
svn commit: r1156480 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
conf/hive-default.xml ql/src/java/org/apache/hadoop/hive/ql/Driver.java
ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
Author: nzhang
Date: Thu Aug 11 04:18:17 2011
New Revision: 1156480
URL: http://svn.apache.org/viewvc?rev=1156480&view=rev
Log:
HIVE-2346. Add hooks to run when execution fails. (Kevin Wilfong via Ning Zhang)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1156480&r1=1156479&r2=1156480&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Aug 11 04:18:17 2011
@@ -122,6 +122,7 @@ public class HiveConf extends Configurat
MAXREDUCERS("hive.exec.reducers.max", 999),
PREEXECHOOKS("hive.exec.pre.hooks", ""),
POSTEXECHOOKS("hive.exec.post.hooks", ""),
+ ONFAILUREHOOKS("hive.exec.failure.hooks", ""),
EXECPARALLEL("hive.exec.parallel", false), // parallel query launching
EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8),
HIVESPECULATIVEEXECREDUCERS("hive.mapred.reduce.tasks.speculative.execution", true),
Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1156480&r1=1156479&r2=1156480&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Thu Aug 11 04:18:17 2011
@@ -533,6 +533,12 @@
</property>
<property>
+ <name>hive.exec.failure.hooks</name>
+ <value></value>
+ <description>Comma-separated list of on-failure hooks to be invoked for each statement. An on-failure hook is specified as the name of Java class which implements the org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext interface.</description>
+</property>
+
+<property>
<name>hive.merge.mapfiles</name>
<value>true</value>
<description>Merge small files at the end of a map-only job</description>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1156480&r1=1156479&r2=1156480&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Thu Aug 11 04:18:17 2011
@@ -24,7 +24,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Stack;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -357,7 +356,7 @@ public class Driver implements CommandPr
public boolean isInitialized() {
return this.init;
}
-
+
public HiveOperation getOp() {
return this.op;
}
@@ -976,7 +975,7 @@ public class Driver implements CommandPr
private List<Hook> getPreExecHooks() throws Exception {
- ArrayList<Hook> pehooks = new ArrayList<Hook>();
+ List<Hook> pehooks = new ArrayList<Hook>();
String pestr = conf.getVar(HiveConf.ConfVars.PREEXECHOOKS);
pestr = pestr.trim();
if (pestr.equals("")) {
@@ -999,7 +998,7 @@ public class Driver implements CommandPr
}
private List<Hook> getPostExecHooks() throws Exception {
- ArrayList<Hook> pehooks = new ArrayList<Hook>();
+ List<Hook> pehooks = new ArrayList<Hook>();
String pestr = conf.getVar(HiveConf.ConfVars.POSTEXECHOOKS);
pestr = pestr.trim();
if (pestr.equals("")) {
@@ -1021,6 +1020,29 @@ public class Driver implements CommandPr
return pehooks;
}
+ private List<Hook> getOnFailureHooks() throws Exception {
+ List<Hook> ofhooks = new ArrayList<Hook>();
+ String ofstr = conf.getVar(HiveConf.ConfVars.ONFAILUREHOOKS);
+ ofstr = ofstr.trim();
+ if (ofstr.equals("")) {
+ return ofhooks;
+ }
+
+ String[] ofClasses = ofstr.split(",");
+
+ for (String ofClass : ofClasses) {
+ try {
+ ofhooks.add((Hook) Class.forName(ofClass.trim(), true, JavaUtils.getClassLoader())
+ .newInstance());
+ } catch (ClassNotFoundException e) {
+ console.printError("On Failure Hook Class not found:" + e.getMessage());
+ throw e;
+ }
+ }
+
+ return ofhooks;
+ }
+
public int execute() throws CommandNeedRetryException {
Utilities.PerfLogBegin(LOG, "Driver.execute");
@@ -1139,6 +1161,16 @@ public class Driver implements CommandPr
continue;
} else {
+ hookContext.setHookType(HookContext.HookType.ON_FAILURE_HOOK);
+ // Get all the failure execution hooks and execute them.
+ for (Hook ofh : getOnFailureHooks()) {
+ Utilities.PerfLogBegin(LOG, "FailureHook." + ofh.getClass().getSimpleName());
+
+ ((ExecuteWithHookContext) ofh).run(hookContext);
+
+ Utilities.PerfLogEnd(LOG, "FailureHook." + ofh.getClass().getSimpleName());
+ }
+
// TODO: This error messaging is not very informative. Fix that.
errorMessage = "FAILED: Execution Error, return code " + exitVal + " from "
+ tsk.getClass().getName();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java?rev=1156480&r1=1156479&r2=1156480&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java Thu Aug 11 04:18:17 2011
@@ -39,7 +39,7 @@ import org.apache.hadoop.security.UserGr
public class HookContext {
static public enum HookType {
- PRE_EXEC_HOOK, POST_EXEC_HOOK
+ PRE_EXEC_HOOK, POST_EXEC_HOOK, ON_FAILURE_HOOK
}
private QueryPlan queryPlan;