You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/03/16 17:07:08 UTC
svn commit: r1301610 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/test/org/apache/hadoop/hive/ql/hooks/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: namit
Date: Fri Mar 16 16:07:07 2012
New Revision: 1301610
URL: http://svn.apache.org/viewvc?rev=1301610&view=rev
Log:
HIVE-2871 Add a new hook to run at the beginning and end of the Driver.run method
(Kevin Wilfong via namit)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHook.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContextImpl.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyHooksRunInOrder.java
hive/trunk/ql/src/test/queries/clientpositive/hook_order.q
hive/trunk/ql/src/test/results/clientpositive/hook_order.q.out
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1301610&r1=1301609&r2=1301610&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Mar 16 16:07:07 2012
@@ -577,6 +577,9 @@ public class HiveConf extends Configurat
// whether insert into external tables is allowed
HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true),
+ // A comma separated list of hooks which implement HiveDriverRunHook and will be run at the
+ // beginning and end of Driver.run, these will be run in the order specified
+ HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", ""),
;
public final String varname;
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1301610&r1=1301609&r2=1301610&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Fri Mar 16 16:07:07 2012
@@ -1251,4 +1251,12 @@
<description>The default filesystem and jobtracker for a region</description>
</property>
+<property>
+ <name>hive.exec.driver.run.hooks</name>
+ <value></value>
+ <description>A comma separated list of hooks which implement HiveDriverRunHook and will be run at the
+ beginning and end of Driver.run, these will be run in the order specified
+ </description>
+</property>
+
</configuration>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1301610&r1=1301609&r2=1301610&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Mar 16 16:07:07 2012
@@ -69,8 +69,8 @@ import org.apache.hadoop.hive.ql.lockmgr
import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject;
-import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
import org.apache.hadoop.hive.ql.lockmgr.LockException;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
@@ -885,6 +885,23 @@ public class Driver implements CommandPr
public CommandProcessorResponse run(String command) throws CommandNeedRetryException {
errorMessage = null;
SQLState = null;
+
+ HiveDriverRunHookContext hookContext = new HiveDriverRunHookContextImpl(conf);
+ // Get all the driver run hooks and pre-execute them.
+ List<HiveDriverRunHook> driverRunHooks;
+ try {
+ driverRunHooks = getHooks(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS, HiveDriverRunHook.class);
+ for (HiveDriverRunHook driverRunHook : driverRunHooks) {
+ driverRunHook.preDriverRun(hookContext);
+ }
+ } catch (Exception e) {
+ errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
+ SQLState = ErrorMsg.findSQLState(e.getMessage());
+ console.printError(errorMessage + "\n"
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ return new CommandProcessorResponse(12, errorMessage, SQLState);
+ }
+
// Reset the perf logger
PerfLogger perfLogger = PerfLogger.getPerfLogger(true);
perfLogger.PerfLogBegin(LOG, PerfLogger.DRIVER_RUN);
@@ -943,6 +960,20 @@ public class Driver implements CommandPr
perfLogger.PerfLogEnd(LOG, PerfLogger.DRIVER_RUN);
perfLogger.close(LOG, plan);
+
+ // Take all the driver run hooks and post-execute them.
+ try {
+ for (HiveDriverRunHook driverRunHook : driverRunHooks) {
+ driverRunHook.postDriverRun(hookContext);
+ }
+ } catch (Exception e) {
+ errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
+ SQLState = ErrorMsg.findSQLState(e.getMessage());
+ console.printError(errorMessage + "\n"
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ return new CommandProcessorResponse(12, errorMessage, SQLState);
+ }
+
return new CommandProcessorResponse(ret);
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHook.java?rev=1301610&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHook.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHook.java Fri Mar 16 16:07:07 2012
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import org.apache.hadoop.hive.ql.hooks.Hook;
+
+/**
+ * HiveDriverRunHook allows Hive to be extended with custom
+ * logic for processing commands.
+ *
+ *<p>
+ *
+ * Note that the lifetime of an instantiated hook object is scoped to
+ * the analysis of a single statement; hook instances are never reused.
+ */
+public interface HiveDriverRunHook extends Hook {
+ /**
+ * Invoked before Hive begins any processing of a command in the Driver,
+ * notably before compilation and any customizable performance logging.
+ */
+ public void preDriverRun(
+ HiveDriverRunHookContext hookContext) throws Exception;
+
+ /**
+ * Invoked after Hive performs any processing of a command, just before a
+ * response is returned to the entity calling the Driver.
+ */
+ public void postDriverRun(
+ HiveDriverRunHookContext hookContext) throws Exception;
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContext.java?rev=1301610&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContext.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContext.java Fri Mar 16 16:07:07 2012
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * Context information provided by Hive to implementations of
+ * HiveDriverRunHook.
+ */
+public interface HiveDriverRunHookContext extends Configurable{
+
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContextImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContextImpl.java?rev=1301610&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContextImpl.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/HiveDriverRunHookContextImpl.java Fri Mar 16 16:07:07 2012
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class HiveDriverRunHookContextImpl implements HiveDriverRunHookContext {
+
+ Configuration conf;
+
+ public HiveDriverRunHookContextImpl(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyHooksRunInOrder.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyHooksRunInOrder.java?rev=1301610&r1=1301609&r2=1301610&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyHooksRunInOrder.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyHooksRunInOrder.java Fri Mar 16 16:07:07 2012
@@ -22,6 +22,8 @@ import java.util.List;
import junit.framework.Assert;
+import org.apache.hadoop.hive.ql.HiveDriverRunHook;
+import org.apache.hadoop.hive.ql.HiveDriverRunHookContext;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.HookContext.HookType;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -47,6 +49,8 @@ public class VerifyHooksRunInOrder {
private static boolean postHookRunFirstRan = false;
private static boolean staticAnalysisPreHookFirstRan = false;
private static boolean staticAnalysisPostHookFirstRan = false;
+ private static boolean driverRunPreHookFirstRan = false;
+ private static boolean driverRunPostHookFirstRan = false;
public static class RunFirst implements ExecuteWithHookContext {
public void run(HookContext hookContext) {
@@ -155,4 +159,70 @@ public class VerifyHooksRunInOrder {
staticAnalysisPostHookFirstRan);
}
}
+
+ public static class RunFirstDriverRunHook implements HiveDriverRunHook {
+
+ @Override
+ public void preDriverRun(HiveDriverRunHookContext hookContext) throws Exception {
+ LogHelper console = SessionState.getConsole();
+
+ if (console == null) {
+ return;
+ }
+
+ // This is simply to verify that the hooks were in fact run
+ console.printError("Running RunFirst for Pre Driver Run Hook");
+
+ driverRunPreHookFirstRan = true;
+ }
+
+ @Override
+ public void postDriverRun(HiveDriverRunHookContext hookContext) throws Exception {
+ LogHelper console = SessionState.getConsole();
+
+ if (console == null) {
+ return;
+ }
+
+ // This is simply to verify that the hooks were in fact run
+ console.printError("Running RunFirst for Post Driver Run Hook");
+
+ driverRunPostHookFirstRan = true;
+ }
+
+ }
+
+ public static class RunSecondDriverRunHook implements HiveDriverRunHook {
+
+ @Override
+ public void preDriverRun(HiveDriverRunHookContext hookContext) throws Exception {
+ LogHelper console = SessionState.getConsole();
+
+ if (console == null) {
+ return;
+ }
+
+ // This is simply to verify that the hooks were in fact run
+ console.printError("Running RunSecond for Pre Driver Run Hook");
+
+ Assert.assertTrue("Driver Run Hooks did not run in the order specified.",
+ driverRunPreHookFirstRan);
+ }
+
+ @Override
+ public void postDriverRun(HiveDriverRunHookContext hookContext) throws Exception {
+ LogHelper console = SessionState.getConsole();
+
+ if (console == null) {
+ return;
+ }
+
+ // This is simply to verify that the hooks were in fact run
+ console.printError("Running RunSecond for Post Driver Run Hook");
+
+ Assert.assertTrue("Driver Run Hooks did not run in the order specified.",
+ driverRunPostHookFirstRan);
+ }
+
+ }
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/hook_order.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/hook_order.q?rev=1301610&r1=1301609&r2=1301610&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/hook_order.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/hook_order.q Fri Mar 16 16:07:07 2012
@@ -1,9 +1,11 @@
SET hive.exec.pre.hooks=org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunFirst,org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunSecond;
SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunFirst,org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunSecond;
SET hive.semantic.analyzer.hook=org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunFirstSemanticAnalysisHook,org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunSecondSemanticAnalysisHook;
+SET hive.exec.driver.run.hooks=org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunFirstDriverRunHook,org.apache.hadoop.hive.ql.hooks.VerifyHooksRunInOrder$RunSecondDriverRunHook;
SELECT count(*) FROM src;
SET hive.exec.pre.hooks=;
SET hive.exec.post.hooks=;
SET hive.semantic.analyzer.hook=;
+SET hive.exec.driver.run.hooks=;
Modified: hive/trunk/ql/src/test/results/clientpositive/hook_order.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/hook_order.q.out?rev=1301610&r1=1301609&r2=1301610&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/hook_order.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/hook_order.q.out Fri Mar 16 16:07:07 2012
@@ -1,3 +1,5 @@
+Running RunFirst for Pre Driver Run Hook
+Running RunSecond for Pre Driver Run Hook
Running RunFirst for Pre Analysis Hook
Running RunSecond for Pre Analysis Hook
Running RunFirst for Post Analysis Hook
@@ -6,4 +8,6 @@ Running RunFirst for PRE_EXEC_HOOK
Running RunSecond for PRE_EXEC_HOOK
Running RunFirst for POST_EXEC_HOOK
Running RunSecond for POST_EXEC_HOOK
+Running RunFirst for Post Driver Run Hook
+Running RunSecond for Post Driver Run Hook
500