You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2010/11/24 03:58:43 UTC
svn commit: r1038450 - in /hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/hooks/
ql/src/test/org/apache/hadoop/hive/ql/hooks/
Author: jvs
Date: Wed Nov 24 02:58:43 2010
New Revision: 1038450
URL: http://svn.apache.org/viewvc?rev=1038450&view=rev
Log:
HIVE-1785 change Pre/Post Query Hooks to take in 1 parameter: HookContext
(Liyin Tang via jvs)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/ExecuteWithHookContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Hook.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
Modified:
hive/trunk/CHANGES.txt
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecute.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecute.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java
Modified: hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/CHANGES.txt (original)
+++ hive/trunk/CHANGES.txt Wed Nov 24 02:58:43 2010
@@ -271,6 +271,9 @@ Trunk - Unreleased
HIVE-1801 HiveInputFormat or CombineHiveInputFormat always sync blocks of RCFile twice
(Siying Dong via He Yongqiang)
+ HIVE-1785 change Pre/Post Query Hooks to take in 1 parameter: HookContext
+ (Liyin Tang via jvs)
+
OPTIMIZATIONS
BUG FIXES
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Wed Nov 24 02:58:43 2010
@@ -55,6 +55,9 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.TaskRunner;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
+import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
+import org.apache.hadoop.hive.ql.hooks.Hook;
+import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.apache.hadoop.hive.ql.hooks.PostExecute;
import org.apache.hadoop.hive.ql.hooks.PreExecute;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
@@ -702,8 +705,8 @@ public class Driver implements CommandPr
return new CommandProcessorResponse(ret);
}
- private List<PreExecute> getPreExecHooks() throws Exception {
- ArrayList<PreExecute> pehooks = new ArrayList<PreExecute>();
+ private List<Hook> getPreExecHooks() throws Exception {
+ ArrayList<Hook> pehooks = new ArrayList<Hook>();
String pestr = conf.getVar(HiveConf.ConfVars.PREEXECHOOKS);
pestr = pestr.trim();
if (pestr.equals("")) {
@@ -714,7 +717,7 @@ public class Driver implements CommandPr
for (String peClass : peClasses) {
try {
- pehooks.add((PreExecute) Class.forName(peClass.trim(), true, JavaUtils.getClassLoader())
+ pehooks.add((Hook) Class.forName(peClass.trim(), true, JavaUtils.getClassLoader())
.newInstance());
} catch (ClassNotFoundException e) {
console.printError("Pre Exec Hook Class not found:" + e.getMessage());
@@ -725,8 +728,8 @@ public class Driver implements CommandPr
return pehooks;
}
- private List<PostExecute> getPostExecHooks() throws Exception {
- ArrayList<PostExecute> pehooks = new ArrayList<PostExecute>();
+ private List<Hook> getPostExecHooks() throws Exception {
+ ArrayList<Hook> pehooks = new ArrayList<Hook>();
String pestr = conf.getVar(HiveConf.ConfVars.POSTEXECHOOKS);
pestr = pestr.trim();
if (pestr.equals("")) {
@@ -737,7 +740,7 @@ public class Driver implements CommandPr
for (String peClass : peClasses) {
try {
- pehooks.add((PostExecute) Class.forName(peClass.trim(), true, JavaUtils.getClassLoader())
+ pehooks.add((Hook) Class.forName(peClass.trim(), true, JavaUtils.getClassLoader())
.newInstance());
} catch (ClassNotFoundException e) {
console.printError("Post Exec Hook Class not found:" + e.getMessage());
@@ -773,12 +776,18 @@ public class Driver implements CommandPr
}
resStream = null;
- // Get all the pre execution hooks and execute them.
- for (PreExecute peh : getPreExecHooks()) {
- peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(), ShimLoader
- .getHadoopShims().getUGIForConf(conf));
+ HookContext hookContext = new HookContext(plan, conf);
+
+ for (Hook peh : getPreExecHooks()) {
+ if (peh instanceof ExecuteWithHookContext) {
+ ((ExecuteWithHookContext) peh).run(hookContext);
+ } else if (peh instanceof PreExecute) {
+ ((PreExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
+ ShimLoader.getHadoopShims().getUGIForConf(conf));
+ }
}
+
int jobs = Utilities.getMRTasks(plan.getRootTasks()).size();
if (jobs > 0) {
console.printInfo("Total MapReduce jobs = " + jobs);
@@ -820,6 +829,7 @@ public class Driver implements CommandPr
TaskResult tskRes = pollTasks(running.keySet());
TaskRunner tskRun = running.remove(tskRes);
Task<? extends Serializable> tsk = tskRun.getTask();
+ hookContext.addCompleteTask(tskRun);
int exitVal = tskRes.getExitVal();
if (exitVal != 0) {
@@ -885,12 +895,17 @@ public class Driver implements CommandPr
}
// Get all the post execution hooks and execute them.
- for (PostExecute peh : getPostExecHooks()) {
- peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
- (SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo()
- : null), ShimLoader.getHadoopShims().getUGIForConf(conf));
+ for (Hook peh : getPostExecHooks()) {
+ if (peh instanceof ExecuteWithHookContext) {
+ ((ExecuteWithHookContext) peh).run(hookContext);
+ } else if (peh instanceof PostExecute) {
+ ((PostExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
+ (SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo()
+ : null), ShimLoader.getHadoopShims().getUGIForConf(conf));
+ }
}
+
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE,
String.valueOf(0));
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/ExecuteWithHookContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/ExecuteWithHookContext.java?rev=1038450&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/ExecuteWithHookContext.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/ExecuteWithHookContext.java Wed Nov 24 02:58:43 2010
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+/**
+ *
+ * ExecuteWithHookContext is a new interface that the Pre/Post Execute Hook can run with the HookContext.
+ *
+ */
+
+public interface ExecuteWithHookContext extends Hook {
+
+ /**
+ *
+ * @param hookContext
+ * The hook context passed to each hooks.
+ * @throws Exception
+ */
+ void run(HookContext hookContext) throws Exception;
+
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Hook.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Hook.java?rev=1038450&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Hook.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Hook.java Wed Nov 24 02:58:43 2010
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+/**
+ *
+ * The new interface for all the pre execute hooks and post execute hooks
+ *
+ */
+public interface Hook {
+
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java?rev=1038450&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java Wed Nov 24 02:58:43 2010
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
+/**
+ * Hook Context keeps all the necessary information for all the hooks.
+ * New implemented hook can get the query plan, job conf and the list of all completed tasks from this hook context
+ */
+public class HookContext {
+ private QueryPlan queryPlan;
+ private HiveConf conf;
+ private List<TaskRunner> completeTaskList;
+ private Set<ReadEntity> inputs;
+ private Set<WriteEntity> outputs;
+ private LineageInfo linfo;
+ private UserGroupInformation ugi;
+
+
+ public HookContext(QueryPlan queryPlan, HiveConf conf) throws Exception{
+ this.queryPlan = queryPlan;
+ this.conf = conf;
+ completeTaskList = new ArrayList<TaskRunner>();
+ inputs = queryPlan.getInputs();
+ outputs = queryPlan.getOutputs();
+ ugi = ShimLoader.getHadoopShims().getUGIForConf(conf);
+ linfo= null;
+ if(SessionState.get() != null){
+ linfo = SessionState.get().getLineageState().getLineageInfo();
+ }
+ }
+
+ public QueryPlan getQueryPlan() {
+ return queryPlan;
+ }
+
+ public void setQueryPlan(QueryPlan queryPlan) {
+ this.queryPlan = queryPlan;
+ }
+
+ public HiveConf getConf() {
+ return conf;
+ }
+
+ public void setConf(HiveConf conf) {
+ this.conf = conf;
+ }
+
+ public List<TaskRunner> getCompleteTaskList() {
+ return completeTaskList;
+ }
+
+ public void setCompleteTaskList(List<TaskRunner> completeTaskList) {
+ this.completeTaskList = completeTaskList;
+ }
+
+ public void addCompleteTask(TaskRunner completeTaskRunner) {
+ completeTaskList.add(completeTaskRunner);
+ }
+
+ public Set<ReadEntity> getInputs() {
+ return inputs;
+ }
+
+ public void setInputs(Set<ReadEntity> inputs) {
+ this.inputs = inputs;
+ }
+
+ public Set<WriteEntity> getOutputs() {
+ return outputs;
+ }
+
+ public void setOutputs(Set<WriteEntity> outputs) {
+ this.outputs = outputs;
+ }
+
+ public LineageInfo getLinfo() {
+ return linfo;
+ }
+
+ public void setLinfo(LineageInfo linfo) {
+ this.linfo = linfo;
+ }
+
+ public UserGroupInformation getUgi() {
+ return ugi;
+ }
+
+ public void setUgi(UserGroupInformation ugi) {
+ this.ugi = ugi;
+ }
+
+
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecute.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecute.java?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecute.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecute.java Wed Nov 24 02:58:43 2010
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.UserGr
* The post execute hook interface. A list of such hooks can be configured to be
* called after compilation and before execution.
*/
-public interface PostExecute {
+public interface PostExecute extends Hook {
/**
* The run command that is called just before the execution of the query.
@@ -43,6 +43,7 @@ public interface PostExecute {
* @param ugi
* The user group security information.
*/
+ @Deprecated
void run(SessionState sess, Set<ReadEntity> inputs,
Set<WriteEntity> outputs, LineageInfo lInfo,
UserGroupInformation ugi) throws Exception;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecute.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecute.java?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecute.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecute.java Wed Nov 24 02:58:43 2010
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.UserGr
* The pre execute hook interface. A list of such hooks can be configured to be
* called after compilation and before execution.
*/
-public interface PreExecute {
+public interface PreExecute extends Hook {
/**
* The run command that is called just before the execution of the query.
@@ -41,6 +41,7 @@ public interface PreExecute {
* @param ugi
* The user group security information.
*/
+ @Deprecated
public void run(SessionState sess, Set<ReadEntity> inputs,
Set<WriteEntity> outputs, UserGroupInformation ugi)
throws Exception;
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java Wed Nov 24 02:58:43 2010
@@ -31,9 +31,17 @@ import org.apache.hadoop.security.UserGr
* Implementation of a pre execute hook that prevents modifications
* of read-only tables used by the test framework
*/
-public class EnforceReadOnlyTables implements PreExecute {
+public class EnforceReadOnlyTables implements ExecuteWithHookContext {
@Override
+ public void run(HookContext hookContext) throws Exception {
+ SessionState ss = SessionState.get();
+ Set<ReadEntity> inputs = hookContext.getInputs();
+ Set<WriteEntity> outputs = hookContext.getOutputs();
+ UserGroupInformation ugi = hookContext.getUgi();
+ this.run(ss,inputs,outputs,ugi);
+ }
+
public void run(SessionState sess, Set<ReadEntity> inputs,
Set<WriteEntity> outputs, UserGroupInformation ugi)
throws Exception {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java Wed Nov 24 02:58:43 2010
@@ -38,7 +38,7 @@ import org.apache.hadoop.security.UserGr
* Implementation of a post execute hook that simply prints out its parameters
* to standard output.
*/
-public class PostExecutePrinter implements PostExecute {
+public class PostExecutePrinter implements ExecuteWithHookContext {
public class DependencyKeyComp implements
Comparator<Map.Entry<DependencyKey, Dependency>> {
@@ -94,6 +94,15 @@ public class PostExecutePrinter implemen
}
@Override
+ public void run(HookContext hookContext) throws Exception {
+ SessionState ss = SessionState.get();
+ Set<ReadEntity> inputs = hookContext.getInputs();
+ Set<WriteEntity> outputs = hookContext.getOutputs();
+ LineageInfo linfo = hookContext.getLinfo();
+ UserGroupInformation ugi = hookContext.getUgi();
+ this.run(ss,inputs,outputs,linfo,ugi);
+ }
+
public void run(SessionState sess, Set<ReadEntity> inputs,
Set<WriteEntity> outputs, LineageInfo linfo,
UserGroupInformation ugi) throws Exception {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java?rev=1038450&r1=1038449&r2=1038450&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java Wed Nov 24 02:58:43 2010
@@ -31,9 +31,17 @@ import org.apache.hadoop.security.UserGr
* Implementation of a pre execute hook that simply prints out its parameters to
* standard output.
*/
-public class PreExecutePrinter implements PreExecute {
+public class PreExecutePrinter implements ExecuteWithHookContext {
@Override
+ public void run(HookContext hookContext) throws Exception {
+ SessionState ss = SessionState.get();
+ Set<ReadEntity> inputs = hookContext.getInputs();
+ Set<WriteEntity> outputs = hookContext.getOutputs();
+ UserGroupInformation ugi = hookContext.getUgi();
+ this.run(ss,inputs,outputs,ugi);
+ }
+
public void run(SessionState sess, Set<ReadEntity> inputs,
Set<WriteEntity> outputs, UserGroupInformation ugi)
throws Exception {