Author: namit
Date: Fri Jan 29 19:19:21 2010
New Revision: 904599
URL: http://svn.apache.org/viewvc?rev=904599&view=rev
Log:
HIVE-1092. Add job counter for conditional tasks
(He Yongqiang via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=904599&r1=904598&r2=904599&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Jan 29 19:19:21 2010
@@ -101,6 +101,9 @@
HIVE-1110. Add a counter to denote that skewjoin got triggered
(He Yongqiang via namit)
+ HIVE-1092. Add job counter for conditional tasks
+ (He Yongqiang via namit)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
@@ -1550,4 +1553,4 @@
HIVE-1087. Let user script write out binary data into a table
(Zheng Shao via Ning Zhang)
-
+
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=904599&r1=904598&r2=904599&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Jan 29 19:19:21 2010
@@ -527,8 +527,7 @@
// Launch upto maxthreads tasks
while (runnable.peek() != null && running.size() < maxthreads) {
Task<? extends Serializable> tsk = runnable.remove();
- curJobNo = launchTask(tsk, queryId, noName, running, jobname, jobs,
- curJobNo, driverCxt);
+ launchTask(tsk, queryId, noName, running, jobname, jobs, driverCxt);
}
// poll the Tasks to see which one completed
@@ -628,9 +627,9 @@
* @return the updated number of last the map-reduce job launched
*/
- public int launchTask(Task<? extends Serializable> tsk, String queryId,
+ public void launchTask(Task<? extends Serializable> tsk, String queryId,
boolean noName, Map<TaskResult, TaskRunner> running, String jobname,
- int jobs, int curJobNo, DriverContext cxt) {
+ int jobs, DriverContext cxt) {
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().startTask(queryId, tsk,
@@ -641,8 +640,8 @@
conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname + "("
+ tsk.getId() + ")");
}
- curJobNo++;
- console.printInfo("Launching Job " + curJobNo + " out of " + jobs);
+ cxt.incCurJobNo(1);
+ console.printInfo("Launching Job " + cxt.getCurJobNo() + " out of " + jobs);
}
tsk.initialize(conf, plan, cxt);
TaskResult tskRes = new TaskResult();
@@ -657,7 +656,7 @@
tskRun.runSequential();
}
running.put(tskRes, tskRun);
- return curJobNo;
+ return;
}
/**
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java?rev=904599&r1=904598&r2=904599&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java Fri Jan 29 19:19:21 2010
@@ -27,6 +27,9 @@
public class DriverContext {
Queue<Task<? extends Serializable>> runnable = new LinkedList<Task<? extends Serializable>>();
+
+ // how many jobs have been started
+ int curJobNo;
public DriverContext(Queue<Task<? extends Serializable>> runnable) {
this.runnable = runnable;
@@ -55,4 +58,12 @@
tsk.setQueued();
}
+ public int getCurJobNo() {
+ return curJobNo;
+ }
+
+ public void incCurJobNo(int amount) {
+ this.curJobNo = this.curJobNo + amount;
+ }
+
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java?rev=904599&r1=904598&r2=904599&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java Fri Jan 29 19:19:21 2010
@@ -86,6 +86,7 @@
console.printInfo(ExecDriver.getJobEndMsg(""
+ Utilities.randGen.nextInt())
+ ", job is filtered out (removed at runtime).");
+ driverContext.incCurJobNo(1);
if (tsk.getChildTasks() != null) {
for (Task<? extends Serializable> child : tsk.getChildTasks()) {
child.parentTasks.remove(tsk);