You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/09/18 08:33:01 UTC
svn commit: r998415 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
Author: namit
Date: Sat Sep 18 06:33:01 2010
New Revision: 998415
URL: http://svn.apache.org/viewvc?rev=998415&view=rev
Log:
HIVE-1617. ScriptOperator's AutoProgressor can lead to an infinite loop
(Paul Yang via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Sat Sep 18 06:33:01 2010
@@ -132,6 +132,9 @@ Trunk - Unreleased
HIVE-1645. ability to specify parent directory for zookeeper lock manager
(namit via He Yongqiang)
+ HIVE-1617. ScriptOperator's AutoProgressor can lead to an infinite loop
+ (Paul Yang via namit)
+
HIVE-1226. support filter pushdown against non-native tables
(jvs via He Yongqiang)
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Sep 18 06:33:01 2010
@@ -182,6 +182,7 @@ public class HiveConf extends Configurat
HIVEADDEDARCHIVES("hive.added.archives.path", ""),
// for hive script operator
+ HIVES_AUTO_PROGRESS_TIMEOUT("hive.auto.progress.timeout", 0),
HIVETABLENAME("hive.table.name", ""),
HIVEPARTITIONNAME("hive.partition.name", ""),
HIVESCRIPTAUTOPROGRESS("hive.script.auto.progress", false),
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Sat Sep 18 06:33:01 2010
@@ -695,6 +695,14 @@
</description>
</property>
+<property>
+ <name>hive.auto.progress.timeout</name>
+ <value>0</value>
+ <description>
+ How long to run autoprogressor for the script/UDTF operators (in seconds).
+ Set to 0 for forever.
+ </description>
+</property>
<!-- HBase Storage Handler Parameters -->
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AutoProgressor.java Sat Sep 18 06:33:01 2010
@@ -28,7 +28,8 @@ import org.apache.hadoop.mapred.Reporter
/**
* AutoProgressor periodically sends updates to the job tracker so that it
* doesn't consider this task attempt dead if there is a long period of
- * inactivity.
+ * inactivity. This can be configured with a timeout so that it doesn't run
+ * indefinitely.
*/
public class AutoProgressor {
protected Log LOG = LogFactory.getLog(this.getClass().getName());
@@ -38,9 +39,12 @@ public class AutoProgressor {
// duration, a progress report is sent to the tracker so that the tracker
// does not think that the job is dead.
Timer rpTimer = null;
+ // Timer that tops rpTimer after a long timeout, e.g. 1 hr
+ Timer srpTimer = null;
// Name of the class to report for
String logClassName = null;
int notificationInterval;
+ int timeout;
Reporter reporter;
class ReporterTask extends TimerTask {
@@ -70,18 +74,68 @@ public class AutoProgressor {
}
}
+ class StopReporterTimerTask extends TimerTask {
+
+ /**
+ * Task to stop the reporter timer once we hit the timeout
+ */
+ private final ReporterTask rt;
+
+ public StopReporterTimerTask(ReporterTask rp) {
+ this.rt = rp;
+ }
+
+ @Override
+ public void run() {
+ if (rt != null) {
+ LOG.info("Stopping reporter timer for " + logClassName);
+ rt.cancel();
+ }
+ }
+ }
+
+ /**
+ *
+ * @param logClassName
+ * @param reporter
+ * @param notificationInterval - interval for reporter updates (in ms)
+ */
AutoProgressor(String logClassName, Reporter reporter,
int notificationInterval) {
this.logClassName = logClassName;
this.reporter = reporter;
this.notificationInterval = notificationInterval;
+ this.timeout = 0;
+ }
+
+ /**
+ *
+ * @param logClassName
+ * @param reporter
+ * @param notificationInterval - interval for reporter updates (in ms)
+ * @param timeout - when the autoprogressor should stop reporting (in ms)
+ */
+ AutoProgressor(String logClassName, Reporter reporter,
+ int notificationInterval, int timeout) {
+ this.logClassName = logClassName;
+ this.reporter = reporter;
+ this.notificationInterval = notificationInterval;
+ this.timeout = timeout;
}
public void go() {
LOG.info("Running ReporterTask every " + notificationInterval
+ " miliseconds.");
rpTimer = new Timer(true);
- rpTimer.scheduleAtFixedRate(new ReporterTask(reporter), 0,
- notificationInterval);
+
+
+ ReporterTask rt = new ReporterTask(reporter);
+ rpTimer.scheduleAtFixedRate(rt, 0, notificationInterval);
+
+ if (timeout > 0) {
+ srpTimer = new Timer(true);
+ StopReporterTimerTask srt = new StopReporterTimerTask(rt);
+ srpTimer.schedule(srt, timeout);
+ }
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java Sat Sep 18 06:33:01 2010
@@ -309,7 +309,8 @@ public class ScriptOperator extends Oper
if (HiveConf
.getBoolVar(hconf, HiveConf.ConfVars.HIVESCRIPTAUTOPROGRESS)) {
autoProgressor = new AutoProgressor(this.getClass().getName(),
- reporter, Utilities.getDefaultNotificationInterval(hconf));
+ reporter, Utilities.getDefaultNotificationInterval(hconf),
+ HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT) * 1000);
autoProgressor.go();
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java?rev=998415&r1=998414&r2=998415&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java Sat Sep 18 06:33:01 2010
@@ -76,7 +76,8 @@ public class UDTFOperator extends Operat
// for a while
if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEUDTFAUTOPROGRESS)) {
autoProgressor = new AutoProgressor(this.getClass().getName(), reporter,
- Utilities.getDefaultNotificationInterval(hconf));
+ Utilities.getDefaultNotificationInterval(hconf),
+ HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT) * 1000);
autoProgressor.go();
}