You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/07/18 08:36:19 UTC
svn commit: r1362782 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
conf/hive-default.xml.template
ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
Author: namit
Date: Wed Jul 18 06:36:19 2012
New Revision: 1362782
URL: http://svn.apache.org/viewvc?rev=1362782&view=rev
Log:
HIVE-3230 Make logging of plan progress in HadoopJobExecHelper configurable
(Kevin Wilfong via namit)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1362782&r1=1362781&r2=1362782&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Jul 18 06:36:19 2012
@@ -387,6 +387,12 @@ public class HiveConf extends Configurat
//Location of Hive run time structured log file
HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/" + System.getProperty("user.name")),
+ // Whether to log the plan's progress every time a job's progress is checked
+ HIVE_LOG_INCREMENTAL_PLAN_PROGRESS("hive.querylog.enable.plan.progress", true),
+
+ // The interval between logging the plan's progress in milliseconds
+ HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL("hive.querylog.plan.progress.interval", 60000L),
+
// Default serde and record reader for user scripts
HIVESCRIPTSERDE("hive.script.serde", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"),
HIVESCRIPTRECORDREADER("hive.script.recordreader",
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1362782&r1=1362781&r2=1362782&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Wed Jul 18 06:36:19 2012
@@ -708,6 +708,38 @@
</property>
<property>
+ <name>hive.querylog.location</name>
+ <value>/tmp/${user.name}</value>
+ <description>
+ Location of Hive run time structured log file
+ </description>
+</property>
+
+<property>
+ <name>hive.querylog.enable.plan.progress</name>
+ <value>true</value>
+ <description>
+ Whether to log the plan's progress every time a job's progress is checked.
+ These logs are written to the location specified by hive.querylog.location
+ </description>
+</property>
+
+<property>
+ <name>hive.querylog.plan.progress.interval</name>
+ <value>60000</value>
+ <description>
+ The interval to wait between logging the plan's progress in milliseconds.
+ If there is a whole number percentage change in the progress of the mappers or the reducers,
+ the progress is logged regardless of this value.
+ The actual interval will be the ceiling of (this value divided by the value of
+ hive.exec.counters.pull.interval) multiplied by the value of hive.exec.counters.pull.interval
+ I.e. if it is not divide evenly by the value of hive.exec.counters.pull.interval it will be
+ logged less frequently than specified.
+ This only has an effect if hive.querylog.enable.plan.progress is set to true.
+ </description>
+</property>
+
+<property>
<name>hive.enforce.bucketing</name>
<value>false</value>
<description>Whether bucketing is enforced. If true, while inserting into the table, bucketing is enforced. </description>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java?rev=1362782&r1=1362781&r2=1362782&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java Wed Jul 18 06:36:19 2012
@@ -221,7 +221,8 @@ public class HadoopJobExecHelper {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
//DecimalFormat longFormatter = new DecimalFormat("###,###");
long reportTime = System.currentTimeMillis();
- long maxReportInterval = 60 * 1000; // One minute
+ long maxReportInterval =
+ HiveConf.getLongVar(job, HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL);
boolean fatal = false;
StringBuilder errMsg = new StringBuilder();
long pullInterval = HiveConf.getLongVar(job, HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL);
@@ -355,8 +356,10 @@ public class HadoopJobExecHelper {
ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs);
ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
Keys.TASK_HADOOP_PROGRESS, output);
- ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
- this.callBackObj.logPlanProgress(ss);
+ if (ss.getConf().getBoolVar(HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS)) {
+ ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
+ this.callBackObj.logPlanProgress(ss);
+ }
}
console.printInfo(output);
lastReport = report;