You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/20 05:33:17 UTC
svn commit: r1470101 - in /hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/profiler/
ql/src/test/org/apache/hadoop/hive/ql/exec/ ql/src/test/org/apache/...
Author: hashutosh
Date: Sat Apr 20 03:33:16 2013
New Revision: 1470101
URL: http://svn.apache.org/r1470101
Log:
HIVE-4318 : OperatorHooks hit performance even when not used (Gunther Hagleitner via Ashutosh Chauhan)
Removed:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java
hive/trunk/ql/src/test/queries/clientpositive/hiveprofiler0.q
hive/trunk/ql/src/test/queries/clientpositive/hiveprofiler_script0.q
hive/trunk/ql/src/test/queries/clientpositive/hiveprofiler_union0.q
hive/trunk/ql/src/test/queries/clientpositive/operatorhook.q
hive/trunk/ql/src/test/results/clientpositive/hiveprofiler0.q.out
hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_script0.q.out
hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_union0.q.out
hive/trunk/ql/src/test/results/clientpositive/operatorhook.q.out
Modified:
hive/trunk/build-common.xml
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1470101&r1=1470100&r2=1470101&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Sat Apr 20 03:33:16 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,operatorhook.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1470101&r1=1470100&r2=1470101&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Apr 20 03:33:16 2013
@@ -190,7 +190,6 @@ public class HiveConf extends Configurat
PREEXECHOOKS("hive.exec.pre.hooks", ""),
POSTEXECHOOKS("hive.exec.post.hooks", ""),
ONFAILUREHOOKS("hive.exec.failure.hooks", ""),
- OPERATORHOOKS("hive.exec.operator.hooks", ""),
CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", ""),
EXECPARALLEL("hive.exec.parallel", false), // parallel query launching
EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8),
@@ -565,18 +564,6 @@ public class HiveConf extends Configurat
HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024 * 1024 * 1024), // 10G
HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true),
- //Profiler
- HIVEPROFILERDBCLASS("hive.profiler.dbclass","jdbc:derby"),
- HIVEPROFILERJDBCDRIVER("hive.profiler.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"),
- HIVEPROFILERDBCONNECTIONSTRING("hive.profiler.dbconnectionstring",
- "jdbc:derby:;databaseName=TempProfilerStore;create=true"), // automatically create database
- // default timeout for JDBC connection
- HIVE_PROFILER_JDBC_TIMEOUT("hive.profiler.jdbc.timeout", 30),
- HIVE_PROFILER_RETRIES_MAX("hive.stats.retries.max",
- 0), // maximum # of retries to insert/select/delete the stats DB
- HIVE_PROFILER_RETRIES_WAIT("hive.stats.retries.wait",
- 3000), // # milliseconds to wait before the next retry
-
// Statistics
HIVESTATSAUTOGATHER("hive.stats.autogather", true),
HIVESTATSDBCLASS("hive.stats.dbclass",
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1470101&r1=1470100&r2=1470101&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Sat Apr 20 03:33:16 2013
@@ -1123,42 +1123,6 @@
</property>
<property>
- <name>hive.profiler.dbclass</name>
- <value>jdbc:derby</value>
- <description>The default database that stores temporary hive statistics.</description>
-</property>
-
-<property>
- <name>hive.profiler.jdbcdriver</name>
- <value>org.apache.derby.jdbc.EmbeddedDriver</value>
- <description>The JDBC driver for the database that stores temporary hive statistics.</description>
-</property>
-
-<property>
- <name>hive.profiler.dbconnectionstring</name>
- <value>jdbc:derby:;databaseName=TempStatsStore;create=true</value>
- <description>The default connection string for the database that stores temporary hive statistics.</description>
-</property>
-
-<property>
- <name>hive.profiler.jdbc.timeout</name>
- <value>30</value>
- <description>Timeout value (number of seconds) used by JDBC connection and statements.</description>
-</property>
-
-<property>
- <name>hive.profiler.retries.max</name>
- <value>0</value>
- <description>Maximum number of retries when profiler publisher/aggregator got an exception updating intermediate database. Default is no tries on failures.</description>
-</property>
-
-<property>
- <name>hive.profiler.retries.wait</name>
- <value>3000</value>
- <description>The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues baseWindow * (failure 1) * (random number between [0.0,1.0]).</description>
-</property>
-
-<property>
<name>hive.stats.dbclass</name>
<value>jdbc:derby</value>
<description>The default database that stores temporary hive statistics.</description>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=1470101&r1=1470100&r2=1470101&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Sat Apr 20 03:33:16 2013
@@ -50,7 +50,6 @@ public class ExecMapper extends MapReduc
private JobConf jc;
private boolean abort = false;
private Reporter rp;
- private List<OperatorHook> opHooks;
public static final Log l4j = LogFactory.getLog("ExecMapper");
private static boolean done;
@@ -99,7 +98,6 @@ public class ExecMapper extends MapReduc
mo.setExecContext(execContext);
mo.initializeLocalWork(jc);
mo.initialize(jc, null);
- opHooks = OperatorHookUtils.getOperatorHooks(jc);
if (localWork == null) {
return;
@@ -132,7 +130,6 @@ public class ExecMapper extends MapReduc
rp = reporter;
mo.setOutputCollector(oc);
mo.setReporter(rp);
- mo.setOperatorHooks(opHooks);
MapredContext.get().setReporter(reporter);
}
// reset the execContext for each new row
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java?rev=1470101&r1=1470100&r2=1470101&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java Sat Apr 20 03:33:16 2013
@@ -66,7 +66,6 @@ public class ExecReducer extends MapRedu
private long nextCntr = 1;
private static String[] fieldNames;
- private List<OperatorHook> opHooks;
public static final Log l4j = LogFactory.getLog("ExecReducer");
private boolean isLogInfoEnabled = false;
@@ -152,7 +151,6 @@ public class ExecReducer extends MapRedu
try {
l4j.info(reducer.dump(0));
reducer.initialize(jc, rowObjectInspector);
- opHooks = OperatorHookUtils.getOperatorHooks(jc);
} catch (Throwable e) {
abort = true;
if (e instanceof OutOfMemoryError) {
@@ -183,7 +181,6 @@ public class ExecReducer extends MapRedu
rp = reporter;
reducer.setOutputCollector(oc);
reducer.setReporter(rp);
- reducer.setOperatorHooks(opHooks);
MapredContext.get().setReporter(reporter);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1470101&r1=1470100&r2=1470101&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Sat Apr 20 03:33:16 2013
@@ -55,7 +55,6 @@ public abstract class Operator<T extends
// Bean methods
private static final long serialVersionUID = 1L;
- List<OperatorHook> operatorHooks;
private Configuration configuration;
protected List<Operator<? extends OperatorDesc>> childOperators;
@@ -241,17 +240,6 @@ public abstract class Operator<T extends
return id;
}
- public void setOperatorHooks(List<OperatorHook> opHooks){
- operatorHooks = opHooks;
- if (childOperators == null) {
- return;
- }
-
- for (Operator<? extends OperatorDesc> op : childOperators) {
- op.setOperatorHooks(opHooks);
- }
- }
-
public void setReporter(Reporter rep) {
reporter = rep;
@@ -436,34 +424,6 @@ public abstract class Operator<T extends
}
}
- private void enterOperatorHooks(OperatorHookContext opHookContext) throws HiveException {
- if (this.operatorHooks == null) {
- return;
- }
- for(OperatorHook opHook : this.operatorHooks) {
- opHook.enter(opHookContext);
- }
- }
-
- private void exitOperatorHooks(OperatorHookContext opHookContext) throws HiveException {
- if (this.operatorHooks == null) {
- return;
- }
- for(OperatorHook opHook : this.operatorHooks) {
- opHook.exit(opHookContext);
- }
- }
-
- private void closeOperatorHooks(OperatorHookContext opHookContext) throws HiveException {
- if (this.operatorHooks == null) {
- return;
- }
- for(OperatorHook opHook : this.operatorHooks) {
- opHook.close(opHookContext);
- }
- }
-
-
/**
* Collects all the parent's output object inspectors and calls actual
* initialization method.
@@ -536,16 +496,10 @@ public abstract class Operator<T extends
}
beginTime = System.currentTimeMillis();
- OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
- enterOperatorHooks(opHookContext);
processOp(row, tag);
- exitOperatorHooks(opHookContext);
totalTime += (System.currentTimeMillis() - beginTime);
} else {
- OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
- enterOperatorHooks(opHookContext);
processOp(row, tag);
- exitOperatorHooks(opHookContext);
}
}
@@ -630,7 +584,6 @@ public abstract class Operator<T extends
LOG.info(id + " forwarded " + cntr + " rows");
- closeOperatorHooks(new OperatorHookContext(this));
// call the operator specific close routine
closeOp(abort);