You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/20 05:17:08 UTC
svn commit: r1470100 - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec:
FileSinkOperator.java GroupByOperator.java Operator.java
Author: hashutosh
Date: Sat Apr 20 03:17:08 2013
New Revision: 1470100
URL: http://svn.apache.org/r1470100
Log:
HIVE-4378 : Counters hit performance even when not used (Gunther Hagleitner via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1470100&r1=1470099&r2=1470100&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Sat Apr 20 03:17:08 2013
@@ -755,7 +755,9 @@ public class FileSinkOperator extends Te
// check # of dp
if (valToPaths.size() > maxPartitions) {
// throw fatal error
- incrCounter(fatalErrorCntr, 1);
+ if (counterNameToEnum != null) {
+ incrCounter(fatalErrorCntr, 1);
+ }
fatalError = true;
LOG.error("Fatal error was thrown due to exceeding number of dynamic partitions");
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=1470100&r1=1470099&r2=1470100&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Sat Apr 20 03:17:08 2013
@@ -1076,7 +1076,7 @@ public class GroupByOperator extends Ope
try {
// put the hash related stats in statsMap if applicable, so that they
// are sent to jt as counters
- if (hashAggr) {
+ if (hashAggr && counterNameToEnum != null) {
incrCounter(counterNameHashOut, numRowsHashTbl);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1470100&r1=1470099&r2=1470100&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Sat Apr 20 03:17:08 2013
@@ -525,12 +525,28 @@ public abstract class Operator<T extends
if (fatalError) {
return;
}
- OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
- preProcessCounter();
- enterOperatorHooks(opHookContext);
- processOp(row, tag);
- exitOperatorHooks(opHookContext);
- postProcessCounter();
+
+ if (counterNameToEnum != null) {
+ inputRows++;
+ if ((inputRows % 1000) == 0) {
+ incrCounter(numInputRowsCntr, inputRows);
+ incrCounter(timeTakenCntr, totalTime);
+ inputRows = 0;
+ totalTime = 0;
+ }
+
+ beginTime = System.currentTimeMillis();
+ OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
+ enterOperatorHooks(opHookContext);
+ processOp(row, tag);
+ exitOperatorHooks(opHookContext);
+ totalTime += (System.currentTimeMillis() - beginTime);
+ } else {
+ OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
+ enterOperatorHooks(opHookContext);
+ processOp(row, tag);
+ exitOperatorHooks(opHookContext);
+ }
}
// If a operator wants to do some work at the beginning of a group
@@ -606,9 +622,11 @@ public abstract class Operator<T extends
state = State.CLOSE;
LOG.info(id + " finished. closing... ");
- incrCounter(numInputRowsCntr, inputRows);
- incrCounter(numOutputRowsCntr, outputRows);
- incrCounter(timeTakenCntr, totalTime);
+ if (counterNameToEnum != null) {
+ incrCounter(numInputRowsCntr, inputRows);
+ incrCounter(numOutputRowsCntr, outputRows);
+ incrCounter(timeTakenCntr, totalTime);
+ }
LOG.info(id + " forwarded " + cntr + " rows");
@@ -822,9 +840,11 @@ public abstract class Operator<T extends
protected void forward(Object row, ObjectInspector rowInspector)
throws HiveException {
- if ((++outputRows % 1000) == 0) {
- incrCounter(numOutputRowsCntr, outputRows);
- outputRows = 0;
+ if (counterNameToEnum != null) {
+ if ((++outputRows % 1000) == 0) {
+ incrCounter(numOutputRowsCntr, outputRows);
+ outputRows = 0;
+ }
}
if (isLogInfoEnabled) {
@@ -1158,39 +1178,12 @@ public abstract class Operator<T extends
protected transient Object groupKeyObject;
/**
- * this is called before operator process to buffer some counters.
- */
- private void preProcessCounter() {
- inputRows++;
- if ((inputRows % 1000) == 0) {
- incrCounter(numInputRowsCntr, inputRows);
- incrCounter(timeTakenCntr, totalTime);
- inputRows = 0;
- totalTime = 0;
- }
- beginTime = System.currentTimeMillis();
- }
-
- /**
- * this is called after operator process to buffer some counters.
- */
- private void postProcessCounter() {
- if (counterNameToEnum != null) {
- totalTime += (System.currentTimeMillis() - beginTime);
- }
- }
-
- /**
* this is called in operators in map or reduce tasks.
*
* @param name
* @param amount
*/
protected void incrCounter(String name, long amount) {
- if(counterNameToEnum == null) {
- return;
- }
-
String counterName = getWrappedCounterName(name);
ProgressCounter pc = counterNameToEnum.get(counterName);