You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ch...@apache.org on 2015/11/25 04:01:24 UTC
hive git commit: HIVE-12466: SparkCounter not initialized error (Rui
via Chengxiang)
Repository: hive
Updated Branches:
refs/heads/spark b934ad1cc -> eddb8ca1d
HIVE-12466: SparkCounter not initialized error (Rui via Chengxiang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eddb8ca1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eddb8ca1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eddb8ca1
Branch: refs/heads/spark
Commit: eddb8ca1d345f5a18f0b75895c92620e69956f6a
Parents: b934ad1
Author: chengxiang <ch...@apache.com>
Authored: Wed Nov 25 11:07:12 2015 +0800
Committer: chengxiang <ch...@apache.com>
Committed: Wed Nov 25 11:07:12 2015 +0800
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FileSinkOperator.java | 17 ++++++++++-------
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 14 +++++++++-----
.../hadoop/hive/ql/exec/spark/SparkTask.java | 4 ++--
3 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/eddb8ca1/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 7459bba..99cdb95 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -430,13 +430,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
cntr = 1;
logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS);
- String suffix = Integer.toString(conf.getDestTableId());
- String fullName = conf.getTableInfo().getTableName();
- if (fullName != null) {
- suffix = suffix + "_" + fullName.toLowerCase();
- }
-
- statsMap.put(Counter.RECORDS_OUT + "_" + suffix, row_count);
+ statsMap.put(getCounterName(Counter.RECORDS_OUT), row_count);
} catch (HiveException e) {
throw e;
} catch (Exception e) {
@@ -445,6 +439,15 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
}
+ public String getCounterName(Counter counter) {
+ String suffix = Integer.toString(conf.getDestTableId());
+ String fullName = conf.getTableInfo().getTableName();
+ if (fullName != null) {
+ suffix = suffix + "_" + fullName.toLowerCase();
+ }
+ return counter + "_" + suffix;
+ }
+
private void logOutputFormatError(Configuration hconf, HiveException ex) {
StringWriter errorWriter = new StringWriter();
errorWriter.append("Failed to create output format; configuration: ");
http://git-wip-us.apache.org/repos/asf/hive/blob/eddb8ca1/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
index ef5ee95..f4730ec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
@@ -164,11 +164,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
cntr = 1;
logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS);
- String context = hconf.get(Operator.CONTEXT_NAME_KEY, "");
- if (context != null && !context.isEmpty()) {
- context = "_" + context.replace(" ","_");
- }
- statsMap.put(Counter.RECORDS_OUT_INTERMEDIATE + context, recordCounter);
+ statsMap.put(getCounterName(Counter.RECORDS_OUT_INTERMEDIATE, hconf), recordCounter);
List<ExprNodeDesc> keys = conf.getKeyCols();
@@ -250,6 +246,14 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
}
}
+ public String getCounterName(Counter counter, Configuration hconf) {
+ String context = hconf.get(Operator.CONTEXT_NAME_KEY, "");
+ if (context != null && !context.isEmpty()) {
+ context = "_" + context.replace(" ", "_");
+ }
+ return counter + context;
+ }
+
/**
* Initializes array of ExprNodeEvaluator. Adds Union field for distinct
http://git-wip-us.apache.org/repos/asf/hive/blob/eddb8ca1/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index 336d490..eaeffee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -384,11 +384,11 @@ public class SparkTask extends Task<SparkWork> {
for (Operator<? extends OperatorDesc> operator : work.getAllOperators()) {
if (operator instanceof FileSinkOperator) {
for (FileSinkOperator.Counter counter : FileSinkOperator.Counter.values()) {
- hiveCounters.add(counter.toString());
+ hiveCounters.add(((FileSinkOperator) operator).getCounterName(counter));
}
} else if (operator instanceof ReduceSinkOperator) {
for (ReduceSinkOperator.Counter counter : ReduceSinkOperator.Counter.values()) {
- hiveCounters.add(counter.toString());
+ hiveCounters.add(((ReduceSinkOperator) operator).getCounterName(counter, conf));
}
} else if (operator instanceof ScriptOperator) {
for (ScriptOperator.Counter counter : ScriptOperator.Counter.values()) {