You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/04/28 02:02:50 UTC
[4/4] hive git commit: HIVE-19338 : isExplicitAnalyze method may be
incorrect in BasicStatsTask (Sergey Shelukhin,
reviewed by Jesus Camacho Rodriguez)
HIVE-19338 : isExplicitAnalyze method may be incorrect in BasicStatsTask (Sergey Shelukhin, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0930154f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0930154f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0930154f
Branch: refs/heads/branch-3
Commit: 0930154fdf4ffc35de5bc2fe56cd03a1f48f7598
Parents: 596c811
Author: sergey <se...@apache.org>
Authored: Fri Apr 27 18:57:37 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Apr 27 18:57:50 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java | 1 +
.../org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java | 1 +
ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java | 1 +
.../hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java | 1 +
.../java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java | 7 ++++++-
5 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
index 06b7c93..64f9c70 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
@@ -112,6 +112,7 @@ public class GenMRTableScan1 implements NodeProcessor {
// The MR task is just a simple TableScanOperator
BasicStatsWork statsWork = new BasicStatsWork(table.getTableSpec());
+ statsWork.setIsExplicitAnalyze(true);
statsWork.setNoScanAnalyzeCommand(noScan);
StatsWork columnStatsWork = new StatsWork(table, statsWork, parseCtx.getConf());
http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
index 5961059..03cceac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
@@ -115,6 +115,7 @@ public class ProcessAnalyzeTable implements NodeProcessor {
// The Tez task is just a simple TableScanOperator
BasicStatsWork basicStatsWork = new BasicStatsWork(table.getTableSpec());
+ basicStatsWork.setIsExplicitAnalyze(true);
basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand());
StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, parseContext.getConf());
columnStatsWork.collectStatsFromAggregator(tableScan.getConf());
http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index c268ddc..df1d9cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -392,6 +392,7 @@ public abstract class TaskCompiler {
return TaskFactory.get(columnStatsWork);
} else {
BasicStatsWork statsWork = new BasicStatsWork(tableScan.getConf().getTableMetadata().getTableSpec());
+ statsWork.setIsExplicitAnalyze(true);
StatsWork columnStatsWork = new StatsWork(table, statsWork, parseContext.getConf());
columnStatsWork.collectStatsFromAggregator(tableScan.getConf());
columnStatsWork.setSourceTask(currentTask);
http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
index e81d6f3..28d4de7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
@@ -123,6 +123,7 @@ public class SparkProcessAnalyzeTable implements NodeProcessor {
// The Spark task is just a simple TableScanOperator
BasicStatsWork basicStatsWork = new BasicStatsWork(table.getTableSpec());
+ basicStatsWork.setIsExplicitAnalyze(true);
basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand());
StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, parseContext.getConf());
columnStatsWork.collectStatsFromAggregator(tableScan.getConf());
http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
index 55d05a1..40def60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
public class BasicStatsWork implements Serializable {
private static final long serialVersionUID = 1L;
+ private boolean isExplicitAnalyze;
private TableSpec tableSpecs; // source table spec -- for TableScanOperator
private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator
private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator
@@ -164,7 +165,7 @@ public class BasicStatsWork implements Serializable {
public boolean isExplicitAnalyze() {
// ANALYZE TABLE
- return (getTableSpecs() != null);
+ return isExplicitAnalyze;
}
public boolean isTargetRewritten() {
// ANALYZE TABLE
@@ -202,4 +203,8 @@ public class BasicStatsWork implements Serializable {
}
}
+ public void setIsExplicitAnalyze(boolean b) {
+ this.isExplicitAnalyze = b;
+ }
+
}