You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/04/27 01:31:42 UTC
[hive] branch master updated (a234475 -> 3ec51a2)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from a234475 HIVE-21584: Java 11 preparation: system class loader is not URLClassLoader (Zoltan Matyus, reviewed by Adam Szita, Zoltan Haindrich)
new 75f86e1 HIVE-21657: Disable flaky cbo_rp_limit.q in TestMiniLlapLocalCliDriver (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
new b1d9a21 HIVE-21645: Include CBO json plan in explain formatted (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
new 4cab80d HIVE-21633: Estimate range for value generated by aggregate function in statistics annotation (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
new 3ec51a2 HIVE-21650: QOutProcessor should provide configurable partial masks for qtests (Aditya Shah, reviewed by Laszlo Bodor, Jesus Camacho Rodriguez)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +
.../apache/hadoop/hive/cli/control/CliConfigs.java | 3 +-
.../hadoop/hive/cli/control/CoreCliDriver.java | 24 +++
.../org/apache/hadoop/hive/ql/QOutProcessor.java | 30 ++--
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 4 +-
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 13 +-
.../stats/annotation/StatsRulesProcFactory.java | 78 +++++++++
.../hadoop/hive/ql/parse/CalcitePlanner.java | 5 +-
.../hadoop/hive/ql/exec/TestExplainTask.java | 2 +-
.../clientpositive/acid_table_directories_test.q | 34 ++++
.../acid_table_directories_test.q.out | 186 +++++++++++++++++++++
ql/src/test/results/clientpositive/concat_op.q.out | 6 +-
.../clientpositive/groupby_grouping_window.q.out | 12 +-
ql/src/test/results/clientpositive/input4.q.out | 2 +-
.../clientpositive/llap/subquery_scalar.q.out | 8 +-
.../clientpositive/llap/subquery_select.q.out | 12 +-
.../llap/vector_groupby_grouping_window.q.out | 12 +-
.../perf/tez/constraints/query78.q.out | 14 +-
.../results/clientpositive/perf/tez/query78.q.out | 16 +-
ql/src/test/results/clientpositive/plan_json.q.out | 2 +-
.../clientpositive/vector_outer_join3.q.out | 6 +-
.../clientpositive/vector_outer_join4.q.out | 6 +-
.../clientpositive/vector_outer_join6.q.out | 4 +-
23 files changed, 417 insertions(+), 68 deletions(-)
create mode 100644 ql/src/test/queries/clientpositive/acid_table_directories_test.q
create mode 100644 ql/src/test/results/clientpositive/acid_table_directories_test.q.out
[hive] 01/04: HIVE-21657: Disable flaky cbo_rp_limit.q in
TestMiniLlapLocalCliDriver (Jesus Camacho Rodriguez,
reviewed by Vineet Garg)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 75f86e1fc5ec6d48226fd1a23b49a29d6188f410
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Fri Apr 26 18:21:47 2019 -0700
HIVE-21657: Disable flaky cbo_rp_limit.q in TestMiniLlapLocalCliDriver (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
---
.../src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index f069d63..1108e49 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -257,7 +257,8 @@ public class CliConfigs {
excludeQuery("schema_evol_orc_vec_part_llap_io.q"); // Disabled in HIVE-19509
excludeQuery("load_dyn_part3.q"); // Disabled in HIVE-20662. Enable in HIVE-20663.
excludeQuery("cbo_limit.q"); //Disabled in HIVE-20860. Enable in HIVE-20972
- excludeQuery("rfc5424_parser_file_pruning.q"); // Disabled in HIVE-21427
+ excludeQuery("rfc5424_parser_file_pruning.q"); // Disabled in HIVE-21427
+ excludeQuery("cbo_rp_limit.q"); // Disabled in HIVE-21657
setResultsDir("ql/src/test/results/clientpositive/llap");
setLogDir("itests/qtest/target/qfile-results/clientpositive");
[hive] 04/04: HIVE-21650: QOutProcessor should provide configurable
partial masks for qtests (Aditya Shah, reviewed by Laszlo Bodor,
Jesus Camacho Rodriguez)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 3ec51a237d87af8122583a8ffe9afbb658f27d5e
Author: Aditya Shah <ad...@gmail.com>
AuthorDate: Fri Apr 26 18:31:04 2019 -0700
HIVE-21650: QOutProcessor should provide configurable partial masks for qtests (Aditya Shah, reviewed by Laszlo Bodor, Jesus Camacho Rodriguez)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +
.../hadoop/hive/cli/control/CoreCliDriver.java | 24 +++
.../org/apache/hadoop/hive/ql/QOutProcessor.java | 30 ++--
.../clientpositive/acid_table_directories_test.q | 34 ++++
.../acid_table_directories_test.q.out | 186 +++++++++++++++++++++
5 files changed, 269 insertions(+), 11 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 37a0a6d..0c2bd1e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -631,6 +631,12 @@ public class HiveConf extends Configuration {
true),
HIVE_MAPJOIN_TESTING_NO_HASH_TABLE_LOAD("hive.mapjoin.testing.no.hash.table.load", false, "internal use only, true when in testing map join",
true),
+ HIVE_ADDITIONAL_PARTIAL_MASKS_PATTERN("hive.qtest.additional.partial.mask.pattern", "",
+ "internal use only, used in only qtests. Provide additional partial masks pattern" +
+ "for qtests as a ',' separated list"),
+ HIVE_ADDITIONAL_PARTIAL_MASKS_REPLACEMENT_TEXT("hive.qtest.additional.partial.mask.replacement.text", "",
+ "internal use only, used in only qtests. Provide additional partial masks replacement" +
+ "text for qtests as a ',' separated list"),
HIVE_IN_REPL_TEST_FILES_SORTED("hive.in.repl.test.files.sorted", false,
"internal usage only, set to true if the file listing is required in sorted order during bootstrap load", true),
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index 7ed8388..8465474 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -23,6 +23,8 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.QTestArguments;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
@@ -187,7 +189,9 @@ public class CoreCliDriver extends CliAdapter {
qt.failed(ecode, fname, debugHint);
}
+ setupAdditionalPartialMasks();
QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ resetAdditionalPartialMasks();
if (result.getReturnCode() != 0) {
failed = true;
String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? debugHint
@@ -206,4 +210,24 @@ public class CoreCliDriver extends CliAdapter {
}
assertTrue("Test passed", true);
}
+
+ private void setupAdditionalPartialMasks() {
+ String patternStr = HiveConf.getVar(qt.getConf(), ConfVars.HIVE_ADDITIONAL_PARTIAL_MASKS_PATTERN);
+ String replacementStr = HiveConf.getVar(qt.getConf(), ConfVars.HIVE_ADDITIONAL_PARTIAL_MASKS_REPLACEMENT_TEXT);
+ if (patternStr != null && replacementStr != null && !replacementStr.isEmpty() && !patternStr.isEmpty()) {
+ String[] patterns = patternStr.split(",");
+ String[] replacements = replacementStr.split(",");
+ if (patterns.length != replacements.length) {
+ throw new RuntimeException("Count mismatch for additional partial masks and their replacements");
+ }
+ for (int i = 0; i < patterns.length; i++) {
+ qt.getQOutProcessor().addPatternWithMaskComment(patterns[i],
+ String.format("### %s ###", replacements[i]));
+ }
+ }
+ }
+
+ private void resetAdditionalPartialMasks() {
+ qt.getQOutProcessor().resetPatternwithMaskComments();
+ }
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index b87d904..ec61b34 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -293,18 +293,22 @@ public class QOutProcessor {
partialPlanMask = ppm.toArray(new PatternReplacementPair[ppm.size()]);
}
- /* This list may be modified by specific cli drivers to mask strings that change on every test */
+
@SuppressWarnings("serial")
- private final List<Pair<Pattern, String>> patternsWithMaskComments =
- new ArrayList<Pair<Pattern, String>>() {
- {
- add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*",
- "### BLOBSTORE_STAGING_PATH ###"));
- add(toPatternPair(PATH_HDFS_WITH_DATE_USER_GROUP_REGEX, String.format("%s %s$3$4 %s $6%s",
- HDFS_USER_MASK, HDFS_GROUP_MASK, HDFS_DATE_MASK, HDFS_MASK)));
- add(toPatternPair(PATH_HDFS_REGEX, String.format("$1%s", HDFS_MASK)));
- }
- };
+ private ArrayList<Pair<Pattern, String>> initPatternWithMaskComments() {
+ return new ArrayList<Pair<Pattern, String>>() {
+ {
+ add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*",
+ "### BLOBSTORE_STAGING_PATH ###"));
+ add(toPatternPair(PATH_HDFS_WITH_DATE_USER_GROUP_REGEX, String.format("%s %s$3$4 %s $6%s",
+ HDFS_USER_MASK, HDFS_GROUP_MASK, HDFS_DATE_MASK, HDFS_MASK)));
+ add(toPatternPair(PATH_HDFS_REGEX, String.format("$1%s", HDFS_MASK)));
+ }
+ };
+ }
+
+ /* This list may be modified by specific cli drivers to mask strings that change on every test */
+ private List<Pair<Pattern, String>> patternsWithMaskComments = initPatternWithMaskComments();
private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) {
return ImmutablePair.of(Pattern.compile(patternStr), maskComment);
@@ -334,4 +338,8 @@ public class QOutProcessor {
return false;
}
+ public void resetPatternwithMaskComments() {
+ patternsWithMaskComments = initPatternWithMaskComments();
+ }
+
}
diff --git a/ql/src/test/queries/clientpositive/acid_table_directories_test.q b/ql/src/test/queries/clientpositive/acid_table_directories_test.q
new file mode 100644
index 0000000..9bb8e10
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/acid_table_directories_test.q
@@ -0,0 +1,34 @@
+set hive.mapred.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+set hive.qtest.additional.partial.mask.pattern=.*acidparttable\/p=(100|200)\/base.*,.*acidparttable/p=(100|200)/delta.*;
+set hive.qtest.additional.partial.mask.replacement.text=ACID BASE DIR,ACID DELTA DIR;
+
+-- create a source table where the IOW data select from
+create table srctbl (key char(1), value int);
+insert into table srctbl values ('d', 4), ('e', 5), ('f', 6), ('i', 9), ('j', 10);
+select * from srctbl;
+
+-- insert overwrite on partitioned acid table
+drop table if exists acidparttbl;
+create table acidparttbl (key char(1), value int) partitioned by (p int) clustered by (value) into 2 buckets stored as orc location 'pfile://${system:test.tmp.dir}/acidparttable' TBLPROPERTIES ("transactional"="true");
+
+insert into table acidparttbl partition(p=100) values ('a', 1), ('b', 2), ('c', 3);
+select p, key, value from acidparttbl order by p, key;
+
+insert overwrite table acidparttbl partition(p=100) select key, value from srctbl where key in ('d', 'e', 'f');
+select p, key, value from acidparttbl order by p, key;
+
+insert into table acidparttbl partition(p) values ('g', 7, 100), ('h', 8, 200);
+select p, key, value from acidparttbl order by p, key;
+
+insert overwrite table acidparttbl partition(p) values ('i', 9, 100), ('j', 10, 200);
+select p, key, value from acidparttbl order by p, key;
+
+-- check directories of the table
+dfs -ls -R 'pfile://${system:test.tmp.dir}/acidparttable';
+
+drop table acidparttbl;
+drop table srctbl;
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/acid_table_directories_test.q.out b/ql/src/test/results/clientpositive/acid_table_directories_test.q.out
new file mode 100644
index 0000000..1fec361
--- /dev/null
+++ b/ql/src/test/results/clientpositive/acid_table_directories_test.q.out
@@ -0,0 +1,186 @@
+PREHOOK: query: create table srctbl (key char(1), value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srctbl
+POSTHOOK: query: create table srctbl (key char(1), value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srctbl
+PREHOOK: query: insert into table srctbl values ('d', 4), ('e', 5), ('f', 6), ('i', 9), ('j', 10)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@srctbl
+POSTHOOK: query: insert into table srctbl values ('d', 4), ('e', 5), ('f', 6), ('i', 9), ('j', 10)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@srctbl
+POSTHOOK: Lineage: srctbl.key SCRIPT []
+POSTHOOK: Lineage: srctbl.value SCRIPT []
+PREHOOK: query: select * from srctbl
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srctbl
+#### A masked pattern was here ####
+POSTHOOK: query: select * from srctbl
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srctbl
+#### A masked pattern was here ####
+d 4
+e 5
+f 6
+i 9
+j 10
+PREHOOK: query: drop table if exists acidparttbl
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists acidparttbl
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acidparttbl
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acidparttbl
+PREHOOK: query: insert into table acidparttbl partition(p=100) values ('a', 1), ('b', 2), ('c', 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@acidparttbl@p=100
+POSTHOOK: query: insert into table acidparttbl partition(p=100) values ('a', 1), ('b', 2), ('c', 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@acidparttbl@p=100
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).key SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).value SCRIPT []
+PREHOOK: query: select p, key, value from acidparttbl order by p, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acidparttbl
+PREHOOK: Input: default@acidparttbl@p=100
+#### A masked pattern was here ####
+POSTHOOK: query: select p, key, value from acidparttbl order by p, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acidparttbl
+POSTHOOK: Input: default@acidparttbl@p=100
+#### A masked pattern was here ####
+100 a 1
+100 b 2
+100 c 3
+PREHOOK: query: insert overwrite table acidparttbl partition(p=100) select key, value from srctbl where key in ('d', 'e', 'f')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srctbl
+PREHOOK: Output: default@acidparttbl@p=100
+POSTHOOK: query: insert overwrite table acidparttbl partition(p=100) select key, value from srctbl where key in ('d', 'e', 'f')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srctbl
+POSTHOOK: Output: default@acidparttbl@p=100
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).key SIMPLE [(srctbl)srctbl.FieldSchema(name:key, type:char(1), comment:null), ]
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).value SIMPLE [(srctbl)srctbl.FieldSchema(name:value, type:int, comment:null), ]
+PREHOOK: query: select p, key, value from acidparttbl order by p, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acidparttbl
+PREHOOK: Input: default@acidparttbl@p=100
+#### A masked pattern was here ####
+POSTHOOK: query: select p, key, value from acidparttbl order by p, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acidparttbl
+POSTHOOK: Input: default@acidparttbl@p=100
+#### A masked pattern was here ####
+100 d 4
+100 e 5
+100 f 6
+PREHOOK: query: insert into table acidparttbl partition(p) values ('g', 7, 100), ('h', 8, 200)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@acidparttbl
+POSTHOOK: query: insert into table acidparttbl partition(p) values ('g', 7, 100), ('h', 8, 200)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@acidparttbl@p=100
+POSTHOOK: Output: default@acidparttbl@p=200
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).key SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).value SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=200).key SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=200).value SCRIPT []
+PREHOOK: query: select p, key, value from acidparttbl order by p, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acidparttbl
+PREHOOK: Input: default@acidparttbl@p=100
+PREHOOK: Input: default@acidparttbl@p=200
+#### A masked pattern was here ####
+POSTHOOK: query: select p, key, value from acidparttbl order by p, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acidparttbl
+POSTHOOK: Input: default@acidparttbl@p=100
+POSTHOOK: Input: default@acidparttbl@p=200
+#### A masked pattern was here ####
+100 d 4
+100 e 5
+100 f 6
+100 g 7
+200 h 8
+PREHOOK: query: insert overwrite table acidparttbl partition(p) values ('i', 9, 100), ('j', 10, 200)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@acidparttbl
+POSTHOOK: query: insert overwrite table acidparttbl partition(p) values ('i', 9, 100), ('j', 10, 200)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@acidparttbl@p=100
+POSTHOOK: Output: default@acidparttbl@p=200
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).key SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=100).value SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=200).key SCRIPT []
+POSTHOOK: Lineage: acidparttbl PARTITION(p=200).value SCRIPT []
+PREHOOK: query: select p, key, value from acidparttbl order by p, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acidparttbl
+PREHOOK: Input: default@acidparttbl@p=100
+PREHOOK: Input: default@acidparttbl@p=200
+#### A masked pattern was here ####
+POSTHOOK: query: select p, key, value from acidparttbl order by p, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acidparttbl
+POSTHOOK: Input: default@acidparttbl@p=100
+POSTHOOK: Input: default@acidparttbl@p=200
+#### A masked pattern was here ####
+100 i 9
+200 j 10
+#### A masked pattern was here ####
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+#### A masked pattern was here ####
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID BASE DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+### ACID DELTA DIR ###
+PREHOOK: query: drop table acidparttbl
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@acidparttbl
+PREHOOK: Output: default@acidparttbl
+POSTHOOK: query: drop table acidparttbl
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@acidparttbl
+POSTHOOK: Output: default@acidparttbl
+PREHOOK: query: drop table srctbl
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srctbl
+PREHOOK: Output: default@srctbl
+POSTHOOK: query: drop table srctbl
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srctbl
+POSTHOOK: Output: default@srctbl
[hive] 03/04: HIVE-21633: Estimate range for value generated by
aggregate function in statistics annotation (Jesus Camacho Rodriguez,
reviewed by Vineet Garg)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 4cab80d8a8007c3cc332aa33279d8c7ebde48ed2
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Thu Apr 18 13:32:38 2019 -0700
HIVE-21633: Estimate range for value generated by aggregate function in statistics annotation (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
Close apache/hive#603
---
.../stats/annotation/StatsRulesProcFactory.java | 78 ++++++++++++++++++++++
.../clientpositive/groupby_grouping_window.q.out | 12 ++--
.../clientpositive/llap/subquery_scalar.q.out | 8 +--
.../clientpositive/llap/subquery_select.q.out | 12 ++--
.../llap/vector_groupby_grouping_window.q.out | 12 ++--
.../perf/tez/constraints/query78.q.out | 14 ++--
.../results/clientpositive/perf/tez/query78.q.out | 16 ++---
7 files changed, 115 insertions(+), 37 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 6a1c210..0258e36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -84,7 +84,12 @@ import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
import org.apache.hadoop.hive.ql.stats.OperatorStats;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
@@ -1479,6 +1484,7 @@ public class StatsRulesProcFactory {
// if UDAFs are present, new columns needs to be added
if (!aggDesc.isEmpty() && stats != null) {
List<ColStatistics> aggColStats = Lists.newArrayList();
+ int idx = 0;
for (ColumnInfo ci : rs.getSignature()) {
// if the columns in row schema is not contained in column
@@ -1492,6 +1498,7 @@ public class StatsRulesProcFactory {
cs.setCountDistint(stats.getNumRows());
cs.setNumNulls(0);
cs.setAvgColLen(StatsUtils.getAvgColLenOf(conf, ci.getObjectInspector(), colType));
+ computeAggregateColumnMinMax(cs, conf, aggDesc.get(idx++), colType, parentStats);
aggColStats.add(cs);
}
}
@@ -1524,6 +1531,77 @@ public class StatsRulesProcFactory {
return null;
}
+ /**
+ * If possible, sets the min / max value for the column based on the aggregate function
+ * being calculated and its input.
+ */
+ private static void computeAggregateColumnMinMax(ColStatistics cs, HiveConf conf, AggregationDesc agg, String aggType,
+ Statistics parentStats) throws SemanticException {
+ if (agg.getParameters() != null && agg.getParameters().size() == 1) {
+ ColStatistics parentCS = StatsUtils.getColStatisticsFromExpression(
+ conf, parentStats, agg.getParameters().get(0));
+ if (parentCS != null && parentCS.getRange() != null &&
+ parentCS.getRange().minValue != null && parentCS.getRange().maxValue != null) {
+ long valuesCount = agg.getDistinct() ?
+ parentCS.getCountDistint() :
+ parentStats.getNumRows() - parentCS.getNumNulls();
+ Range range = parentCS.getRange();
+ // Get the aggregate function matching the name in the query.
+ GenericUDAFResolver udaf =
+ FunctionRegistry.getGenericUDAFResolver(agg.getGenericUDAFName());
+ if (udaf instanceof GenericUDAFCount) {
+ cs.setRange(new Range(0, valuesCount));
+ } else if (udaf instanceof GenericUDAFMax || udaf instanceof GenericUDAFMin) {
+ cs.setRange(new Range(range.minValue, range.maxValue));
+ } else if (udaf instanceof GenericUDAFSum) {
+ switch (aggType) {
+ case serdeConstants.TINYINT_TYPE_NAME:
+ case serdeConstants.SMALLINT_TYPE_NAME:
+ case serdeConstants.DATE_TYPE_NAME:
+ case serdeConstants.INT_TYPE_NAME:
+ case serdeConstants.BIGINT_TYPE_NAME:
+ long maxValueLong = range.maxValue.longValue();
+ long minValueLong = range.minValue.longValue();
+ // If min value is less or equal to max value (legal)
+ if (minValueLong <= maxValueLong && minValueLong >= 0) {
+ // min = minValue, max = (minValue + maxValue) * 0.5 * parentNumRows
+ cs.setRange(new Range(
+ minValueLong,
+ StatsUtils.safeMult(
+ StatsUtils.safeMult(StatsUtils.safeAdd(minValueLong, maxValueLong), 0.5),
+ valuesCount)));
+ }
+ break;
+ case serdeConstants.FLOAT_TYPE_NAME:
+ case serdeConstants.DOUBLE_TYPE_NAME:
+ double maxValueDouble = range.maxValue.doubleValue();
+ double minValueDouble = range.minValue.doubleValue();
+ // If min value is less or equal to max value (legal)
+ if (minValueDouble <= maxValueDouble && minValueDouble >= 0) {
+ // min = minValue, max = (minValue + maxValue) * 0.5 * parentNumRows
+ cs.setRange(new Range(
+ minValueDouble,
+ (minValueDouble + maxValueDouble) * 0.5 * valuesCount));
+ }
+ break;
+ default:
+ if (aggType.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) {
+ BigDecimal maxValueBD = new BigDecimal(range.maxValue.toString());
+ BigDecimal minValueBD = new BigDecimal(range.minValue.toString());
+ // If min value is less or equal to max value (legal)
+ if (minValueBD.compareTo(maxValueBD) <= 0 && minValueBD.compareTo(BigDecimal.ZERO) >= 0) {
+ // min = minValue, max = (minValue + maxValue) * 0.5 * parentNumRows
+ cs.setRange(new Range(
+ minValueBD,
+ minValueBD.add(maxValueBD).multiply(new BigDecimal(0.5)).multiply(new BigDecimal(valuesCount))));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
private long getParentNumRows(GroupByOperator op, List<ExprNodeDesc> gbyKeys, HiveConf conf) {
if(gbyKeys == null || gbyKeys.isEmpty()) {
return op.getParentOperators().get(0).getStatistics().getNumRows();
diff --git a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
index e6cc459..7f687da 100644
--- a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
+++ b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
@@ -75,7 +75,7 @@ STAGE PLANS:
pruneGroupingSetId: true
Filter Operator
predicate: (_col3 > 0) (type: boolean)
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -91,14 +91,14 @@ STAGE PLANS:
key expressions: _col0 (type: int), _col3 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: int)
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey1 (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
PTF Operator
Function definitions:
Input definition
@@ -119,14 +119,14 @@ STAGE PLANS:
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index 0d8ff14..5817f98 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -1500,7 +1500,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.0
@@ -1537,16 +1537,16 @@ STAGE PLANS:
Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (_col1 > 100) (type: boolean)
- Statistics: Num rows: 4 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
index fc70407..d58905c 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
@@ -5057,16 +5057,16 @@ STAGE PLANS:
Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (_col2 > 0L) (type: boolean)
- Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col1 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -5534,16 +5534,16 @@ STAGE PLANS:
Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (_col2 > 0L) (type: boolean)
- Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col1 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 6
Execution mode: llap
Reduce Operator Tree:
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_window.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_window.q.out
index 3a9ea79..5e391bf 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_window.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_window.q.out
@@ -151,7 +151,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterLongColGreaterLongScalar(col 2:int, val 0)
predicate: (_col3 > 0) (type: boolean)
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col3 (type: int)
sort order: ++
@@ -163,7 +163,7 @@ STAGE PLANS:
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
partitionColumns: 0:int
valueColumns: 1:int
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: int)
Reducer 3
Execution mode: vectorized, llap
@@ -188,7 +188,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 2, 1]
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
PTF Operator
Function definitions:
Input definition
@@ -222,7 +222,7 @@ STAGE PLANS:
outputTypes: [int, int, int, int]
partitionExpressions: [col 0:int]
streamingColumns: [3]
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -230,13 +230,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 2, 1, 3]
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/query78.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/query78.q.out
index 888d335..792540f 100644
--- a/ql/src/test/results/clientpositive/perf/tez/constraints/query78.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/constraints/query78.q.out
@@ -158,18 +158,18 @@ Stage-0
File Output Operator [FS_269]
Limit [LIM_268] (rows=100 width=484)
Number of rows:100
- Select Operator [SEL_267] (rows=203549242538 width=483)
+ Select Operator [SEL_267] (rows=1831943309558 width=483)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_73]
- Select Operator [SEL_72] (rows=203549242538 width=719)
+ Select Operator [SEL_72] (rows=1831943309558 width=719)
Output:["_col0","_col1","_col6","_col7","_col8","_col9","_col10","_col11","_col12"]
- Merge Join Operator [MERGEJOIN_220] (rows=203549242538 width=703)
+ Merge Join Operator [MERGEJOIN_220] (rows=1831943309558 width=703)
Conds:RS_69._col0, _col1=RS_266._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col11","_col12","_col13"]
<-Reducer 12 [ONE_TO_ONE_EDGE] vectorized
FORWARD [RS_266]
PartitionCols:_col0, _col1
- Filter Operator [FIL_265] (rows=13513323 width=239)
+ Filter Operator [FIL_265] (rows=40539971 width=239)
predicate:(_col2 > 0L)
Group By Operator [GBY_264] (rows=40539971 width=239)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1
@@ -228,7 +228,7 @@ Stage-0
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_69]
PartitionCols:_col0, _col1
- Merge Join Operator [MERGEJOIN_219] (rows=7613716536 width=471)
+ Merge Join Operator [MERGEJOIN_219] (rows=22841150061 width=471)
Conds:RS_244._col1=RS_256._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col6","_col7","_col8"]
<-Reducer 3 [SIMPLE_EDGE] vectorized
SHUFFLE [RS_244]
@@ -287,9 +287,9 @@ Stage-0
<-Reducer 9 [SIMPLE_EDGE] vectorized
SHUFFLE [RS_256]
PartitionCols:_col0
- Select Operator [SEL_255] (rows=33694814 width=235)
+ Select Operator [SEL_255] (rows=101084444 width=235)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_254] (rows=33694814 width=239)
+ Filter Operator [FIL_254] (rows=101084444 width=239)
predicate:(_col2 > 0L)
Select Operator [SEL_253] (rows=101084444 width=239)
Output:["_col1","_col2","_col3","_col4"]
diff --git a/ql/src/test/results/clientpositive/perf/tez/query78.q.out b/ql/src/test/results/clientpositive/perf/tez/query78.q.out
index e66d6f5..9ce2cdb 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query78.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query78.q.out
@@ -158,20 +158,20 @@ Stage-0
File Output Operator [FS_276]
Limit [LIM_275] (rows=100 width=484)
Number of rows:100
- Select Operator [SEL_274] (rows=203549242531 width=483)
+ Select Operator [SEL_274] (rows=1831943309424 width=483)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_76]
- Select Operator [SEL_75] (rows=203549242531 width=719)
+ Select Operator [SEL_75] (rows=1831943309424 width=719)
Output:["_col0","_col1","_col2","_col6","_col7","_col8","_col9","_col10","_col11","_col12"]
- Merge Join Operator [MERGEJOIN_223] (rows=203549242531 width=715)
+ Merge Join Operator [MERGEJOIN_223] (rows=1831943309424 width=715)
Conds:RS_72._col1=RS_273._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9","_col11","_col12","_col13","_col14","_col15"]
<-Reducer 12 [SIMPLE_EDGE] vectorized
SHUFFLE [RS_273]
PartitionCols:_col0
- Select Operator [SEL_272] (rows=33694814 width=247)
+ Select Operator [SEL_272] (rows=101084444 width=247)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_271] (rows=33694814 width=239)
+ Filter Operator [FIL_271] (rows=101084444 width=239)
predicate:(_col2 > 0L)
Select Operator [SEL_270] (rows=101084444 width=239)
Output:["_col1","_col2","_col3","_col4"]
@@ -234,7 +234,7 @@ Stage-0
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_72]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_222] (rows=3053485049 width=471)
+ Merge Join Operator [MERGEJOIN_222] (rows=9160455599 width=471)
Conds:RS_248._col1, _col0=RS_260._col1, _col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9"]
<-Reducer 3 [ONE_TO_ONE_EDGE] vectorized
FORWARD [RS_248]
@@ -295,9 +295,9 @@ Stage-0
<-Reducer 9 [ONE_TO_ONE_EDGE] vectorized
FORWARD [RS_260]
PartitionCols:_col1, _col0
- Select Operator [SEL_259] (rows=13513323 width=239)
+ Select Operator [SEL_259] (rows=40539971 width=239)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_258] (rows=13513323 width=239)
+ Filter Operator [FIL_258] (rows=40539971 width=239)
predicate:(_col2 > 0L)
Group By Operator [GBY_257] (rows=40539971 width=239)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1
[hive] 02/04: HIVE-21645: Include CBO json plan in explain
formatted (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit b1d9a21ebf298e2af2708a90efc2fc548da89085
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Thu Apr 25 17:41:51 2019 -0700
HIVE-21645: Include CBO json plan in explain formatted (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
Close apache/hive#607
---
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 4 ++--
ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java | 13 +++++++++++--
.../org/apache/hadoop/hive/ql/parse/CalcitePlanner.java | 5 ++++-
.../org/apache/hadoop/hive/ql/exec/TestExplainTask.java | 2 +-
ql/src/test/results/clientpositive/concat_op.q.out | 6 +++---
ql/src/test/results/clientpositive/input4.q.out | 2 +-
ql/src/test/results/clientpositive/plan_json.q.out | 2 +-
ql/src/test/results/clientpositive/vector_outer_join3.q.out | 6 +++---
ql/src/test/results/clientpositive/vector_outer_join4.q.out | 6 +++---
ql/src/test/results/clientpositive/vector_outer_join6.q.out | 4 ++--
10 files changed, 31 insertions(+), 19 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 4f14fa5..3a20130 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1047,7 +1047,7 @@ public class Driver implements IDriver {
if (conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_SHOW_GRAPH)) {
JSONObject jsonPlan = task.getJSONPlan(
null, rootTasks, sem.getFetchTask(), true, true, true, sem.getCboInfo(),
- plan.getOptimizedQueryString());
+ plan.getOptimizedCBOPlan(), plan.getOptimizedQueryString());
if (jsonPlan.getJSONObject(ExplainTask.STAGE_DEPENDENCIES) != null &&
jsonPlan.getJSONObject(ExplainTask.STAGE_DEPENDENCIES).length() <=
conf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_GRAPH_SIZE)) {
@@ -1057,7 +1057,7 @@ public class Driver implements IDriver {
}
} else {
task.getJSONPlan(ps, rootTasks, sem.getFetchTask(), false, true, true, sem.getCboInfo(),
- plan.getOptimizedQueryString());
+ plan.getOptimizedCBOPlan(), plan.getOptimizedQueryString());
ret = baos.toString();
}
} catch (Exception e) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index b9d1e0b..8223d6b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -237,12 +237,12 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
throws Exception {
return getJSONPlan(out, work.getRootTasks(), work.getFetchTask(),
work.isFormatted(), work.getExtended(), work.isAppendTaskType(), work.getCboInfo(),
- work.getOptimizedSQL());
+ work.getCboPlan(), work.getOptimizedSQL());
}
public JSONObject getJSONPlan(PrintStream out, List<Task<?>> tasks, Task<?> fetchTask,
boolean jsonOutput, boolean isExtended, boolean appendTaskType, String cboInfo,
- String optimizedSQL) throws Exception {
+ String cboPlan, String optimizedSQL) throws Exception {
// If the user asked for a formatted output, dump the json output
// in the output stream
@@ -252,6 +252,15 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
out = null;
}
+ if (cboPlan != null) {
+ if (jsonOutput) {
+ outJSONObject.put("CBOPlan", cboPlan);
+ } else {
+ out.print("CBO PLAN:");
+ out.println(cboPlan);
+ }
+ }
+
if (optimizedSQL != null) {
if (jsonOutput) {
outJSONObject.put("optimizedSQL", optimizedSQL);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index bc150fb..a7cdddb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -554,7 +554,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
// Do not include join cost
this.ctx.setCalcitePlan(RelOptUtil.toString(newPlan));
}
- } else if (explainConfig.isExtended() || explainConfig.isFormatted()) {
+ } else if (explainConfig.isFormatted()) {
+ this.ctx.setCalcitePlan(HiveRelOptUtil.toJsonString(newPlan));
+ this.ctx.setOptimizedSql(getOptimizedSql(newPlan));
+ } else if (explainConfig.isExtended()) {
this.ctx.setOptimizedSql(getOptimizedSql(newPlan));
}
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java
index a0c0413..8ba612e 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java
@@ -218,7 +218,7 @@ public class TestExplainTask {
JsonNode result = objectMapper.readTree(uut.getJSONPlan(null, tasks, null, true,
- false, false, "Plan Optimized by CBO", null).toString());
+ false, false, "Plan Optimized by CBO", null, null).toString());
JsonNode expected = objectMapper.readTree("{\"cboInfo\":\"Plan Optimized by CBO\", \"STAGE DEPENDENCIES\":{\"mockTaskId\":" +
"{\"ROOT STAGE\":\"TRUE\",\"BACKUP STAGE\":\"backup-id-mock\"}},\"STAGE PLANS\":" +
"{\"mockTaskId\":{}}}");
diff --git a/ql/src/test/results/clientpositive/concat_op.q.out b/ql/src/test/results/clientpositive/concat_op.q.out
index 9c4fca2..3892f33 100644
--- a/ql/src/test/results/clientpositive/concat_op.q.out
+++ b/ql/src/test/results/clientpositive/concat_op.q.out
@@ -311,7 +311,7 @@ POSTHOOK: query: explain formatted select key || value from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":" [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": [...]
PREHOOK: query: explain formatted select key || value || key from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
@@ -320,7 +320,7 @@ POSTHOOK: query: explain formatted select key || value || key from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `key` || `value` || `key` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","Opera [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": [...]
PREHOOK: query: explain formatted select key || value || key || value from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
@@ -329,4 +329,4 @@ POSTHOOK: query: explain formatted select key || value || key || value from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `key` || `value` || `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"fa [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": [...]
diff --git a/ql/src/test/results/clientpositive/input4.q.out b/ql/src/test/results/clientpositive/input4.q.out
index c8951e8..6311193 100644
--- a/ql/src/test/results/clientpositive/input4.q.out
+++ b/ql/src/test/results/clientpositive/input4.q.out
@@ -53,7 +53,7 @@ SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4_n0 AS Input4Alias
POSTHOOK: type: QUERY
POSTHOOK: Input: default@input4_n0
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `value`, `key`\nFROM `default`.`input4_n0`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","columns:":["value","key"],"database:":"default","Statistics:":"Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE","table:":"input4_n0","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Ope [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"input4_n0\"\n ],\n \"table:alias\": \"input4alias\",\n \"inputs\": [],\n \"rowCount\": 1.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n [...]
PREHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4_n0 AS Input4Alias
PREHOOK: type: QUERY
PREHOOK: Input: default@input4_n0
diff --git a/ql/src/test/results/clientpositive/plan_json.q.out b/ql/src/test/results/clientpositive/plan_json.q.out
index b16e574..40b2644 100644
--- a/ql/src/test/results/clientpositive/plan_json.q.out
+++ b/ql/src/test/results/clientpositive/plan_json.q.out
@@ -6,4 +6,4 @@ POSTHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_7"}}}}}}
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 0.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \" [...]
diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
index 57a4f0c..c7f49ef 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
@@ -248,7 +248,7 @@ left outer join small_alltypesorc_a_n1 hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a_n1
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.ena [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 10.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n } [...]
PREHOOK: query: select count(*) from (select c.cstring1
from small_alltypesorc_a_n1 c
left outer join small_alltypesorc_a_n1 cd
@@ -292,7 +292,7 @@ left outer join small_alltypesorc_a_n1 hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a_n1
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vect [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 14.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n [...]
PREHOOK: query: select count(*) from (select c.cstring1
from small_alltypesorc_a_n1 c
left outer join small_alltypesorc_a_n1 cd
@@ -336,7 +336,7 @@ left outer join small_alltypesorc_a_n1 hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a_n1
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL AND `cbigint` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL AND `cint` IS NOT [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 26.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n [...]
PREHOOK: query: select count(*) from (select c.cstring1
from small_alltypesorc_a_n1 c
left outer join small_alltypesorc_a_n1 cd
diff --git a/ql/src/test/results/clientpositive/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/vector_outer_join4.q.out
index 1cfd6f7..3f8cb68 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join4.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join4.q.out
@@ -262,7 +262,7 @@ left outer join small_alltypesorc_b cd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 139.86666666666667,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\" [...]
PREHOOK: query: select *
from small_alltypesorc_b c
left outer join small_alltypesorc_b cd
@@ -347,7 +347,7 @@ left outer join small_alltypesorc_b hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"}," [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n [...]
PREHOOK: query: select c.ctinyint
from small_alltypesorc_b c
left outer join small_alltypesorc_b hd
@@ -794,7 +794,7 @@ left outer join small_alltypesorc_b hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS t [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n [...]
PREHOOK: query: select count(*) from (select c.ctinyint
from small_alltypesorc_b c
left outer join small_alltypesorc_b cd
diff --git a/ql/src/test/results/clientpositive/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/vector_outer_join6.q.out
index 7269177..6c6899b 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join6.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join6.q.out
@@ -138,7 +138,7 @@ POSTHOOK: Input: default@tjoin1_n0
POSTHOOK: Input: default@tjoin2_n0
POSTHOOK: Input: default@tjoin3
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled I [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin1_n0\"\n ],\n \"table:alias\": \"tjoin1_n0\",\n \"inputs\": [],\n \"rowCount\": 3.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n [...]
PREHOOK: query: select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from
(select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
PREHOOK: type: QUERY
@@ -173,7 +173,7 @@ POSTHOOK: Input: default@tjoin1_n0
POSTHOOK: Input: default@tjoin2_n0
POSTHOOK: Input: default@tjoin3
#### A masked pattern was here ####
-{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimize [...]
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin1_n0\"\n ],\n \"table:alias\": \"tjoin1_n0\",\n \"inputs\": [],\n \"rowCount\": 3.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n [...]
PREHOOK: query: select tj1rnum, tj2rnum as rnumt3 from
(select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
PREHOOK: type: QUERY