You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/21 19:51:49 UTC
[1/4] hive git commit: HIVE-4367: enhance TRUNCATE syntax to drop
data of external table (Jason Dere, reviewed by Teddy Choi)
Repository: hive
Updated Branches:
refs/heads/master-txnstats 31ee8704e -> 5c8b5d8b6
HIVE-4367: enhance TRUNCATE syntax to drop data of external table (Jason Dere, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e569ef09
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e569ef09
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e569ef09
Branch: refs/heads/master-txnstats
Commit: e569ef09052b9ed705d61cb8a3bd5d963127abde
Parents: 06a4f98
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Jul 20 17:08:15 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Fri Jul 20 17:08:15 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/parse/DDLSemanticAnalyzer.java | 12 ++-
.../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 1 +
.../apache/hadoop/hive/ql/parse/HiveParser.g | 11 ++-
.../clientpositive/truncate_external_force.q | 16 ++++
.../llap/truncate_external_force.q.out | 85 ++++++++++++++++++++
6 files changed, 121 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e569ef09/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index d5a33bd..654185d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -728,6 +728,7 @@ minillaplocal.query.files=\
tez_union_multiinsert.q,\
tez_vector_dynpart_hashjoin_1.q,\
tez_vector_dynpart_hashjoin_2.q,\
+ truncate_external_force.q,\
uber_reduce.q,\
udaf_collect_set_2.q,\
udaf_all_keyword.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/e569ef09/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index b6825ae..9373df6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1447,8 +1447,11 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
String tableName = getUnescapedName((ASTNode) root.getChild(0));
Table table = getTable(tableName, true);
- if (table.getTableType() != TableType.MANAGED_TABLE) {
- throw new SemanticException(ErrorMsg.TRUNCATE_FOR_NON_MANAGED_TABLE.format(tableName));
+ boolean isForce = ast.getFirstChildWithType(HiveParser.TOK_FORCE) != null;
+ if (!isForce) {
+ if (table.getTableType() != TableType.MANAGED_TABLE) {
+ throw new SemanticException(ErrorMsg.TRUNCATE_FOR_NON_MANAGED_TABLE.format(tableName));
+ }
}
if (table.isNonNative()) {
throw new SemanticException(ErrorMsg.TRUNCATE_FOR_NON_NATIVE_TABLE.format(tableName)); //TODO
@@ -1491,9 +1494,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
// Is this a truncate column command
List<String> columnNames = null;
- if (ast.getChildCount() == 2) {
+ ASTNode colNamesNode = (ASTNode) ast.getFirstChildWithType(HiveParser.TOK_TABCOLNAME);
+ if (colNamesNode != null) {
try {
- columnNames = getColumnNames((ASTNode)ast.getChild(1));
+ columnNames = getColumnNames(colNamesNode);
// It would be possible to support this, but this is such a pointless command.
if (AcidUtils.isInsertOnlyTable(table.getParameters())) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e569ef09/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
index 7a63cc4..6641e0d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
@@ -335,6 +335,7 @@ KW_PRIMARY: 'PRIMARY';
KW_FOREIGN: 'FOREIGN';
KW_REFERENCES: 'REFERENCES';
KW_CONSTRAINT: 'CONSTRAINT';
+KW_FORCE: 'FORCE';
KW_ENFORCED: 'ENFORCED';
KW_VALIDATE: 'VALIDATE';
KW_NOVALIDATE: 'NOVALIDATE';
http://git-wip-us.apache.org/repos/asf/hive/blob/e569ef09/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 75a25d5..1f53321 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -334,6 +334,7 @@ TOK_TABNAME;
TOK_TABSRC;
TOK_RESTRICT;
TOK_CASCADE;
+TOK_FORCE;
TOK_TABLESKEWED;
TOK_TABCOLVALUE;
TOK_TABCOLVALUE_PAIR;
@@ -985,6 +986,13 @@ ifNotExists
-> ^(TOK_IFNOTEXISTS)
;
+force
+@init { msgs.push("force clause"); }
+@after { msgs.pop(); }
+ : KW_FORCE
+ -> ^(TOK_FORCE)
+ ;
+
rewriteEnabled
@init { pushMsg("rewrite enabled clause", state); }
@after { popMsg(state); }
@@ -1106,7 +1114,8 @@ createTableStatement
truncateTableStatement
@init { pushMsg("truncate table statement", state); }
@after { popMsg(state); }
- : KW_TRUNCATE KW_TABLE tablePartitionPrefix (KW_COLUMNS LPAREN columnNameList RPAREN)? -> ^(TOK_TRUNCATETABLE tablePartitionPrefix columnNameList?);
+ : KW_TRUNCATE KW_TABLE tablePartitionPrefix (KW_COLUMNS LPAREN columnNameList RPAREN)? force?
+ -> ^(TOK_TRUNCATETABLE tablePartitionPrefix columnNameList? force?);
dropTableStatement
@init { pushMsg("drop statement", state); }
http://git-wip-us.apache.org/repos/asf/hive/blob/e569ef09/ql/src/test/queries/clientpositive/truncate_external_force.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/truncate_external_force.q b/ql/src/test/queries/clientpositive/truncate_external_force.q
new file mode 100644
index 0000000..834cfaa
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/truncate_external_force.q
@@ -0,0 +1,16 @@
+--! qt:dataset:src
+
+create external table external1 (key string, value string) stored as textfile;
+load data local inpath '../../data/files/kv1.txt' into table external1;
+select count(*) from external1;
+truncate table external1 force;
+select count(*) from external1;
+
+
+-- Partitioned table
+create external table external2 (key string, value string) partitioned by (p1 string) stored as textfile;
+load data local inpath '../../data/files/kv1.txt' into table external2 partition (p1='abc');
+select count(*) from external2;
+truncate table external2 partition (p1='abc') force;
+select count(*) from external2;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/e569ef09/ql/src/test/results/clientpositive/llap/truncate_external_force.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/truncate_external_force.q.out b/ql/src/test/results/clientpositive/llap/truncate_external_force.q.out
new file mode 100644
index 0000000..f1f04d9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/truncate_external_force.q.out
@@ -0,0 +1,85 @@
+PREHOOK: query: create external table external1 (key string, value string) stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@external1
+POSTHOOK: query: create external table external1 (key string, value string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@external1
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table external1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@external1
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table external1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@external1
+PREHOOK: query: select count(*) from external1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@external1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from external1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@external1
+#### A masked pattern was here ####
+500
+PREHOOK: query: truncate table external1 force
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@external1
+POSTHOOK: query: truncate table external1 force
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@external1
+PREHOOK: query: select count(*) from external1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@external1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from external1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@external1
+#### A masked pattern was here ####
+0
+PREHOOK: query: create external table external2 (key string, value string) partitioned by (p1 string) stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@external2
+POSTHOOK: query: create external table external2 (key string, value string) partitioned by (p1 string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@external2
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table external2 partition (p1='abc')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@external2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table external2 partition (p1='abc')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@external2
+POSTHOOK: Output: default@external2@p1=abc
+PREHOOK: query: select count(*) from external2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@external2
+PREHOOK: Input: default@external2@p1=abc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from external2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@external2
+POSTHOOK: Input: default@external2@p1=abc
+#### A masked pattern was here ####
+500
+PREHOOK: query: truncate table external2 partition (p1='abc') force
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@external2@p1=abc
+POSTHOOK: query: truncate table external2 partition (p1='abc') force
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@external2@p1=abc
+PREHOOK: query: select count(*) from external2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@external2
+PREHOOK: Input: default@external2@p1=abc
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from external2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@external2
+POSTHOOK: Input: default@external2@p1=abc
+#### A masked pattern was here ####
+0
[4/4] hive git commit: HIVE-19416 : merge master into branch (Sergey
Shelukhin) 0721
Posted by se...@apache.org.
HIVE-19416 : merge master into branch (Sergey Shelukhin) 0721
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5c8b5d8b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5c8b5d8b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5c8b5d8b
Branch: refs/heads/master-txnstats
Commit: 5c8b5d8b62beaffedd0681683638bf8bb7d98402
Parents: ddef895 788daf6
Author: sergey <se...@apache.org>
Authored: Sat Jul 21 12:51:36 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Sat Jul 21 12:51:36 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/GenMRTableScan1.java | 5 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 12 +-
.../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 1 +
.../apache/hadoop/hive/ql/parse/HiveParser.g | 11 +-
.../hive/ql/parse/ProcessAnalyzeTable.java | 5 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 6 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 5 +-
.../hive/ql/stats/BasicStatsNoJobTask.java | 9 +
.../queries/clientpositive/acid_no_buckets.q | 8 +
.../clientpositive/truncate_external_force.q | 16 ++
.../clientpositive/acid_table_stats.q.out | 16 +-
.../materialized_view_create_rewrite.q.out | 58 ++---
.../clientpositive/llap/acid_no_buckets.q.out | 210 +++++++++++++++++--
.../materialized_view_create_rewrite_4.q.out | 6 +-
.../materialized_view_create_rewrite_5.q.out | 2 +-
.../llap/truncate_external_force.q.out | 85 ++++++++
17 files changed, 393 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5c8b5d8b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5c8b5d8b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5c8b5d8b/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/acid_table_stats.q.out
index f3ada34,2fe4f97..32badd0
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@@ -174,10 -211,9 +176,10 @@@ Database: defaul
Table: acid
#### A masked pattern was here ####
Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 2
numRows 1000
- rawDataSize 208000
+ rawDataSize 0
totalSize 4063
#### A masked pattern was here ####
@@@ -225,10 -261,10 +227,10 @@@ Database: defaul
Table: acid
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 2
numRows 1000
- rawDataSize 208000
+ rawDataSize 0
totalSize 4063
#### A masked pattern was here ####
@@@ -317,11 -389,10 +319,11 @@@ Database: defaul
Table: acid
#### A masked pattern was here ####
Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 4
- numRows 3000
+ numRows 2000
- rawDataSize 208000
+ rawDataSize 0
- totalSize 8118
+ totalSize 8126
#### A masked pattern was here ####
# Storage Information
@@@ -364,11 -437,10 +368,11 @@@ Database: defaul
Table: acid
#### A masked pattern was here ####
Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 4
numRows 2000
- rawDataSize 416000
+ rawDataSize 0
- totalSize 8118
+ totalSize 8126
#### A masked pattern was here ####
# Storage Information
@@@ -433,11 -541,11 +437,11 @@@ STAGE PLANS
TableScan
alias: acid
filterExpr: (ds = '2008-04-08') (type: boolean)
- Statistics: Num rows: 2000 Data size: 416000 Basic stats: COMPLETE Column stats: NONE
- Statistics: Num rows: 2000 Data size: 81180 Basic stats: COMPLETE Column stats: NONE
++ Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 2000 Data size: 416000 Basic stats: COMPLETE Column stats: NONE
- Statistics: Num rows: 2000 Data size: 81180 Basic stats: COMPLETE Column stats: NONE
++ Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(key)
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/5c8b5d8b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
index fe46bfd,bfa204e..42bfa4b
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
@@@ -949,8 -948,8 +949,8 @@@ Table Parameters
bucketing_version 2
numFiles 3
numRows 3
- rawDataSize 248
+ rawDataSize 0
- totalSize 1508
+ totalSize 1500
transactional true
transactional_properties default
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/5c8b5d8b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
----------------------------------------------------------------------
[3/4] hive git commit: HIVE-20046 : remove NUM_FILES check (Sergey
Shelukhin)
Posted by se...@apache.org.
HIVE-20046 : remove NUM_FILES check (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddef8952
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddef8952
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddef8952
Branch: refs/heads/master-txnstats
Commit: ddef89525eaef9f5eeea4029475a24b891a41df7
Parents: 31ee870
Author: sergey <se...@apache.org>
Authored: Sat Jul 21 12:32:43 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Sat Jul 21 12:32:43 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/metastore/ObjectStore.java | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ddef8952/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b319e68..9c46b68 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -12473,13 +12473,6 @@ public class ObjectStore implements RawStore, Configurable {
return false;
}
- // TODO## NUM_FILES could also be set to 0 by invalid update. We need to have a negative test. Or remove this and fix stuff.
- // If the NUM_FILES of the table/partition is 0, return 'true' from this method.
- // Since newly initialized empty table has 0 for the parameter.
- if (Long.parseLong(statsParams.get(StatsSetupConst.NUM_FILES)) == 0) {
- return true;
- }
-
if (queryValidWriteIdList != null) { // Can be null when stats are being reset to invalid.
ValidWriteIdList list4TheQuery = new ValidReaderWriteIdList(queryValidWriteIdList);
// Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it),
[2/4] hive git commit: HIVE-20115 : Acid tables should not use footer
scan for analyze (Sergey Shelukhin, reviewed by Eugene Koifman)
Posted by se...@apache.org.
HIVE-20115 : Acid tables should not use footer scan for analyze (Sergey Shelukhin, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/788daf60
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/788daf60
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/788daf60
Branch: refs/heads/master-txnstats
Commit: 788daf609fd9c666dcb42451539917edd4d820ad
Parents: e569ef0
Author: sergey <se...@apache.org>
Authored: Sat Jul 21 12:16:36 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Sat Jul 21 12:16:36 2018 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/GenMRTableScan1.java | 5 +-
.../hive/ql/parse/ProcessAnalyzeTable.java | 5 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 6 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 5 +-
.../hive/ql/stats/BasicStatsNoJobTask.java | 9 +
.../queries/clientpositive/acid_no_buckets.q | 8 +
.../clientpositive/acid_table_stats.q.out | 28 +--
.../materialized_view_create_rewrite.q.out | 58 ++---
.../clientpositive/llap/acid_no_buckets.q.out | 210 +++++++++++++++++--
.../materialized_view_create_rewrite_4.q.out | 10 +-
.../materialized_view_create_rewrite_5.q.out | 2 +-
11 files changed, 279 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
index 64f9c70..6295d7f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
import org.apache.hadoop.hive.ql.lib.Node;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.hive.ql.plan.BasicStatsWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask;
import org.apache.hadoop.mapred.InputFormat;
/**
@@ -84,8 +86,7 @@ public class GenMRTableScan1 implements NodeProcessor {
if (parseCtx.getQueryProperties().isAnalyzeCommand()) {
boolean noScan = parseCtx.getQueryProperties().isNoScanAnalyzeCommand();
- if (OrcInputFormat.class.isAssignableFrom(inputFormat) ||
- MapredParquetInputFormat.class.isAssignableFrom(inputFormat)) {
+ if (BasicStatsNoJobTask.canUseFooterScan(table, inputFormat)) {
// For ORC and Parquet, all the following statements are the same
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
index 03cceac..31c5c4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +40,7 @@ import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.BasicStatsWork;
import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask;
import org.apache.hadoop.mapred.InputFormat;
/**
@@ -86,8 +88,7 @@ public class ProcessAnalyzeTable implements NodeProcessor {
assert alias != null;
TezWork tezWork = context.currentTask.getWork();
- if (OrcInputFormat.class.isAssignableFrom(inputFormat) ||
- MapredParquetInputFormat.class.isAssignableFrom(inputFormat)) {
+ if (BasicStatsNoJobTask.canUseFooterScan(table, inputFormat)) {
// For ORC & Parquet, all the following statements are the same
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 49709e5..7a2a2c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.parse;
import com.google.common.collect.Interner;
import com.google.common.collect.Interners;
+
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.HiveStatsUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -62,6 +63,7 @@ import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.DefaultFetchFormatter;
import org.apache.hadoop.hive.serde2.NoOpFetchFormatter;
@@ -381,7 +383,9 @@ public abstract class TaskCompiler {
TableSpec tableSpec = new TableSpec(table, partitions);
tableScan.getConf().getTableMetadata().setTableSpec(tableSpec);
- if (inputFormat.equals(OrcInputFormat.class)) {
+ // Note: this should probably use BasicStatsNoJobTask.canUseFooterScan, but it doesn't check
+ // Parquet for some reason. I'm keeping the existing behavior for now.
+ if (inputFormat.equals(OrcInputFormat.class) && !AcidUtils.isTransactionalTable(table)) {
// For ORC, there is no Tez Job for table stats.
StatsWork columnStatsWork = new StatsWork(table, parseContext.getConf());
columnStatsWork.setFooterScan();
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
index 28d4de7..a9ba302 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.BasicStatsWork;
+import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask;
import org.apache.hadoop.mapred.InputFormat;
import com.google.common.base.Preconditions;
@@ -96,8 +98,7 @@ public class SparkProcessAnalyzeTable implements NodeProcessor {
Preconditions.checkArgument(alias != null, "AssertionError: expected alias to be not null");
SparkWork sparkWork = context.currentTask.getWork();
- if (OrcInputFormat.class.isAssignableFrom(inputFormat) ||
- MapredParquetInputFormat.class.isAssignableFrom(inputFormat)) {
+ if (BasicStatsNoJobTask.canUseFooterScan(table, inputFormat)) {
// For ORC & Parquet, all the following statements are the same
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
index 3128ee8..fd06098 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -71,6 +73,8 @@ import com.google.common.collect.Multimaps;
* faster to compute the table/partition statistics by reading the footer than scanning all the
* rows. This task can be used for computing basic stats like numFiles, numRows, fileSize,
* rawDataSize from ORC footer.
+ * However, this cannot be used for full ACID tables, since some of the files may contain updates
+ * and deletes to existing rows, so summing up the per-file row counts is invalid.
**/
public class BasicStatsNoJobTask implements IStatsProcessor {
@@ -86,6 +90,11 @@ public class BasicStatsNoJobTask implements IStatsProcessor {
console = new LogHelper(LOG);
}
+ public static boolean canUseFooterScan(
+ Table table, Class<? extends InputFormat> inputFormat) {
+ return (OrcInputFormat.class.isAssignableFrom(inputFormat) && !AcidUtils.isFullAcidTable(table))
+ || MapredParquetInputFormat.class.isAssignableFrom(inputFormat);
+ }
@Override
public void initialize(CompilationOpContext opContext) {
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/test/queries/clientpositive/acid_no_buckets.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_no_buckets.q b/ql/src/test/queries/clientpositive/acid_no_buckets.q
index bcf9e06..552010a 100644
--- a/ql/src/test/queries/clientpositive/acid_no_buckets.q
+++ b/ql/src/test/queries/clientpositive/acid_no_buckets.q
@@ -28,8 +28,16 @@ select ds, hr, key, value from srcpart_acid where value like '%updated' order by
insert into srcpart_acid PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003');
select ds, hr, key, value from srcpart_acid where cast(key as integer) > 1000 order by ds, hr, cast(key as integer);
+describe formatted srcpart_acid;
+describe formatted srcpart_acid key;
+
analyze table srcpart_acid PARTITION(ds, hr) compute statistics;
analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns;
+
+-- make sure the stats stay the same after analyze (insert and update above also update stats)
+describe formatted srcpart_acid;
+describe formatted srcpart_acid key;
+
explain delete from srcpart_acid where key in( '1001', '213', '43');
--delete some rows from initial load, some that were updated and some that were inserted
delete from srcpart_acid where key in( '1001', '213', '43');
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out
index 841a5a4..2fe4f97 100644
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@ -182,11 +182,13 @@ POSTHOOK: Input: default@acid@ds=2008-04-08
PREHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@acid
+PREHOOK: Input: default@acid@ds=2008-04-08
PREHOOK: Output: default@acid
PREHOOK: Output: default@acid@ds=2008-04-08
POSTHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@acid
+POSTHOOK: Input: default@acid@ds=2008-04-08
POSTHOOK: Output: default@acid
POSTHOOK: Output: default@acid@ds=2008-04-08
PREHOOK: query: desc formatted acid partition(ds='2008-04-08')
@@ -209,10 +211,9 @@ Database: default
Table: acid
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 2
numRows 1000
- rawDataSize 208000
+ rawDataSize 0
totalSize 4063
#### A masked pattern was here ####
@@ -260,10 +261,10 @@ Database: default
Table: acid
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
+ COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 2
numRows 1000
- rawDataSize 208000
+ rawDataSize 0
totalSize 4063
#### A masked pattern was here ####
@@ -303,9 +304,9 @@ STAGE PLANS:
TableScan
alias: acid
filterExpr: (ds = '2008-04-08') (type: boolean)
- Statistics: Num rows: 1000 Data size: 208000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 40630 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 1000 Data size: 208000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 40630 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -390,7 +391,7 @@ Table: acid
Partition Parameters:
numFiles 4
numRows 3000
- rawDataSize 208000
+ rawDataSize 0
totalSize 8118
#### A masked pattern was here ####
@@ -407,11 +408,13 @@ Storage Desc Params:
PREHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@acid
+PREHOOK: Input: default@acid@ds=2008-04-08
PREHOOK: Output: default@acid
PREHOOK: Output: default@acid@ds=2008-04-08
POSTHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@acid
+POSTHOOK: Input: default@acid@ds=2008-04-08
POSTHOOK: Output: default@acid
POSTHOOK: Output: default@acid@ds=2008-04-08
PREHOOK: query: desc formatted acid partition(ds='2008-04-08')
@@ -434,10 +437,9 @@ Database: default
Table: acid
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 4
numRows 2000
- rawDataSize 416000
+ rawDataSize 0
totalSize 8118
#### A masked pattern was here ####
@@ -466,9 +468,9 @@ STAGE PLANS:
TableScan
alias: acid
filterExpr: (ds = '2008-04-08') (type: boolean)
- Statistics: Num rows: 2000 Data size: 416000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 81180 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 2000 Data size: 416000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 81180 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -539,11 +541,11 @@ STAGE PLANS:
TableScan
alias: acid
filterExpr: (ds = '2008-04-08') (type: boolean)
- Statistics: Num rows: 2000 Data size: 416000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 81180 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 2000 Data size: 416000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 81180 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(key)
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
index 4db5d70..7813aac 100644
--- a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
@@ -175,32 +175,32 @@ STAGE PLANS:
TableScan
alias: cmv_basetable_n10
filterExpr: (a = 3) (type: boolean)
- Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 9740 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
- Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 3896 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: decimal(10,2))
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 3896 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 3896 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(10,2))
TableScan
alias: cmv_basetable_n10
filterExpr: ((d = 3) and (a = 3)) (type: boolean)
- Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 9740 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: decimal(10,2))
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(10,2))
Reduce Operator Tree:
Join Operator
@@ -210,14 +210,14 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 7794 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 7794 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 7794 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -298,17 +298,17 @@ STAGE PLANS:
TableScan
alias: cmv_basetable_n10
filterExpr: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 9740 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: decimal(10,2))
outputColumnNames: _col1
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(10,2))
TableScan
alias: default.cmv_mat_view2_n4
@@ -325,14 +325,14 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 4130 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col0 (type: int), _col6 (type: decimal(10,2))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 4130 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 4130 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -399,32 +399,32 @@ STAGE PLANS:
TableScan
alias: cmv_basetable_n10
filterExpr: (a = 3) (type: boolean)
- Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 9740 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
- Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 3896 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: decimal(10,2))
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 3896 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 3896 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(10,2))
TableScan
alias: cmv_basetable_n10
filterExpr: ((d = 3) and (a = 3)) (type: boolean)
- Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 9740 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: decimal(10,2))
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1948 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(10,2))
Reduce Operator Tree:
Join Operator
@@ -434,14 +434,14 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 7794 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 7794 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 7794 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
index 36a6a5d..9762de6 100644
--- a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
@@ -58,6 +58,10 @@ POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acid
PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
@@ -66,6 +70,10 @@ PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acid
POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
@@ -239,9 +247,74 @@ POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
2008-04-08 11 1001 val1001
2008-04-08 11 1002 val1002
2008-04-08 11 1003 val1003
+PREHOOK: query: describe formatted srcpart_acid
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@srcpart_acid
+POSTHOOK: query: describe formatted srcpart_acid
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@srcpart_acid
+# col_name data_type comment
+key string
+value string
+
+# Partition Information
+# col_name data_type comment
+ds string
+hr string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ bucketing_version 2
+ numFiles 9
+ numPartitions 4
+ numRows 2003
+ rawDataSize 0
+ totalSize 17988
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: describe formatted srcpart_acid key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@srcpart_acid
+POSTHOOK: query: describe formatted srcpart_acid key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@srcpart_acid
+col_name key
+data_type string
+min
+max
+num_nulls 0
+distinct_count 316
+avg_col_len 2.812
+max_col_len 3
+num_trues
+num_falses
+bitVector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {}
PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acid
PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
@@ -250,6 +323,10 @@ PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acid
POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
@@ -281,6 +358,67 @@ POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
#### A masked pattern was here ####
+PREHOOK: query: describe formatted srcpart_acid
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@srcpart_acid
+POSTHOOK: query: describe formatted srcpart_acid
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@srcpart_acid
+# col_name data_type comment
+key string
+value string
+
+# Partition Information
+# col_name data_type comment
+ds string
+hr string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ bucketing_version 2
+ numFiles 9
+ numPartitions 4
+ numRows 2003
+ rawDataSize 0
+ totalSize 17988
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: describe formatted srcpart_acid key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@srcpart_acid
+POSTHOOK: query: describe formatted srcpart_acid key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@srcpart_acid
+col_name key
+data_type string
+min
+max
+num_nulls 0
+distinct_count 316
+avg_col_len 2.812
+max_col_len 3
+num_trues
+num_falses
+bitVector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {}
PREHOOK: query: explain delete from srcpart_acid where key in( '1001', '213', '43')
PREHOOK: type: QUERY
POSTHOOK: query: explain delete from srcpart_acid where key in( '1001', '213', '43')
@@ -304,19 +442,19 @@ STAGE PLANS:
TableScan
alias: srcpart_acid
filterExpr: (key) IN ('1001', '213', '43') (type: boolean)
- Statistics: Num rows: 2015 Data size: 916825 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 2003 Data size: 911365 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key) IN ('1001', '213', '43') (type: boolean)
- Statistics: Num rows: 101 Data size: 45955 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 45500 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string), _col2 (type: string)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -326,10 +464,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -561,6 +699,10 @@ POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
PREHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acidb
PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
@@ -569,6 +711,10 @@ PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acidb
POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
@@ -745,6 +891,10 @@ POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
PREHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acidb
PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
@@ -753,6 +903,10 @@ PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acidb
POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
@@ -807,19 +961,19 @@ STAGE PLANS:
TableScan
alias: srcpart_acidb
filterExpr: (key) IN ('1001', '213', '43') (type: boolean)
- Statistics: Num rows: 2015 Data size: 916825 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 2003 Data size: 911365 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key) IN ('1001', '213', '43') (type: boolean)
- Statistics: Num rows: 101 Data size: 45955 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 45500 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string), _col2 (type: string)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -829,10 +983,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 101 Data size: 44844 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 100 Data size: 44400 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1064,6 +1218,10 @@ POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
PREHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acidv
PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
@@ -1072,6 +1230,10 @@ PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acidv
POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
@@ -1264,6 +1426,10 @@ POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
PREHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acidv
PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
@@ -1272,6 +1438,10 @@ PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acidv
POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
@@ -1850,6 +2020,10 @@ POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
PREHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acidvb
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acidvb
PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
@@ -1858,6 +2032,10 @@ PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acidvb
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acidvb
POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
@@ -2050,6 +2228,10 @@ POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
PREHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart_acidvb
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
PREHOOK: Output: default@srcpart_acidvb
PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
@@ -2058,6 +2240,10 @@ PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
POSTHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart_acidvb
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
POSTHOOK: Output: default@srcpart_acidvb
POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
index eda3985..bfa204e 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
@@ -265,11 +265,11 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MATERIALIZED_VIEW
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"_c2\":\"true\",\"a\":\"true\",\"c\":\"true\"}}
+ COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"_c2\":\"true\",\"a\":\"true\",\"c\":\"true\"}}
bucketing_version 2
numFiles 2
numRows 2
- rawDataSize 248
+ rawDataSize 0
totalSize 736
transactional true
transactional_properties default
@@ -494,11 +494,11 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MATERIALIZED_VIEW
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"_c2\":\"true\",\"a\":\"true\",\"c\":\"true\"}}
+ COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"_c2\":\"true\",\"a\":\"true\",\"c\":\"true\"}}
bucketing_version 2
numFiles 2
numRows 2
- rawDataSize 248
+ rawDataSize 0
totalSize 736
transactional true
transactional_properties default
@@ -948,7 +948,7 @@ Table Parameters:
bucketing_version 2
numFiles 3
numRows 3
- rawDataSize 248
+ rawDataSize 0
totalSize 1508
transactional true
transactional_properties default
http://git-wip-us.apache.org/repos/asf/hive/blob/788daf60/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
index 99832ff..44eca6d 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
@@ -405,7 +405,7 @@ Table Parameters:
bucketing_version 2
numFiles 2
numRows 5
- rawDataSize 348
+ rawDataSize 0
totalSize 1071
transactional true
transactional_properties default