You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/07/25 09:49:50 UTC
[1/2] hive git commit: HIVE-20069: Fix reoptimization in case of DPP
and Semijoin optimization (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 7eba98837 -> 9326ba696
HIVE-20069: Fix reoptimization in case of DPP and Semijoin optimization (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da3b516d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da3b516d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da3b516d
Branch: refs/heads/branch-3
Commit: da3b516d84c28fe76e2e70a5652c9c25839ed365
Parents: 7eba988
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Mon Jul 23 13:54:09 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Wed Jul 25 10:54:57 2018 +0200
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 2 +
.../hadoop/hive/ql/parse/TezCompiler.java | 72 ++++
.../hive/ql/plan/mapper/StatsSources.java | 9 +-
.../hadoop/hive/ql/stats/OperatorStats.java | 7 +
ql/src/test/queries/clientpositive/reopt_dpp.q | 62 +++
.../queries/clientpositive/reopt_semijoin.q | 76 ++++
.../results/clientpositive/llap/reopt_dpp.q.out | 259 ++++++++++++
.../clientpositive/llap/reopt_semijoin.q.out | 420 +++++++++++++++++++
8 files changed, 905 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index d28fa67..863bf95 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -476,6 +476,8 @@ minillaplocal.query.files=\
bucketmapjoin6.q,\
bucketmapjoin7.q,\
bucketpruning1.q,\
+ reopt_dpp.q,\
+ reopt_semijoin.q,\
retry_failure.q,\
retry_failure_stat_changes.q,\
retry_failure_oom.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 205e726..cffa176 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
import org.apache.hadoop.hive.ql.optimizer.stats.annotation.AnnotateWithStatistics;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
@@ -107,8 +108,10 @@ import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.ql.stats.OperatorStats;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBloomFilter.GenericUDAFBloomFilterEvaluator;
import org.slf4j.Logger;
@@ -220,6 +223,10 @@ public class TezCompiler extends TaskCompiler {
}
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Shared scans optimization");
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
+ markOperatorsWithUnstableRuntimeStats(procCtx);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "markOperatorsWithUnstableRuntimeStats");
+
// need a new run of the constant folding because we might have created lots
// of "and true and true" conditions.
// Rather than run the full constant folding just need to shortcut AND/OR expressions
@@ -1036,6 +1043,71 @@ public class TezCompiler extends TaskCompiler {
ogw.startWalking(topNodes, null);
}
+ private static class MarkTsOfSemijoinsAsIncorrect implements NodeProcessor {
+
+ private PlanMapper planMapper;
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs)
+ throws SemanticException {
+ ParseContext pCtx = ((OptimizeTezProcContext) procCtx).parseContext;
+ planMapper = pCtx.getContext().getPlanMapper();
+ if (nd instanceof ReduceSinkOperator) {
+ ReduceSinkOperator rs = (ReduceSinkOperator) nd;
+ SemiJoinBranchInfo sjInfo = pCtx.getRsToSemiJoinBranchInfo().get(rs);
+ if (sjInfo == null) {
+ return null;
+ }
+ walkSubtree(sjInfo.getTsOp());
+ }
+ if (nd instanceof AppMasterEventOperator) {
+ AppMasterEventOperator ame = (AppMasterEventOperator) nd;
+ AppMasterEventDesc c = ame.getConf();
+ if (c instanceof DynamicPruningEventDesc) {
+ DynamicPruningEventDesc dped = (DynamicPruningEventDesc) c;
+ mark(dped.getTableScan());
+ }
+ }
+ return null;
+ }
+
+ private void walkSubtree(Operator<?> root) {
+ Deque<Operator<?>> deque = new LinkedList<>();
+ deque.add(root);
+ while (!deque.isEmpty()) {
+ Operator<?> op = deque.pollLast();
+ mark(op);
+ if (op instanceof ReduceSinkOperator) {
+ // Done with this branch
+ } else {
+ deque.addAll(op.getChildOperators());
+ }
+ }
+ }
+
+ private void mark(Operator<?> op) {
+ planMapper.link(op, new OperatorStats.IncorrectRuntimeStatsMarker());
+ }
+
+ }
+
+ private void markOperatorsWithUnstableRuntimeStats(OptimizeTezProcContext procCtx) throws SemanticException {
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ opRules.put(
+ new RuleRegExp("R1",
+ ReduceSinkOperator.getOperatorName() + "%"),
+ new MarkTsOfSemijoinsAsIncorrect());
+ opRules.put(
+ new RuleRegExp("R2",
+ AppMasterEventOperator.getOperatorName() + "%"),
+ new MarkTsOfSemijoinsAsIncorrect());
+ Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx);
+ List<Node> topNodes = new ArrayList<Node>();
+ topNodes.addAll(procCtx.parseContext.getTopOps().values());
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
+ ogw.startWalking(topNodes, null);
+ }
+
private boolean findParallelSemiJoinBranch(Operator<?> mapjoin, TableScanOperator bigTableTS,
ParseContext parseContext,
Map<ReduceSinkOperator, TableScanOperator> semijoins) {
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
index 5a62046..823cb87 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
@@ -95,9 +95,14 @@ public class StatsSources {
}
LOG.debug(sb.toString());
}
- if (stat.size() >= 1 && sig.size() >= 1) {
- map.put(sig.get(0), stat.get(0));
+ if (stat.size() < 1 || sig.size() < 1) {
+ continue;
}
+ if (e.getAll(OperatorStats.IncorrectRuntimeStatsMarker.class).size() > 0) {
+ LOG.debug("Ignoring {}, marked with OperatorStats.IncorrectRuntimeStatsMarker", sig.get(0));
+ continue;
+ }
+ map.put(sig.get(0), stat.get(0));
}
return map.build();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java
index d70bb82..0c56c82 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java
@@ -23,6 +23,11 @@ import com.google.common.base.Objects;
* Holds information an operator's statistics.
*/
public final class OperatorStats {
+
+ /** Marker class to help with plan elements which will collect invalid statistics */
+ public static class IncorrectRuntimeStatsMarker {
+ }
+
private String operatorId;
private long outputRecords;
@@ -67,4 +72,6 @@ public final class OperatorStats {
return Objects.equal(operatorId, o.operatorId) &&
Objects.equal(outputRecords, o.outputRecords);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/test/queries/clientpositive/reopt_dpp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/reopt_dpp.q b/ql/src/test/queries/clientpositive/reopt_dpp.q
new file mode 100644
index 0000000..952dcbe
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/reopt_dpp.q
@@ -0,0 +1,62 @@
+set hive.explain.user=true;
+set hive.optimize.index.filter=true;
+set hive.auto.convert.join=true;
+set hive.vectorized.execution.enabled=true;
+
+drop table if exists x1_store_sales;
+drop table if exists x1_date_dim;
+drop table if exists x1_item;
+
+create table x1_store_sales
+(
+ ss_item_sk int
+)
+partitioned by (ss_sold_date_sk int)
+stored as orc;
+
+create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc;
+
+
+insert into x1_date_dim values (1,1,2000,2),
+ (2,2,2001,2);
+insert into x1_store_sales partition (ss_sold_date_sk=1) values (1);
+insert into x1_store_sales partition (ss_sold_date_sk=2) values (2);
+
+alter table x1_store_sales partition (ss_sold_date_sk=1) update statistics set(
+'numRows'='123456',
+'rawDataSize'='1234567');
+
+alter table x1_date_dim update statistics set(
+'numRows'='56',
+'rawDataSize'='81449');
+
+
+-- the following query is designed to produce a DPP plan
+explain
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000;
+
+-- tablescan of s should be 2 or 123456 rows; but never 1
+-- and it should not be a mapjoin :)
+explain reoptimization
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000;
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/test/queries/clientpositive/reopt_semijoin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/reopt_semijoin.q b/ql/src/test/queries/clientpositive/reopt_semijoin.q
new file mode 100644
index 0000000..0eacb8a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/reopt_semijoin.q
@@ -0,0 +1,76 @@
+set hive.explain.user=true;
+set hive.optimize.index.filter=true;
+set hive.auto.convert.join=true;
+set hive.vectorized.execution.enabled=true;
+
+drop table if exists x1_store_sales;
+drop table if exists x1_date_dim;
+
+create table x1_store_sales
+(
+ ss_sold_date_sk int,
+ ss_item_sk int
+)
+stored as orc;
+
+create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc;
+
+insert into x1_date_dim values (1,1,2000,1),
+ (2,2,2001,2),
+ (3,2,2001,3),
+ (4,2,2001,4),
+ (5,2,2001,5),
+ (6,2,2001,6),
+ (7,2,2001,7),
+ (8,2,2001,8);
+
+insert into x1_store_sales values (1,1),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10),(11,11);
+
+alter table x1_store_sales update statistics set(
+'numRows'='123456',
+'rawDataSize'='1234567');
+
+alter table x1_date_dim update statistics set(
+'numRows'='56',
+'rawDataSize'='81449');
+
+
+set hive.auto.convert.join.noconditionaltask.size=1;
+set hive.tez.dynamic.partition.pruning=true;
+set hive.tez.dynamic.semijoin.reduction=true;
+set hive.optimize.index.filter=true;
+set hive.tez.bigtable.minsize.semijoin.reduction=1;
+set hive.tez.min.bloom.filter.entries=1;
+set hive.tez.bloom.filter.factor=1.0f;
+set hive.explain.user=false;
+
+explain
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+;
+
+explain reoptimization
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/test/results/clientpositive/llap/reopt_dpp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/reopt_dpp.q.out b/ql/src/test/results/clientpositive/llap/reopt_dpp.q.out
new file mode 100644
index 0000000..31726f6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/reopt_dpp.q.out
@@ -0,0 +1,259 @@
+PREHOOK: query: drop table if exists x1_store_sales
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists x1_store_sales
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists x1_date_dim
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists x1_date_dim
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists x1_item
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists x1_item
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table x1_store_sales
+(
+ ss_item_sk int
+)
+partitioned by (ss_sold_date_sk int)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@x1_store_sales
+POSTHOOK: query: create table x1_store_sales
+(
+ ss_item_sk int
+)
+partitioned by (ss_sold_date_sk int)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@x1_store_sales
+PREHOOK: query: create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@x1_date_dim
+POSTHOOK: query: create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@x1_date_dim
+PREHOOK: query: insert into x1_date_dim values (1,1,2000,2),
+ (2,2,2001,2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x1_date_dim
+POSTHOOK: query: insert into x1_date_dim values (1,1,2000,2),
+ (2,2,2001,2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x1_date_dim
+POSTHOOK: Lineage: x1_date_dim.d_date_sk SCRIPT []
+POSTHOOK: Lineage: x1_date_dim.d_month_seq SCRIPT []
+POSTHOOK: Lineage: x1_date_dim.d_moy SCRIPT []
+POSTHOOK: Lineage: x1_date_dim.d_year SCRIPT []
+PREHOOK: query: insert into x1_store_sales partition (ss_sold_date_sk=1) values (1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x1_store_sales@ss_sold_date_sk=1
+POSTHOOK: query: insert into x1_store_sales partition (ss_sold_date_sk=1) values (1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x1_store_sales@ss_sold_date_sk=1
+POSTHOOK: Lineage: x1_store_sales PARTITION(ss_sold_date_sk=1).ss_item_sk SCRIPT []
+PREHOOK: query: insert into x1_store_sales partition (ss_sold_date_sk=2) values (2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x1_store_sales@ss_sold_date_sk=2
+POSTHOOK: query: insert into x1_store_sales partition (ss_sold_date_sk=2) values (2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x1_store_sales@ss_sold_date_sk=2
+POSTHOOK: Lineage: x1_store_sales PARTITION(ss_sold_date_sk=2).ss_item_sk SCRIPT []
+PREHOOK: query: alter table x1_store_sales partition (ss_sold_date_sk=1) update statistics set(
+'numRows'='123456',
+'rawDataSize'='1234567')
+PREHOOK: type: ALTERTABLE_UPDATEPARTSTATS
+PREHOOK: Input: default@x1_store_sales
+PREHOOK: Output: default@x1_store_sales@ss_sold_date_sk=1
+POSTHOOK: query: alter table x1_store_sales partition (ss_sold_date_sk=1) update statistics set(
+'numRows'='123456',
+'rawDataSize'='1234567')
+POSTHOOK: type: ALTERTABLE_UPDATEPARTSTATS
+POSTHOOK: Input: default@x1_store_sales
+POSTHOOK: Input: default@x1_store_sales@ss_sold_date_sk=1
+POSTHOOK: Output: default@x1_store_sales@ss_sold_date_sk=1
+PREHOOK: query: alter table x1_date_dim update statistics set(
+'numRows'='56',
+'rawDataSize'='81449')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@x1_date_dim
+PREHOOK: Output: default@x1_date_dim
+POSTHOOK: query: alter table x1_date_dim update statistics set(
+'numRows'='56',
+'rawDataSize'='81449')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@x1_date_dim
+POSTHOOK: Output: default@x1_date_dim
+PREHOOK: query: explain
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_35]
+ Group By Operator [GBY_34] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_33]
+ Group By Operator [GBY_32] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_31] (rows=1728398 width=8)
+ Conds:SEL_30._col0=RS_26._col0(Inner)
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_26]
+ PartitionCols:_col0
+ Select Operator [SEL_25] (rows=28 width=8)
+ Output:["_col0"]
+ Filter Operator [FIL_24] (rows=28 width=8)
+ predicate:((d_year = 2000) and d_date_sk is not null)
+ TableScan [TS_3] (rows=56 width=8)
+ default@x1_date_dim,d,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_year"]
+ Dynamic Partitioning Event Operator [EVENT_29] (rows=1 width=8)
+ Group By Operator [GBY_28] (rows=1 width=8)
+ Output:["_col0"],keys:_col0
+ Select Operator [SEL_27] (rows=28 width=8)
+ Output:["_col0"]
+ Please refer to the previous Select Operator [SEL_25]
+ <-Select Operator [SEL_30] (rows=123457 width=4)
+ Output:["_col0"]
+ TableScan [TS_0] (rows=123457 width=14)
+ default@x1_store_sales,s,Tbl:COMPLETE,Col:COMPLETE
+
+PREHOOK: query: explain reoptimization
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000
+PREHOOK: type: QUERY
+PREHOOK: Input: default@x1_date_dim
+PREHOOK: Input: default@x1_store_sales
+PREHOOK: Input: default@x1_store_sales@ss_sold_date_sk=1
+PREHOOK: Input: default@x1_store_sales@ss_sold_date_sk=2
+#### A masked pattern was here ####
+POSTHOOK: query: explain reoptimization
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@x1_date_dim
+POSTHOOK: Input: default@x1_store_sales
+POSTHOOK: Input: default@x1_store_sales@ss_sold_date_sk=1
+POSTHOOK: Input: default@x1_store_sales@ss_sold_date_sk=2
+#### A masked pattern was here ####
+PREHOOK: query: explain reoptimization
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000
+PREHOOK: type: QUERY
+POSTHOOK: query: explain reoptimization
+select count(*) cnt
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk = d.d_date_sk
+ and d.d_year=2000
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_35]
+ Group By Operator [GBY_34] (runtime: rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_33]
+ Group By Operator [GBY_32] (runtime: rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_31] (runtime: rows=1 width=8)
+ Conds:SEL_30._col0=RS_26._col0(Inner)
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_26]
+ PartitionCols:_col0
+ Select Operator [SEL_25] (runtime: rows=1 width=8)
+ Output:["_col0"]
+ Filter Operator [FIL_24] (runtime: rows=1 width=8)
+ predicate:((d_year = 2000) and d_date_sk is not null)
+ TableScan [TS_3] (runtime: rows=2 width=8)
+ default@x1_date_dim,d,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_year"]
+ Dynamic Partitioning Event Operator [EVENT_29] (runtime: rows=1 width=8)
+ Group By Operator [GBY_28] (runtime: rows=1 width=8)
+ Output:["_col0"],keys:_col0
+ Select Operator [SEL_27] (runtime: rows=1 width=8)
+ Output:["_col0"]
+ Please refer to the previous Select Operator [SEL_25]
+ <-Select Operator [SEL_30] (runtime: rows=1 width=4)
+ Output:["_col0"]
+ TableScan [TS_0] (rows=123457 width=14)
+ default@x1_store_sales,s,Tbl:COMPLETE,Col:COMPLETE
+
http://git-wip-us.apache.org/repos/asf/hive/blob/da3b516d/ql/src/test/results/clientpositive/llap/reopt_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/reopt_semijoin.q.out b/ql/src/test/results/clientpositive/llap/reopt_semijoin.q.out
new file mode 100644
index 0000000..e60b207
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/reopt_semijoin.q.out
@@ -0,0 +1,420 @@
+PREHOOK: query: drop table if exists x1_store_sales
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists x1_store_sales
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists x1_date_dim
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists x1_date_dim
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table x1_store_sales
+(
+ ss_sold_date_sk int,
+ ss_item_sk int
+)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@x1_store_sales
+POSTHOOK: query: create table x1_store_sales
+(
+ ss_sold_date_sk int,
+ ss_item_sk int
+)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@x1_store_sales
+PREHOOK: query: create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@x1_date_dim
+POSTHOOK: query: create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@x1_date_dim
+PREHOOK: query: insert into x1_date_dim values (1,1,2000,1),
+ (2,2,2001,2),
+ (3,2,2001,3),
+ (4,2,2001,4),
+ (5,2,2001,5),
+ (6,2,2001,6),
+ (7,2,2001,7),
+ (8,2,2001,8)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x1_date_dim
+POSTHOOK: query: insert into x1_date_dim values (1,1,2000,1),
+ (2,2,2001,2),
+ (3,2,2001,3),
+ (4,2,2001,4),
+ (5,2,2001,5),
+ (6,2,2001,6),
+ (7,2,2001,7),
+ (8,2,2001,8)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x1_date_dim
+POSTHOOK: Lineage: x1_date_dim.d_date_sk SCRIPT []
+POSTHOOK: Lineage: x1_date_dim.d_month_seq SCRIPT []
+POSTHOOK: Lineage: x1_date_dim.d_moy SCRIPT []
+POSTHOOK: Lineage: x1_date_dim.d_year SCRIPT []
+PREHOOK: query: insert into x1_store_sales values (1,1),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10),(11,11)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x1_store_sales
+POSTHOOK: query: insert into x1_store_sales values (1,1),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10),(11,11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x1_store_sales
+POSTHOOK: Lineage: x1_store_sales.ss_item_sk SCRIPT []
+POSTHOOK: Lineage: x1_store_sales.ss_sold_date_sk SCRIPT []
+PREHOOK: query: alter table x1_store_sales update statistics set(
+'numRows'='123456',
+'rawDataSize'='1234567')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@x1_store_sales
+PREHOOK: Output: default@x1_store_sales
+POSTHOOK: query: alter table x1_store_sales update statistics set(
+'numRows'='123456',
+'rawDataSize'='1234567')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@x1_store_sales
+POSTHOOK: Output: default@x1_store_sales
+PREHOOK: query: alter table x1_date_dim update statistics set(
+'numRows'='56',
+'rawDataSize'='81449')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@x1_date_dim
+PREHOOK: Output: default@x1_date_dim
+POSTHOOK: query: alter table x1_date_dim update statistics set(
+'numRows'='56',
+'rawDataSize'='81449')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@x1_date_dim
+POSTHOOK: Output: default@x1_date_dim
+PREHOOK: query: explain
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 5 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ filterExpr: (ss_sold_date_sk is not null and (ss_sold_date_sk BETWEEN DynamicValue(RS_7_d_d_date_sk_min) AND DynamicValue(RS_7_d_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_d_d_date_sk_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((ss_sold_date_sk BETWEEN DynamicValue(RS_7_d_d_date_sk_min) AND DynamicValue(RS_7_d_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_d_d_date_sk_bloom_filter))) and ss_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: d
+ filterExpr: ((d_moy = 3) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 56 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((d_moy = 3) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1
+ Statistics: Num rows: 86419 Data size: 345676 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain reoptimization
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@x1_date_dim
+PREHOOK: Input: default@x1_store_sales
+#### A masked pattern was here ####
+POSTHOOK: query: explain reoptimization
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@x1_date_dim
+POSTHOOK: Input: default@x1_store_sales
+#### A masked pattern was here ####
+PREHOOK: query: explain reoptimization
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+PREHOOK: type: QUERY
+POSTHOOK: query: explain reoptimization
+select sum(s.ss_item_sk)
+ from
+ x1_store_sales s
+ ,x1_date_dim d
+ where
+ 1=1
+ and s.ss_sold_date_sk=d.d_date_sk
+ and d.d_moy=3
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 5 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ filterExpr: (ss_sold_date_sk is not null and (ss_sold_date_sk BETWEEN DynamicValue(RS_7_d_d_date_sk_min) AND DynamicValue(RS_7_d_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_d_d_date_sk_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((ss_sold_date_sk BETWEEN DynamicValue(RS_7_d_d_date_sk_min) AND DynamicValue(RS_7_d_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_d_d_date_sk_bloom_filter))) and ss_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 123456 Data size: 987648 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: d
+ filterExpr: ((d_moy = 3) and d_date_sk is not null) (type: boolean)
+ Statistics: (RUNTIME) Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((d_moy = 3) and d_date_sk is not null) (type: boolean)
+ Statistics: (RUNTIME) Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: (RUNTIME) Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: (RUNTIME) Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: (RUNTIME) Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: (RUNTIME) Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: (RUNTIME) Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1
+ Statistics: (RUNTIME) Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: (RUNTIME) Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: (RUNTIME) Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: (RUNTIME) Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: (RUNTIME) Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: (RUNTIME) Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
[2/2] hive git commit: HIVE-20231: Backport HIVE-19981 to branch-3
(Daniel Voros via Zoltan Haindrich)
Posted by kg...@apache.org.
HIVE-20231: Backport HIVE-19981 to branch-3 (Daniel Voros via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9326ba69
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9326ba69
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9326ba69
Branch: refs/heads/branch-3
Commit: 9326ba6964da3ffde80169d276373142e978e8df
Parents: da3b516
Author: Daniel Voros <da...@gmail.com>
Authored: Wed Jul 25 11:35:28 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Wed Jul 25 11:35:28 2018 +0200
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +
.../clientpositive/external_table_purge.q | 165 +++++
.../llap/external_table_purge.q.out | 635 +++++++++++++++++++
.../hadoop/hive/metastore/HiveMetaStore.java | 33 +-
.../hive/metastore/utils/MetaStoreUtils.java | 8 +-
7 files changed, 845 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a7b409e..6c2d0ca 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2852,6 +2852,12 @@ public class HiveConf extends Configuration {
" on the assumption that data changes by external applications may have negative effects" +
" on these operations."),
+ HIVE_EXTERNALTABLE_PURGE_DEFAULT("hive.external.table.purge.default", false,
+ "Set to true to set external.table.purge=true on newly created external tables," +
+ " which will specify that the table data should be deleted when the table is dropped." +
+ " Set to false maintain existing behavior that external tables do not delete data" +
+ " when the table is dropped."),
+
HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false,
"Whether to throw an exception if dynamic partition insert generates empty results."),
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 863bf95..78a03db 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -407,6 +407,7 @@ minillap.query.files=acid_bucket_pruning.q,\
reduce_deduplicate_distinct.q, \
remote_script.q,\
file_with_header_footer.q,\
+ external_table_purge.q,\
external_table_with_space_in_location_path.q,\
import_exported_table.q,\
orc_llap_counters.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 26db245..2bd6c5f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -12882,6 +12883,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
}
+
+ if (isExt && HiveConf.getBoolVar(conf, ConfVars.HIVE_EXTERNALTABLE_PURGE_DEFAULT)) {
+ if (retValue.get(MetaStoreUtils.EXTERNAL_TABLE_PURGE) == null) {
+ retValue.put(MetaStoreUtils.EXTERNAL_TABLE_PURGE, "true");
+ }
+ }
+
boolean makeInsertOnly = !isTemporaryTable && HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY);
boolean makeAcid = !isTemporaryTable &&
MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) &&
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/ql/src/test/queries/clientpositive/external_table_purge.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/external_table_purge.q b/ql/src/test/queries/clientpositive/external_table_purge.q
new file mode 100644
index 0000000..144e49a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/external_table_purge.q
@@ -0,0 +1,165 @@
+
+dfs -rmr -f hdfs:///tmp/etp_1;
+
+dfs -mkdir -p hdfs:///tmp/etp_1;
+dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/etp_1/;
+
+create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs:///tmp/etp_1';
+set test.comment=Table should have data;
+set test.comment;
+select count(*) from etp_1;
+
+drop table etp_1;
+
+-- Create external table in same location, data should still be there
+create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs:///tmp/etp_1';
+set test.comment=Table should have data;
+set test.comment;
+select count(*) from etp_1;
+alter table etp_1 set tblproperties ('external.table.purge'='true');
+
+drop table etp_1;
+
+-- Create external table in same location. Data should be gone due to external.table.purge option.
+create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs:///tmp/etp_1';
+set test.comment=Table should have no data;
+set test.comment;
+select count(*) from etp_1;
+
+drop table etp_1;
+
+--
+-- Test hive.external.table.purge.default
+--
+
+dfs -mkdir -p hdfs:///tmp/etp_1;
+dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/etp_1/;
+
+set hive.external.table.purge.default=true;
+
+-- Can still create table and override the default
+create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs:///tmp/etp_1' tblproperties ('external.table.purge'='false');
+show create table etp_1;
+set test.comment=Table should have data;
+set test.comment;
+select count(*) from etp_1;
+
+drop table etp_1;
+
+-- Create with default options, external.table.purge should be set
+create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs:///tmp/etp_1';
+show create table etp_1;
+set test.comment=Table should have data;
+set test.comment;
+select count(*) from etp_1;
+
+drop table etp_1;
+
+-- Data should be gone
+create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs:///tmp/etp_1';
+set test.comment=Table should have no data;
+set test.comment;
+select count(*) from etp_1;
+
+drop table etp_1;
+dfs -rmr -f hdfs:///tmp/etp_1;
+
+set hive.external.table.purge.default=false;
+
+--
+-- Partitioned table
+--
+
+dfs -rmr -f hdfs:///tmp/etp_2;
+dfs -mkdir -p hdfs:///tmp/etp_2/p1=part1;
+dfs -mkdir -p hdfs:///tmp/etp_2/p1=part2;
+dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/etp_2/p1=part1/;
+dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/etp_2/p1=part2/;
+
+create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs:///tmp/etp_2';
+alter table etp_2 add partition (p1='part1');
+alter table etp_2 add partition (p1='part2');
+set test.comment=Table should have full data;
+set test.comment;
+select count(*) from etp_2;
+alter table etp_2 drop partition (p1='part1');
+alter table etp_2 add partition (p1='part1');
+set test.comment=Table should have full data;
+set test.comment;
+select count(*) from etp_2;
+
+drop table etp_2;
+
+-- Create external table in same location, data should still be there
+create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs:///tmp/etp_2';
+alter table etp_2 set tblproperties ('external.table.purge'='true');
+alter table etp_2 add partition (p1='part1');
+alter table etp_2 add partition (p1='part2');
+set test.comment=Table should have full data;
+set test.comment;
+select count(*) from etp_2;
+alter table etp_2 drop partition (p1='part1');
+alter table etp_2 add partition (p1='part1');
+set test.comment=Table should have partial data;
+set test.comment;
+select count(*) from etp_2;
+
+drop table etp_2;
+
+-- Create external table in same location. Data should be gone due to external.table.purge option.
+create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs:///tmp/etp_2';
+alter table etp_2 add partition (p1='part1');
+alter table etp_2 add partition (p1='part2');
+set test.comment=Table should have no data;
+set test.comment;
+select count(*) from etp_2;
+
+drop table etp_2;
+
+-- Test hive.external.table.purge.default
+dfs -mkdir -p hdfs:///tmp/etp_2/p1=part1;
+dfs -mkdir -p hdfs:///tmp/etp_2/p1=part2;
+dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/etp_2/p1=part1/;
+dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/etp_2/p1=part2/;
+
+set hive.external.table.purge.default=true;
+
+-- Can still create table and override the default
+create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs:///tmp/etp_2' tblproperties ('external.table.purge'='false');
+show create table etp_2;
+alter table etp_2 add partition (p1='part1');
+alter table etp_2 add partition (p1='part2');
+set test.comment=Table should have full data;
+set test.comment;
+select count(*) from etp_2;
+
+drop table etp_2;
+
+-- Create with default options, external.table.purge should be set
+create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs:///tmp/etp_2';
+show create table etp_2;
+alter table etp_2 add partition (p1='part1');
+alter table etp_2 add partition (p1='part2');
+set test.comment=Table should have full data;
+set test.comment;
+select count(*) from etp_2;
+alter table etp_2 drop partition (p1='part1');
+alter table etp_2 add partition (p1='part1');
+set test.comment=Table should have partial data;
+set test.comment;
+select count(*) from etp_2;
+
+drop table etp_2;
+
+-- Data should be gone
+create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs:///tmp/etp_2';
+alter table etp_2 add partition (p1='part1');
+alter table etp_2 add partition (p1='part2');
+set test.comment=Table should have no data;
+set test.comment;
+select count(*) from etp_2;
+
+drop table etp_2;
+dfs -rmr -f hdfs:///tmp/etp_2;
+
+set hive.external.table.purge.default=false;
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/ql/src/test/results/clientpositive/llap/external_table_purge.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/external_table_purge.q.out b/ql/src/test/results/clientpositive/llap/external_table_purge.q.out
new file mode 100644
index 0000000..24c778e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/external_table_purge.q.out
@@ -0,0 +1,635 @@
+PREHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_1
+test.comment=Table should have data
+PREHOOK: query: select count(*) from etp_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+500
+PREHOOK: query: drop table etp_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: drop table etp_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_1
+test.comment=Table should have data
+PREHOOK: query: select count(*) from etp_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+500
+PREHOOK: query: alter table etp_1 set tblproperties ('external.table.purge'='true')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: alter table etp_1 set tblproperties ('external.table.purge'='true')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: drop table etp_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: drop table etp_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_1
+test.comment=Table should have no data
+PREHOOK: query: select count(*) from etp_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+PREHOOK: query: drop table etp_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: drop table etp_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###' tblproperties ('external.table.purge'='false')
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###' tblproperties ('external.table.purge'='false')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: show create table etp_1
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@etp_1
+POSTHOOK: query: show create table etp_1
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@etp_1
+CREATE EXTERNAL TABLE `etp_1`(
+ `c1` string,
+ `c2` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+ 'hdfs://### HDFS PATH ###'
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+ 'external.table.purge'='false',
+#### A masked pattern was here ####
+test.comment=Table should have data
+PREHOOK: query: select count(*) from etp_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+500
+PREHOOK: query: drop table etp_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: drop table etp_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: show create table etp_1
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@etp_1
+POSTHOOK: query: show create table etp_1
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@etp_1
+CREATE EXTERNAL TABLE `etp_1`(
+ `c1` string,
+ `c2` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+ 'hdfs://### HDFS PATH ###'
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+ 'external.table.purge'='true',
+#### A masked pattern was here ####
+test.comment=Table should have data
+PREHOOK: query: select count(*) from etp_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+500
+PREHOOK: query: drop table etp_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: drop table etp_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: create external table etp_1 (c1 string, c2 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_1
+test.comment=Table should have no data
+PREHOOK: query: select count(*) from etp_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+PREHOOK: query: drop table etp_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_1
+PREHOOK: Output: default@etp_1
+POSTHOOK: query: drop table etp_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_1
+POSTHOOK: Output: default@etp_1
+PREHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part2
+test.comment=Table should have full data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1000
+PREHOOK: query: alter table etp_2 drop partition (p1='part1')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2@p1=part1
+POSTHOOK: query: alter table etp_2 drop partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+test.comment=Table should have full data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1000
+PREHOOK: query: drop table etp_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: drop table etp_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: alter table etp_2 set tblproperties ('external.table.purge'='true')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 set tblproperties ('external.table.purge'='true')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part2
+test.comment=Table should have full data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1000
+PREHOOK: query: alter table etp_2 drop partition (p1='part1')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2@p1=part1
+POSTHOOK: query: alter table etp_2 drop partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+test.comment=Table should have partial data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+500
+PREHOOK: query: drop table etp_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: drop table etp_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part2
+test.comment=Table should have no data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+PREHOOK: query: drop table etp_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: drop table etp_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###' tblproperties ('external.table.purge'='false')
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###' tblproperties ('external.table.purge'='false')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: show create table etp_2
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@etp_2
+POSTHOOK: query: show create table etp_2
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@etp_2
+CREATE EXTERNAL TABLE `etp_2`(
+ `c1` string,
+ `c2` string)
+PARTITIONED BY (
+ `p1` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+ 'hdfs://### HDFS PATH ###'
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+ 'external.table.purge'='false',
+#### A masked pattern was here ####
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part2
+test.comment=Table should have full data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1000
+PREHOOK: query: drop table etp_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: drop table etp_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: show create table etp_2
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@etp_2
+POSTHOOK: query: show create table etp_2
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@etp_2
+CREATE EXTERNAL TABLE `etp_2`(
+ `c1` string,
+ `c2` string)
+PARTITIONED BY (
+ `p1` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+ 'hdfs://### HDFS PATH ###'
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+ 'external.table.purge'='true',
+#### A masked pattern was here ####
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part2
+test.comment=Table should have full data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1000
+PREHOOK: query: alter table etp_2 drop partition (p1='part1')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2@p1=part1
+POSTHOOK: query: alter table etp_2 drop partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+test.comment=Table should have partial data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+500
+PREHOOK: query: drop table etp_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: drop table etp_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: create external table etp_2 (c1 string, c2 string) partitioned by (p1 string) stored as textfile location 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@etp_2
+PREHOOK: query: alter table etp_2 add partition (p1='part1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part1
+PREHOOK: query: alter table etp_2 add partition (p1='part2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: alter table etp_2 add partition (p1='part2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@etp_2
+POSTHOOK: Output: default@etp_2@p1=part2
+test.comment=Table should have no data
+PREHOOK: query: select count(*) from etp_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@etp_2
+PREHOOK: Input: default@etp_2@p1=part1
+PREHOOK: Input: default@etp_2@p1=part2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from etp_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Input: default@etp_2@p1=part1
+POSTHOOK: Input: default@etp_2@p1=part2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+PREHOOK: query: drop table etp_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@etp_2
+PREHOOK: Output: default@etp_2
+POSTHOOK: query: drop table etp_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@etp_2
+POSTHOOK: Output: default@etp_2
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 1fa37c0..18ba095 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1546,7 +1546,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// If the table is not external and it might not be in a subdirectory of the database
// add it's locations to the list of paths to delete
Path tablePath = null;
- if (table.getSd().getLocation() != null && !isExternal(table)) {
+ boolean tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(table, deleteData);
+ if (table.getSd().getLocation() != null && tableDataShouldBeDeleted) {
tablePath = wh.getDnsPath(new Path(table.getSd().getLocation()));
if (!wh.isWritable(tablePath.getParent())) {
throw new MetaException("Database metadata not deleted since table: " +
@@ -1562,7 +1563,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// For each partition in each table, drop the partitions and get a list of
// partitions' locations which might need to be deleted
partitionPaths = dropPartitionsAndGetLocations(ms, catName, name, table.getTableName(),
- tablePath, table.getPartitionKeys(), deleteData && !isExternal(table));
+ tablePath, table.getPartitionKeys(), tableDataShouldBeDeleted);
// Drop the table but not its data
drop_table(MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf),
@@ -2490,7 +2491,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throws NoSuchObjectException, MetaException, IOException, InvalidObjectException,
InvalidInputException {
boolean success = false;
- boolean isExternal = false;
+ boolean tableDataShouldBeDeleted = false;
Path tblPath = null;
List<Path> partPaths = null;
Table tbl = null;
@@ -2513,7 +2514,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
firePreEvent(new PreDropTableEvent(tbl, deleteData, this));
- isExternal = isExternal(tbl);
+ tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(tbl, deleteData);
if (tbl.getSd().getLocation() != null) {
tblPath = new Path(tbl.getSd().getLocation());
if (!wh.isWritable(tblPath.getParent())) {
@@ -2526,7 +2527,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// Drop the partitions and get a list of locations which need to be deleted
partPaths = dropPartitionsAndGetLocations(ms, catName, dbname, name, tblPath,
- tbl.getPartitionKeys(), deleteData && !isExternal);
+ tbl.getPartitionKeys(), tableDataShouldBeDeleted);
// Drop any constraints on the table
ms.dropConstraint(catName, dbname, name, null, true);
@@ -2548,7 +2549,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
} finally {
if (!success) {
ms.rollbackTransaction();
- } else if (deleteData && !isExternal) {
+ } else if (tableDataShouldBeDeleted) {
// Data needs deletion. Check if trash may be skipped.
// Delete the data in the partitions which have other locations
deletePartitionData(partPaths, ifPurge, db);
@@ -2568,6 +2569,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
return success;
}
+ private boolean checkTableDataShouldBeDeleted(Table tbl, boolean deleteData) {
+ if (deleteData && isExternal(tbl)) {
+ // External table data can be deleted if EXTERNAL_TABLE_PURGE is true
+ return isExternalTablePurge(tbl);
+ }
+ return deleteData;
+ }
+
/**
* Deletes the data in a table's location, if it fails logs an error
*
@@ -2885,6 +2894,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
return MetaStoreUtils.isExternalTable(table);
}
+ private boolean isExternalTablePurge(Table table) {
+ return MetaStoreUtils.isPropertyTrue(table.getParameters(), MetaStoreUtils.EXTERNAL_TABLE_PURGE);
+ }
+
@Override
@Deprecated
public Table get_table(final String dbname, final String name) throws MetaException,
@@ -4116,7 +4129,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
boolean isArchived = false;
Path archiveParentDir = null;
boolean mustPurge = false;
- boolean isExternalTbl = false;
+ boolean tableDataShouldBeDeleted = false;
boolean isSourceOfReplication = false;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
@@ -4134,7 +4147,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
ms.openTransaction();
part = ms.getPartition(catName, db_name, tbl_name, part_vals);
tbl = get_table_core(catName, db_name, tbl_name);
- isExternalTbl = isExternal(tbl);
+ tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(tbl, deleteData);
firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this));
mustPurge = isMustPurge(envContext, tbl);
@@ -4172,7 +4185,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (!success) {
ms.rollbackTransaction();
} else if (deleteData && ((partPath != null) || (archiveParentDir != null))) {
- if (!isExternalTbl) {
+ if (tableDataShouldBeDeleted) {
if (mustPurge) {
LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash.");
}
@@ -4365,7 +4378,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
} finally {
if (!success) {
ms.rollbackTransaction();
- } else if (deleteData && !isExternal(tbl)) {
+ } else if (checkTableDataShouldBeDeleted(tbl, deleteData)) {
LOG.info( mustPurge?
"dropPartition() will purge partition-directories directly, skipping trash."
: "dropPartition() will move partition-directories to trash-directory.");
http://git-wip-us.apache.org/repos/asf/hive/blob/9326ba69/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index 32850fd..cbe89b6 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -147,6 +147,8 @@ public class MetaStoreUtils {
*/
public static final String DB_EMPTY_MARKER = "!";
+ public static final String EXTERNAL_TABLE_PURGE = "external.table.purge";
+
// Right now we only support one special character '/'.
// More special characters can be added accordingly in the future.
// NOTE:
@@ -567,7 +569,11 @@ public class MetaStoreUtils {
}
public static boolean isExternal(Map<String, String> tableParams){
- return "TRUE".equalsIgnoreCase(tableParams.get("EXTERNAL"));
+ return isPropertyTrue(tableParams, "EXTERNAL");
+ }
+
+ public static boolean isPropertyTrue(Map<String, String> tableParams, String prop) {
+ return "TRUE".equalsIgnoreCase(tableParams.get(prop));
}
// check if stats need to be (re)calculated