You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by sa...@apache.org on 2017/02/14 06:02:05 UTC
[1/2] incubator-impala git commit: IMPALA-4263: Fix wrong ommission
of agg/analytic hash exchanges.
Repository: incubator-impala
Updated Branches:
refs/heads/master f982c3f76 -> 1f80396b2
IMPALA-4263: Fix wrong ommission of agg/analytic hash exchanges.
The bug: Our detection of partition compatibility for
grouping aggregations and analytic functions did not take into
account the effect of outer joins within the same fragment.
As a result, we used to incorrectly omit a required hash exchange.
For example, a hash exchange + merge phase is required if the
grouping expressions of an aggregation reference tuples
that are made nullable within the same fragment. The exchange is
needed to bring together NULLs produced by outer-join non-matches.
The fix: Check that the grouping/partition exprs do not reference
tuples that are made nullable within the same fragment.
Testing: Planner tests pass locally.
Change-Id: I121222179378e56836422a69451d840a012c9e54
Reviewed-on: http://gerrit.cloudera.org:8080/5774
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Alex Behm <al...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/cd153d66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/cd153d66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/cd153d66
Branch: refs/heads/master
Commit: cd153d66dc20ebea7a9de7a2ef1c627fee45253c
Parents: f982c3f
Author: Alex Behm <al...@cloudera.com>
Authored: Wed Jan 4 22:19:38 2017 -0800
Committer: Alex Behm <al...@cloudera.com>
Committed: Mon Feb 13 23:00:01 2017 +0000
----------------------------------------------------------------------
.../impala/planner/DistributedPlanner.java | 38 ++++---
.../org/apache/impala/planner/PlanFragment.java | 35 ++++--
.../queries/PlannerTest/aggregation.test | 106 +++++++++++++++++++
.../queries/PlannerTest/analytic-fns.test | 35 ++++++
4 files changed, 190 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/cd153d66/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
index dd03ac8..71ddd5a 100644
--- a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
@@ -19,9 +19,7 @@ package org.apache.impala.planner;
import java.util.ArrayList;
import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.Set;
import org.apache.impala.analysis.AggregateInfo;
import org.apache.impala.analysis.AnalysisContext;
@@ -30,12 +28,17 @@ import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.InsertStmt;
import org.apache.impala.analysis.JoinOperator;
import org.apache.impala.analysis.QueryStmt;
+import org.apache.impala.analysis.TupleId;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.planner.JoinNode.DistributionMode;
import org.apache.impala.planner.RuntimeFilterGenerator.RuntimeFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
@@ -785,20 +788,23 @@ public class DistributedPlanner {
DataPartition parentPartition = null;
if (hasGrouping) {
- // the parent fragment is partitioned on the grouping exprs;
- // substitute grouping exprs to reference the *output* of the agg, not the input
List<Expr> partitionExprs = node.getAggInfo().getPartitionExprs();
if (partitionExprs == null) partitionExprs = groupingExprs;
- partitionExprs = Expr.substituteList(partitionExprs,
- node.getAggInfo().getIntermediateSmap(), ctx_.getRootAnalyzer(), false);
boolean childHasCompatPartition = ctx_.getRootAnalyzer().equivSets(partitionExprs,
childFragment.getDataPartition().getPartitionExprs());
- if (childHasCompatPartition) {
- // The data is already partitioned on the required expressions, we can just do
- // the aggregation in the child fragment without an extra merge step.
+ if (childHasCompatPartition && !childFragment.refsNullableTupleId(partitionExprs)) {
+ // The data is already partitioned on the required expressions. We can do the
+ // aggregation in the child fragment without an extra merge step.
+ // An exchange+merge step is required if the grouping exprs reference a tuple
+ // that is made nullable in 'childFragment' to bring NULLs from outer-join
+ // non-matches together.
childFragment.addPlanRoot(node);
return childFragment;
}
+ // the parent fragment is partitioned on the grouping exprs;
+ // substitute grouping exprs to reference the *output* of the agg, not the input
+ partitionExprs = Expr.substituteList(partitionExprs,
+ node.getAggInfo().getIntermediateSmap(), ctx_.getRootAnalyzer(), false);
parentPartition = DataPartition.hashPartitioned(partitionExprs);
} else {
// the parent fragment is unpartitioned
@@ -973,12 +979,16 @@ public class DistributedPlanner {
Preconditions.checkState(sortNode.isAnalyticSort());
PlanFragment analyticFragment = childFragment;
if (sortNode.getInputPartition() != null) {
- // make sure the childFragment's output is partitioned as required by the sortNode
sortNode.getInputPartition().substitute(
childFragment.getPlanRoot().getOutputSmap(), ctx_.getRootAnalyzer());
- if (!childFragment.getDataPartition().equals(sortNode.getInputPartition())) {
- analyticFragment =
- createParentFragment(childFragment, sortNode.getInputPartition());
+ // Make sure the childFragment's output is partitioned as required by the sortNode.
+ // Even if the fragment and the sort partition exprs are equal, an exchange is
+ // required if the sort partition exprs reference a tuple that is made nullable in
+ // 'childFragment' to bring NULLs from outer-join non-matches together.
+ DataPartition sortPartition = sortNode.getInputPartition();
+ if (!childFragment.getDataPartition().equals(sortPartition)
+ || childFragment.refsNullableTupleId(sortPartition.getPartitionExprs())) {
+ analyticFragment = createParentFragment(childFragment, sortPartition);
}
}
analyticFragment.addPlanRoot(sortNode);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/cd153d66/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
index 0e5134f..168e485 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
@@ -18,30 +18,26 @@
package org.apache.impala.planner;
import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.Set;
import org.apache.impala.analysis.Analyzer;
-import org.apache.impala.analysis.BinaryPredicate;
import org.apache.impala.analysis.Expr;
-import org.apache.impala.analysis.JoinOperator;
-import org.apache.impala.analysis.SlotRef;
-import org.apache.impala.catalog.HdfsFileFormat;
-import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.analysis.TupleId;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.NotImplementedException;
import org.apache.impala.common.TreeNode;
-import org.apache.impala.planner.JoinNode.DistributionMode;
import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TPartitionType;
-import org.apache.impala.thrift.TPlan;
import org.apache.impala.thrift.TPlanFragment;
import org.apache.impala.thrift.TPlanFragmentTree;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* PlanFragments form a tree structure via their ExchangeNodes. A tree of fragments
@@ -385,4 +381,23 @@ public class PlanFragment extends TreeNode<PlanFragment> {
for (PlanFragment child: getChildren()) child.verifyTree();
}
+
+ /**
+ * Returns true if 'exprs' reference a tuple that is made nullable in this fragment,
+ * but not in any of its input fragments.
+ */
+ public boolean refsNullableTupleId(List<Expr> exprs) {
+ Preconditions.checkNotNull(planRoot_);
+ List<TupleId> tids = Lists.newArrayList();
+ for (Expr e: exprs) e.getIds(tids, null);
+ Set<TupleId> nullableTids = Sets.newHashSet(planRoot_.getNullableTupleIds());
+ // Remove all tuple ids that were made nullable in an input fragment.
+ List<ExchangeNode> exchNodes = Lists.newArrayList();
+ planRoot_.collect(ExchangeNode.class, exchNodes);
+ for (ExchangeNode exchNode: exchNodes) {
+ nullableTids.removeAll(exchNode.getNullableTupleIds());
+ }
+ for (TupleId tid: tids) if (nullableTids.contains(tid)) return true;
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/cd153d66/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test b/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test
index b4fe75b..dab490e 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test
@@ -1167,3 +1167,109 @@ PLAN-ROOT SINK
partitions=1/1 files=3 size=193.61MB
runtime filters: RF002 -> l_orderkey, RF003 -> l_returnflag
====
+# IMPALA-4263: Grouping agg needs a merge step because the grouping exprs reference a
+# tuple that is made nullable in the join fragment.
+select /* +straight_join */ t2.id, count(*)
+from functional.alltypes t1
+left outer join /* +shuffle */ functional.alltypessmall t2
+ on t1.id = t2.id
+group by t2.id
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+08:EXCHANGE [UNPARTITIONED]
+|
+07:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: t2.id
+|
+06:EXCHANGE [HASH(t2.id)]
+|
+03:AGGREGATE [STREAMING]
+| output: count(*)
+| group by: t2.id
+|
+02:HASH JOIN [LEFT OUTER JOIN, PARTITIONED]
+| hash predicates: t1.id = t2.id
+|
+|--05:EXCHANGE [HASH(t2.id)]
+| |
+| 01:SCAN HDFS [functional.alltypessmall t2]
+| partitions=4/4 files=4 size=6.32KB
+|
+04:EXCHANGE [HASH(t1.id)]
+|
+00:SCAN HDFS [functional.alltypes t1]
+ partitions=24/24 files=24 size=478.45KB
+====
+# IMPALA-4263: Grouping agg is placed in the join fragment and has no merge step.
+select /* +straight_join */ t1.id, count(*)
+from functional.alltypes t1
+left outer join /* +shuffle */ functional.alltypessmall t2
+ on t1.id = t2.id
+group by t1.id
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+06:EXCHANGE [UNPARTITIONED]
+|
+03:AGGREGATE [FINALIZE]
+| output: count(*)
+| group by: t1.id
+|
+02:HASH JOIN [LEFT OUTER JOIN, PARTITIONED]
+| hash predicates: t1.id = t2.id
+|
+|--05:EXCHANGE [HASH(t2.id)]
+| |
+| 01:SCAN HDFS [functional.alltypessmall t2]
+| partitions=4/4 files=4 size=6.32KB
+|
+04:EXCHANGE [HASH(t1.id)]
+|
+00:SCAN HDFS [functional.alltypes t1]
+ partitions=24/24 files=24 size=478.45KB
+====
+# IMPALA-4263: Grouping agg is placed in the second join fragment and has no merge step.
+# The grouping exprs reference a nullable tuple (t2), but that tuple is made nullable in
+# the first join fragment, so it's correct to place the the aggregation in the second
+# join fragment without a merge step.
+select /* +straight_join */ t2.id, count(*)
+from functional.alltypes t1
+left outer join /* +shuffle */ functional.alltypessmall t2
+ on t1.int_col = t2.int_col
+left outer join /* +shuffle */ functional.alltypestiny t3
+ on t2.id = t3.id
+group by t2.id
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+10:EXCHANGE [UNPARTITIONED]
+|
+05:AGGREGATE [FINALIZE]
+| output: count(*)
+| group by: t2.id
+|
+04:HASH JOIN [LEFT OUTER JOIN, PARTITIONED]
+| hash predicates: t2.id = t3.id
+|
+|--09:EXCHANGE [HASH(t3.id)]
+| |
+| 02:SCAN HDFS [functional.alltypestiny t3]
+| partitions=4/4 files=4 size=460B
+|
+08:EXCHANGE [HASH(t2.id)]
+|
+03:HASH JOIN [LEFT OUTER JOIN, PARTITIONED]
+| hash predicates: t1.int_col = t2.int_col
+|
+|--07:EXCHANGE [HASH(t2.int_col)]
+| |
+| 01:SCAN HDFS [functional.alltypessmall t2]
+| partitions=4/4 files=4 size=6.32KB
+|
+06:EXCHANGE [HASH(t1.int_col)]
+|
+00:SCAN HDFS [functional.alltypes t1]
+ partitions=24/24 files=24 size=478.45KB
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/cd153d66/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test b/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
index 8c25730..de5e4cb 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/analytic-fns.test
@@ -2327,3 +2327,38 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional.alltypes]
partitions=24/24 files=24 size=478.45KB
====
+# IMPALA-4263: Analytic function needs a hash exchange because the partition exprs
+# reference a tuple that is made nullable in the join fragment.
+select /* +straight_join */ count(*) over (partition by t1.id)
+from functional.alltypes t1
+right outer join /* +shuffle */ functional.alltypessmall t2
+ on t1.id = t2.id
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+08:EXCHANGE [UNPARTITIONED]
+|
+04:ANALYTIC
+| functions: count(*)
+| partition by: t1.id
+|
+03:SORT
+| order by: id ASC NULLS FIRST
+|
+07:EXCHANGE [HASH(t1.id)]
+|
+02:HASH JOIN [RIGHT OUTER JOIN, PARTITIONED]
+| hash predicates: t1.id = t2.id
+| runtime filters: RF000 <- t2.id
+|
+|--06:EXCHANGE [HASH(t2.id)]
+| |
+| 01:SCAN HDFS [functional.alltypessmall t2]
+| partitions=4/4 files=4 size=6.32KB
+|
+05:EXCHANGE [HASH(t1.id)]
+|
+00:SCAN HDFS [functional.alltypes t1]
+ partitions=24/24 files=24 size=478.45KB
+ runtime filters: RF000 -> t1.id
+====
[2/2] incubator-impala git commit: IMPALA-4897: AnalysisException:
specified cache pool does not exist
Posted by sa...@apache.org.
IMPALA-4897: AnalysisException: specified cache pool does not exist
A few tests were added with IMPALA-1670 that made use of HDFS caching.
This patch moves these tests to a new file and only executes them
when the default filesystem is HDFS.
There was also a bug where the tests used absolute locations instead
of locations relative to the table they were in which could easily
collide with locations of other tables if they raced. That has been
fixed too.
Also added a testcase for testing alter table ADD multiple PARTITIONS
for non-HDFS filesystems.
Change-Id: Iefe61556bc28ae320f3f41fdc930d37b258d970a
Reviewed-on: http://gerrit.cloudera.org:8080/5972
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1f80396b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1f80396b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1f80396b
Branch: refs/heads/master
Commit: 1f80396b20529e6e77d4edbd7d3d62b29a106568
Parents: cd153d6
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Fri Feb 10 16:12:06 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Feb 14 05:56:33 2017 +0000
----------------------------------------------------------------------
.../QueryTest/alter-table-hdfs-caching.test | 110 +++++++++++++++++++
.../queries/QueryTest/alter-table.test | 80 +++-----------
tests/metadata/test_ddl.py | 5 +
3 files changed, 129 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1f80396b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test
new file mode 100644
index 0000000..71b61f9
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test
@@ -0,0 +1,110 @@
+# This test file covers alter table statements that add multiple partitions and use hdfs caching
+====
+---- QUERY
+create table i1670B_alter (s string) partitioned by (i integer);
+alter table i1670B_alter add
+partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670B_alter/i1'
+cached in 'testPool' with replication=3
+partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670B_alter/i2'
+partition (i=3) uncached;
+show partitions i1670B_alter;
+---- RESULTS
+'1',-1,0,'0B','0B','3','TEXT','false',regex:.*/i1
+'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i2
+'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3
+'Total',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+# IMPALA-1670: Set up i1670C_alter table for the next test case.
+create table i1670C_alter (s string) partitioned by (i integer);
+alter table i1670C_alter add
+partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i2A'
+cached in 'testPool' with replication=2
+partition (i=4) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i4A' uncached;
+show partitions i1670C_alter;
+---- RESULTS
+'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A
+'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A
+'Total',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+# IMPALA-1670: If 'IF NOT EXISTS' is used ALTER TABLE ADD PARTITION works with preexisting
+# partitions. Location and caching options of existing partitions are not modified.
+alter table i1670C_alter add if not exists
+partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i1B'
+partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i2B' uncached
+partition (i=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i3B'
+cached in 'testPool' with replication=3
+partition (i=4) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i4B'
+cached in 'testPool' with replication=4;
+show partitions i1670C_alter;
+---- RESULTS
+'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i1B
+'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A
+'3',-1,0,'0B','0B','3','TEXT','false',regex:.*/i3B
+'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A
+'Total',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+# IMPALA-1670: Partitions without explicit CACHED IN/UNCACHED clause inherit cacheop from
+# the parent table
+create table i1670D_alter (s string) partitioned by (i integer)
+cached in 'testPool' with replication=7;
+alter table i1670D_alter add
+partition (i=1) cached in 'testPool' with replication=5
+partition (i=2)
+partition (i=3) uncached
+partition (i=4);
+show partitions i1670D_alter;
+---- RESULTS
+'1',-1,0,'0B','0B','5','TEXT','false',regex:.*/i=1
+'2',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=2
+'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3
+'4',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=4
+'Total',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+# IMPALA-1670: After INVALIDATE METADATA Impala can access previously added partitions and
+# partition data.
+create table i1670E_alter (a int) partitioned by (x int);
+alter table i1670E_alter add partition (x=1)
+partition (x=2) uncached
+partition (x=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670E_alter/x3'
+cached in 'testPool' with replication=7;
+insert into i1670E_alter partition(x=1) values (1), (2), (3);
+insert into i1670E_alter partition(x=2) values (1), (2), (3), (4);
+insert into i1670E_alter partition(x=3) values (1);
+invalidate metadata i1670E_alter;
+====
+---- QUERY
+show partitions i1670E_alter;
+---- RESULTS
+'1',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=1
+'2',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=2
+'3',-1,1,regex:.*,regex:.*,'7','TEXT','false',regex:.*/x3
+'Total',-1,3,regex:.*,regex:.*,'','','',''
+---- TYPES
+STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+select x, a from i1670E_alter order by x, a;
+---- RESULTS
+1,1
+1,2
+1,3
+2,1
+2,2
+2,3
+2,4
+3,1
+---- TYPES
+INT, INT
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1f80396b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
index e067bc1..ecb6d6e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
@@ -1007,12 +1007,12 @@ INT, STRING
# IMPALA-1670: Support adding multiple partitions in ALTER TABLE ADD PARTITION
create table i1670A_alter (s string) partitioned by (i integer);
alter table i1670A_alter add
-partition (i=1) location '/i1' cached in 'testPool' with replication=3
-partition (i=2) location '/i2'
+partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter/i1'
+partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter/i2'
partition (i=3) uncached;
show partitions i1670A_alter;
---- RESULTS
-'1',-1,0,'0B','0B','3','TEXT','false',regex:.*/i1
+'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i1
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i2
'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3
'Total',-1,0,'0B','0B','','','',''
@@ -1020,81 +1020,29 @@ show partitions i1670A_alter;
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
-# IMPALA-1670: Set up i1670C_alter table for the next test case.
-create table i1670C_alter (s string) partitioned by (i integer);
-alter table i1670C_alter add
-partition (i=2) location '/i2A' cached in 'testPool' with replication=2
-partition (i=4) location '/i4A' uncached;
-show partitions i1670C_alter;
----- RESULTS
-'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A
-'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A
-'Total',-1,0,'0B','0B','','','',''
----- TYPES
-STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-# IMPALA-1670: If 'IF NOT EXISTS' is used ALTER TABLE ADD PARTITION works with preexisting
-# partitions. Location and caching options of existing partitions are not modified.
-alter table i1670C_alter add if not exists
-partition (i=1) location '/i1B'
-partition (i=2) location '/i2B' uncached
-partition (i=3) location '/i3B' cached in 'testPool' with replication=3
-partition (i=4) location '/i4B' cached in 'testPool' with replication=4;
-show partitions i1670C_alter;
----- RESULTS
-'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i1B
-'2',-1,0,'0B','0B','2','TEXT','false',regex:.*/i2A
-'3',-1,0,'0B','0B','3','TEXT','false',regex:.*/i3B
-'4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i4A
-'Total',-1,0,'0B','0B','','','',''
----- TYPES
-STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-# IMPALA-1670: Partitions without explicit CACHED IN/UNCACHED clause inherit cacheop from
-# the parent table
-create table i1670D_alter (s string) partitioned by (i integer)
-cached in 'testPool' with replication=7;
-alter table i1670D_alter add
-partition (i=1) cached in 'testPool' with replication=5
-partition (i=2)
-partition (i=3) uncached
-partition (i=4);
-show partitions i1670D_alter;
----- RESULTS
-'1',-1,0,'0B','0B','5','TEXT','false',regex:.*/i=1
-'2',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=2
-'3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*/i=3
-'4',-1,0,'0B','0B','7','TEXT','false',regex:.*/i=4
-'Total',-1,0,'0B','0B','','','',''
----- TYPES
-STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
# IMPALA-1670: After INVALIDATE METADATA Impala can access previously added partitions and
# partition data.
-create table i1670E_alter (a int) partitioned by (x int);
-alter table i1670E_alter add partition (x=1)
+create table i1670A_alter_1 (a int) partitioned by (x int);
+alter table i1670A_alter_1 add partition (x=1)
partition (x=2) uncached
-partition (x=3) location '/x3' cached in 'testPool' with replication=7;
-insert into i1670E_alter partition(x=1) values (1), (2), (3);
-insert into i1670E_alter partition(x=2) values (1), (2), (3), (4);
-insert into i1670E_alter partition(x=3) values (1);
-invalidate metadata i1670E_alter;
+partition (x=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter_1/x3';
+insert into i1670A_alter_1 partition(x=1) values (1), (2), (3);
+insert into i1670A_alter_1 partition(x=2) values (1), (2), (3), (4);
+insert into i1670A_alter_1 partition(x=3) values (1);
+invalidate metadata i1670A_alter_1;
====
---- QUERY
-show partitions i1670E_alter;
+show partitions i1670A_alter_1;
---- RESULTS
'1',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=1
'2',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x=2
-'3',-1,1,regex:.*,regex:.*,'7','TEXT','false',regex:.*/x3
-'Total',-1,3,regex:.*,regex:.*,'','','',''
+'3',-1,1,regex:.*,'NOT CACHED','NOT CACHED','TEXT','false',regex:.*/x3
+'Total',-1,3,regex:.*,'0B','','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
-select x, a from i1670E_alter order by x, a;
+select x, a from i1670A_alter_1 order by x, a;
---- RESULTS
1,1
1,2
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1f80396b/tests/metadata/test_ddl.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 40c7a99..37ca7cd 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -253,6 +253,11 @@ class TestDdlStatements(TestDdlBase):
file_data='1984')
self.run_test_case('QueryTest/alter-table', vector, use_db=unique_database,
multiple_impalad=self._use_multiple_impalad(vector))
+ # The following tests require HDFS caching which is supported only in the HDFS
+ # filesystem.
+ if IS_HDFS:
+ self.run_test_case('QueryTest/alter-table-hdfs-caching', vector,
+ use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
@UniqueDatabase.parametrize(sync_ddl=True)
def test_alter_set_column_stats(self, vector, unique_database):