You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2020/04/07 21:52:28 UTC
[impala] 02/03: IMPALA-9529: Fix multi-tuple predicates not
assigned in column masking
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5c2cae89f2ba1de55131a261dce7c8e284bb6c0e
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Apr 6 10:14:38 2020 +0800
IMPALA-9529: Fix multi-tuple predicates not assigned in column masking
Column masking is implemented by replacing the masked table with a table
masking view which has masked expressions in its SelectList. However,
nested columns can't be exposed in the SelectList, so we expose them
in the output field of the view in IMPALA-9330. As a result, predicates
that reference both primitive and nested columns of the masked table
become multi-tuple predicates (referencing tuples of the view and the
masked table). Such kinds of predicates are not assigned since they no
longer bound to the view's tuple or the masked table's tuple.
We need to pick up the masked table's tuple id when getting unassigned
predicates for the table masking view. Also need to do this for
assigning predicates to the JoinNode which is the only place that
introduces multi-tuple predicates.
Tests:
- Add tests with multi-tuple predicates referencing nested columns.
- Run CORE tests.
Change-Id: I12f1b59733db5a88324bb0c16085f565edc306b3
Reviewed-on: http://gerrit.cloudera.org:8080/15654
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../java/org/apache/impala/analysis/Analyzer.java | 30 ++-
.../apache/impala/analysis/DescriptorTable.java | 12 +-
.../org/apache/impala/analysis/InlineViewRef.java | 16 +-
.../org/apache/impala/analysis/SlotDescriptor.java | 10 +-
.../java/org/apache/impala/analysis/TableRef.java | 11 +
.../apache/impala/analysis/TupleDescriptor.java | 20 +-
.../java/org/apache/impala/planner/JoinNode.java | 11 +
.../apache/impala/planner/SingleNodePlanner.java | 27 ++-
.../apache/impala/analysis/AnalyzeStmtsTest.java | 2 +-
.../ranger_column_masking_complex_types.test | 264 +++++++++++++++++++++
10 files changed, 371 insertions(+), 32 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index a0772d7..91de1bc 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -64,6 +64,7 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.Pair;
import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.planner.JoinNode;
import org.apache.impala.planner.PlanNode;
import org.apache.impala.rewrite.BetweenToCompoundRule;
import org.apache.impala.rewrite.ConvertToCNFRule;
@@ -1487,8 +1488,24 @@ public class Analyzer {
* Return all unassigned registered conjuncts for node's table ref ids.
* Wrapper around getUnassignedConjuncts(List<TupleId> tupleIds).
*/
- public List<Expr> getUnassignedConjuncts(PlanNode node) {
- return getUnassignedConjuncts(node.getTblRefIds());
+ public final List<Expr> getUnassignedConjuncts(PlanNode node) {
+ List<TupleId> tupleIds = Lists.newCopyOnWriteArrayList(node.getTblRefIds());
+ if (node instanceof JoinNode) {
+ for (TupleId tid : node.getTblRefIds()) {
+ // Pick up TupleId of the masked table since the table masking view and the masked
+ // table are the same in the original query. SlotRefs of table's nested columns
+ // are resolved into table's tuple, but not the table masking view's tuple. As a
+ // result, predicates referencing such nested columns also reference the masked
+ // table's tuple id.
+ TupleDescriptor tuple = getTupleDesc(tid);
+ Preconditions.checkNotNull(tuple);
+ BaseTableRef maskedTable = tuple.getMaskedTable();
+ if (maskedTable != null && maskedTable.exposeNestedColumnsByTableMaskView()) {
+ tupleIds.add(maskedTable.getId());
+ }
+ }
+ }
+ return getUnassignedConjuncts(tupleIds);
}
/**
@@ -2568,21 +2585,22 @@ public class Analyzer {
/**
* Mark all slots that are referenced in exprs as materialized.
+ * Return the affected Tuples.
*/
- public void materializeSlots(List<Expr> exprs) {
+ public Set<TupleDescriptor> materializeSlots(List<Expr> exprs) {
List<SlotId> slotIds = new ArrayList<>();
for (Expr e: exprs) {
Preconditions.checkState(e.isAnalyzed());
e.getIds(null, slotIds);
}
- globalState_.descTbl.markSlotsMaterialized(slotIds);
+ return globalState_.descTbl.markSlotsMaterialized(slotIds);
}
- public void materializeSlots(Expr e) {
+ public Set<TupleDescriptor> materializeSlots(Expr e) {
List<SlotId> slotIds = new ArrayList<>();
Preconditions.checkState(e.isAnalyzed());
e.getIds(null, slotIds);
- globalState_.descTbl.markSlotsMaterialized(slotIds);
+ return globalState_.descTbl.markSlotsMaterialized(slotIds);
}
/**
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java b/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
index ef06c76..7e41e1a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
@@ -39,6 +39,7 @@ import org.apache.impala.thrift.TDescriptorTableSerialized;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* Repository for tuple (and slot) descriptors.
@@ -131,12 +132,17 @@ public class DescriptorTable {
}
/**
- * Marks all slots in list as materialized.
+ * Marks all slots in list as materialized and return the affected Tuples.
*/
- public void markSlotsMaterialized(List<SlotId> ids) {
+ public Set<TupleDescriptor> markSlotsMaterialized(List<SlotId> ids) {
+ Set<TupleDescriptor> affectedTuples = Sets.newHashSet();
for (SlotId id: ids) {
- getSlotDesc(id).setIsMaterialized(true);
+ SlotDescriptor slotDesc = getSlotDesc(id);
+ if (slotDesc.isMaterialized()) continue;
+ slotDesc.setIsMaterialized(true);
+ affectedTuples.add(slotDesc.getParent());
}
+ return affectedTuples;
}
/**
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index f63b689..0482797 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -313,6 +313,11 @@ public class InlineViewRef extends TableRef {
}
fields.add(new StructField(colAlias, selectItemExpr.getType(), null));
}
+
+ // Create the non-materialized tuple and set its type.
+ TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor(
+ getClass().getSimpleName() + " " + getUniqueAlias());
+ result.setIsMaterialized(false);
// If this is a table masking view, the underlying table is wrapped by this so its
// nested columns are not visible to the original query block, because we can't
// expose nested columns in the SelectList. However, we can expose nested columns
@@ -325,6 +330,7 @@ public class InlineViewRef extends TableRef {
if (tblRef instanceof BaseTableRef) {
BaseTableRef baseTbl = (BaseTableRef) tblRef;
FeTable tbl = baseTbl.resolvedPath_.getRootTable();
+ boolean exposeNestedColumn = false;
for (Column col : tbl.getColumnsInHiveOrder()) {
if (!col.getType().isComplexType()) continue;
if (LOG.isTraceEnabled()) {
@@ -332,14 +338,14 @@ public class InlineViewRef extends TableRef {
col.getName(), col.getType().toSql());
}
fields.add(new StructField(col.getName(), col.getType(), null));
+ exposeNestedColumn = true;
+ }
+ if (exposeNestedColumn) {
+ baseTbl.setExposeNestedColumnsByTableMaskView();
}
+ result.setMaskedTable(baseTbl);
}
}
-
- // Create the non-materialized tuple and set its type.
- TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor(
- getClass().getSimpleName() + " " + getUniqueAlias());
- result.setIsMaterialized(false);
result.setType(new StructType(fields));
return result;
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
index e24caf7..1ed175f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
@@ -28,6 +28,8 @@ import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.Type;
import org.apache.impala.thrift.TSlotDescriptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.MoreObjects;
@@ -35,6 +37,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
public class SlotDescriptor {
+ private final static Logger LOG = LoggerFactory.getLogger(SlotDescriptor.class);
private final SlotId id_;
private final TupleDescriptor parent_;
@@ -118,7 +121,12 @@ public class SlotDescriptor {
itemTupleDesc_ = t;
}
public boolean isMaterialized() { return isMaterialized_; }
- public void setIsMaterialized(boolean value) { isMaterialized_ = value; }
+ public void setIsMaterialized(boolean value) {
+ if (isMaterialized_ == value) return;
+ isMaterialized_ = value;
+ LOG.trace("Mark slot(sid={}) of tuple(tid={}) as {}materialized",
+ id_, parent_.getId(), isMaterialized_ ? "" : "non-");
+ }
public boolean getIsNullable() { return isNullable_; }
public void setIsNullable(boolean value) { isNullable_ = value; }
public int getByteSize() { return byteSize_; }
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableRef.java b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
index 47be874..3290edd 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
@@ -132,6 +132,10 @@ public class TableRef extends StmtNode {
// analysis output
protected TupleDescriptor desc_;
+ // true if this table is masked by a table masking view and need to expose its nested
+ // columns via the view.
+ protected boolean exposeNestedColumnsByTableMaskView_ = false;
+
// END: Members that need to be reset()
/////////////////////////////////////////
@@ -197,6 +201,7 @@ public class TableRef extends StmtNode {
allMaterializedTupleIds_ = Lists.newArrayList(other.allMaterializedTupleIds_);
correlatedTupleIds_ = Lists.newArrayList(other.correlatedTupleIds_);
desc_ = other.desc_;
+ exposeNestedColumnsByTableMaskView_ = other.exposeNestedColumnsByTableMaskView_;
}
@Override
@@ -295,6 +300,12 @@ public class TableRef extends StmtNode {
public void setUsingClause(List<String> colNames) { this.usingColNames_ = colNames; }
public TableRef getLeftTblRef() { return leftTblRef_; }
public void setLeftTblRef(TableRef leftTblRef) { this.leftTblRef_ = leftTblRef; }
+ public void setExposeNestedColumnsByTableMaskView() {
+ exposeNestedColumnsByTableMaskView_ = true;
+ }
+ public boolean exposeNestedColumnsByTableMaskView() {
+ return exposeNestedColumnsByTableMaskView_;
+ }
public void setJoinHints(List<PlanHint> hints) {
Preconditions.checkNotNull(hints);
diff --git a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
index 3bdac15..29c459c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
@@ -104,6 +104,11 @@ public class TupleDescriptor {
private int numNullBytes_;
private float avgSerializedSize_; // in bytes; includes serialization overhead
+ // Underlying masked table if this is the tuple of a table masking view.
+ private BaseTableRef maskedTable_ = null;
+ // Tuple of the table masking view that masks this tuple's table.
+ private TupleDescriptor maskedByTuple_ = null;
+
public TupleDescriptor(TupleId id, String debugName) {
id_ = id;
path_ = null;
@@ -190,20 +195,29 @@ public class TupleDescriptor {
return path_.getRootDesc();
}
+ public TupleDescriptor getMaskedByTuple() { return maskedByTuple_; }
+ public BaseTableRef getMaskedTable() { return maskedTable_; }
+ public void setMaskedTable(BaseTableRef table) {
+ Preconditions.checkState(maskedTable_ == null);
+ maskedTable_ = table;
+ table.getDesc().maskedByTuple_ = this;
+ }
+
public String debugString() {
String tblStr = (getTable() == null ? "null" : getTable().getFullName());
List<String> slotStrings = new ArrayList<>();
for (SlotDescriptor slot : slots_) {
slotStrings.add(slot.debugString());
}
- return MoreObjects.toStringHelper(this)
+ MoreObjects.ToStringHelper toStrHelper = MoreObjects.toStringHelper(this)
.add("id", id_.asInt())
.add("name", debugName_)
.add("tbl", tblStr)
.add("byte_size", byteSize_)
.add("is_materialized", isMaterialized_)
- .add("slots", "[" + Joiner.on(", ").join(slotStrings) + "]")
- .toString();
+ .add("slots", "[" + Joiner.on(", ").join(slotStrings) + "]");
+ if (maskedTable_ != null) toStrHelper.add("masks", maskedTable_.getId());
+ return toStrHelper.toString();
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/planner/JoinNode.java b/fe/src/main/java/org/apache/impala/planner/JoinNode.java
index 903e492..bccf900 100644
--- a/fe/src/main/java/org/apache/impala/planner/JoinNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/JoinNode.java
@@ -29,6 +29,7 @@ import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.JoinOperator;
import org.apache.impala.analysis.SlotDescriptor;
import org.apache.impala.analysis.SlotRef;
+import org.apache.impala.analysis.TupleDescriptor;
import org.apache.impala.analysis.TupleId;
import org.apache.impala.catalog.ColumnStats;
import org.apache.impala.catalog.FeTable;
@@ -217,6 +218,16 @@ public abstract class JoinNode extends PlanNode {
// have been collected.
assignConjuncts(analyzer);
createDefaultSmap(analyzer);
+ // Mark slots used by 'conjuncts_' as materialized after substitution. Recompute
+ // memory layout for affected tuples. Note: only tuples of the masked tables could
+ // be affected if they are referenced by multi-tuple predicates.
+ for (TupleDescriptor tuple : analyzer.materializeSlots(conjuncts_)) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Recompute mem layout for " + tuple.debugString());
+ }
+ Preconditions.checkNotNull(tuple.getMaskedByTuple());
+ tuple.recomputeMemLayout();
+ }
assignedConjuncts_ = analyzer.getAssignedConjuncts();
otherJoinConjuncts_ = Expr.substituteList(otherJoinConjuncts_,
getCombinedChildSmap(), analyzer, false);
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index eb19c1b..aaaccb9 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -1151,12 +1151,10 @@ public class SingleNodePlanner {
* If a conjunct is not an On-clause predicate and is safe to propagate it inside the
* inline view, add it to 'evalAfterJoinPreds'.
*/
- private void getConjunctsToInlineView(final Analyzer analyzer,
- final InlineViewRef inlineViewRef, List<Expr> evalInInlineViewPreds,
+ private void getConjunctsToInlineView(final Analyzer analyzer, final String alias,
+ final List<TupleId> tupleIds, List<Expr> evalInInlineViewPreds,
List<Expr> evalAfterJoinPreds) {
- List<Expr> unassignedConjuncts =
- analyzer.getUnassignedConjuncts(inlineViewRef.getId().asList(), true);
- List<TupleId> tupleIds = inlineViewRef.getId().asList();
+ List<Expr> unassignedConjuncts = analyzer.getUnassignedConjuncts(tupleIds, true);
for (Expr e: unassignedConjuncts) {
if (!e.isBoundByTupleIds(tupleIds)) continue;
List<TupleId> tids = new ArrayList<>();
@@ -1176,8 +1174,7 @@ public class SingleNodePlanner {
if (!analyzer.isTrueWithNullSlots(e)) {
evalAfterJoinPreds.add(e);
if (LOG.isTraceEnabled()) {
- LOG.trace(String.format("Can propagate %s to inline view %s",
- e.debugString(), inlineViewRef.getExplicitAlias()));
+ LOG.trace("Can propagate {} to inline view {}", e.debugString(), alias);
}
}
} catch (InternalException ex) {
@@ -1189,8 +1186,7 @@ public class SingleNodePlanner {
continue;
}
if (LOG.isTraceEnabled()) {
- LOG.trace(String.format("Can evaluate %s in inline view %s", e.debugString(),
- inlineViewRef.getExplicitAlias()));
+ LOG.trace("Can evaluate {} in inline view {}", e.debugString(), alias);
}
}
}
@@ -1207,9 +1203,13 @@ public class SingleNodePlanner {
*/
public void migrateConjunctsToInlineView(final Analyzer analyzer,
final InlineViewRef inlineViewRef) throws ImpalaException {
- List<Expr> unassignedConjuncts =
- analyzer.getUnassignedConjuncts(inlineViewRef.getId().asList(), true);
- if (LOG. isTraceEnabled()) {
+ List<TupleId> tids = inlineViewRef.getId().asList();
+ if (inlineViewRef.isTableMaskingView()
+ && inlineViewRef.getUnMaskedTableRef().exposeNestedColumnsByTableMaskView()) {
+ tids.add(inlineViewRef.getUnMaskedTableRef().getId());
+ }
+ List<Expr> unassignedConjuncts = analyzer.getUnassignedConjuncts(tids, true);
+ if (LOG.isTraceEnabled()) {
LOG.trace("unassignedConjuncts: " + Expr.debugString(unassignedConjuncts));
}
if (!canMigrateConjuncts(inlineViewRef)) {
@@ -1223,7 +1223,8 @@ public class SingleNodePlanner {
List<Expr> preds = new ArrayList<>();
List<Expr> evalAfterJoinPreds = new ArrayList<>();
- getConjunctsToInlineView(analyzer, inlineViewRef, preds, evalAfterJoinPreds);
+ getConjunctsToInlineView(analyzer, inlineViewRef.getExplicitAlias(), tids, preds,
+ evalAfterJoinPreds);
unassignedConjuncts.removeAll(preds);
// Migrate the conjuncts by marking the original ones as assigned. They will either
// be ignored if they are identity predicates (e.g. a = a), or be substituted into
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index 7f52760..dba2a85 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -4043,7 +4043,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
testNumberOfMembers(ValuesStmt.class, 0);
// Also check TableRefs.
- testNumberOfMembers(TableRef.class, 21);
+ testNumberOfMembers(TableRef.class, 22);
testNumberOfMembers(BaseTableRef.class, 0);
testNumberOfMembers(InlineViewRef.class, 9);
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
index 667731c..ec9915b 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
@@ -352,3 +352,267 @@ where a2.item = 1 and a3.item = 2 and d2.e = 10 and d3.e = -10
---- TYPES
BIGINT,INT,INT,INT,INT,INT,INT,STRING,INT,STRING
====
+---- QUERY
+# IMPALA-9529: Test predicates that can be resolved to have different tuple ids.
+select id, nested_struct.a from complextypestbl t
+where id = 100 or nested_struct.a = 1;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates that can be resolved to have different tuple ids.
+select id, nested_struct.a from complextypestbl t
+where id + nested_struct.a = 101;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates that can be resolved to have different tuple ids.
+select id, id2 from (
+ select id, id as id2 from functional.alltypestiny
+ union all
+ select id, nested_struct.a as id2 from complextypestbl
+) t
+where id + id2 = 101;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates that can be resolved to have different tuple ids.
+with v as (
+ select id, nested_struct.a as id2 from complextypestbl
+)
+select id, id2 from v
+where id + id2 = 101 or (id = 200 and id2 is null);
+---- RESULTS
+100,1
+200,NULL
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids.
+select t.id, t.nested_struct.a
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+where tiny.id + t.id + t.nested_struct.a = 201;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids.
+select t.id, t.nested_struct.a
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id + t.id + t.nested_struct.a = 201;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids. Make sure nested columns inside
+# the column masking view are materialized.
+select count(1)
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+where (tiny.id + t.id + t.nested_struct.a) is null;
+---- RESULTS
+5
+---- TYPES
+BIGINT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids in GroupBy clause.
+select count(1)
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+group by tiny.id + t.id + t.nested_struct.a;
+---- RESULTS
+1
+5
+1
+---- TYPES
+BIGINT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids in Having clause.
+select count(tiny.id + t.id + t.nested_struct.a)
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+group by t.id
+having count(tiny.id + t.id + t.nested_struct.a) = 1
+---- RESULTS
+1
+1
+---- TYPES
+BIGINT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids in OrderBy clause.
+select t.id
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+order by tiny.id + t.id + t.nested_struct.a, t.id;
+---- RESULTS: VERIFY_IS_EQUAL
+100
+700
+200
+300
+400
+500
+600
+---- TYPES
+BIGINT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids in analytic query.
+select t.id, rank() over(order by t.id)
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+where tiny.id + t.id + t.nested_struct.a is null;
+---- RESULTS
+200,1
+300,2
+400,3
+500,4
+600,5
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids.
+select t.id, t.nested_struct.a
+from functional.alltypestiny tiny
+ left join complextypestbl t
+ on tiny.id = t.id
+where tiny.id + t.id + t.nested_struct.a = 201;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids.
+select t.id, t.nested_struct.a
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+ join functional.alltypestiny tiny2
+ on t.id = tiny2.id
+where tiny.id + t.id + t.nested_struct.a + tiny2.id = 301;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids.
+select t.id, t.nested_struct.a
+from functional.alltypestiny tiny
+ left join complextypestbl t
+ on tiny.id = t.id
+ join functional.alltypestiny tiny2
+ on t.id = tiny2.id
+where tiny.id + t.id + t.nested_struct.a + tiny2.id = 301;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# IMPALA-9529: Test predicates with multiple tuple ids.
+with v as (
+ select t.id, t.nested_struct.a
+ from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+ where tiny.id + t.id + t.nested_struct.a = 201
+)
+select t.id, t.nested_struct.a from v
+ join complextypestbl t
+ on v.id = t.id
+where v.id + t.id + t.nested_struct.a = 201
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+select count(distinct id), count(distinct nested_struct.a) from complextypestbl
+---- RESULTS
+8,3
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+select count(distinct id), count(distinct nested_struct.a) from complextypestbl
+where id + nested_struct.a = 101;
+---- RESULTS
+1,1
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+select count(distinct tiny.int_col), count(distinct t.id), count(distinct t.nested_struct.a)
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+---- RESULTS
+2,7,2
+---- TYPES
+BIGINT,BIGINT,BIGINT
+====
+---- QUERY
+select count(distinct tiny.int_col), count(distinct t.id), count(distinct t.nested_struct.a)
+from functional.alltypestiny tiny
+ join complextypestbl t
+ on tiny.id = t.id
+where tiny.int_col + t.id + t.nested_struct.a = 102
+---- RESULTS
+1,1,1
+---- TYPES
+BIGINT,BIGINT,BIGINT
+====
+---- QUERY
+select tiny.id, t0.nested_struct.a
+from functional.alltypestiny tiny
+ join complextypestbl t0 on tiny.id = t0.id
+ join complextypestbl t1 on tiny.id = t1.id
+ join complextypestbl t2 on tiny.id = t2.id
+ join complextypestbl t3 on tiny.id = t3.id
+where tiny.id + t2.id + t3.nested_struct.a >= 201
+---- RESULTS
+100,1
+700,7
+---- TYPES
+INT,INT
+====
+---- QUERY
+select tiny.id, t0.nested_struct.a
+from functional.alltypestiny tiny
+ join complextypestbl t0 on tiny.id = t0.id
+ join complextypestbl t1 on tiny.id = t1.id
+ join complextypestbl t2 on tiny.id = t2.id
+ join complextypestbl t3 on tiny.id = t3.id
+where (t2.id + t3.id + t3.nested_struct.a = 201 or t2.nested_struct.a is null)
+ and tiny.id + t0.id + t3.nested_struct.a >= 201
+---- RESULTS
+100,1
+---- TYPES
+INT,INT
+====